/*
This file is part of the Notesnook project (https://notesnook.com/)
Copyright (C) 2023 Streetwriters (Private) Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
*/
import { CompactID, JCID, OneNoteReader } from "../reader";
import { ExtendedGUID, GUID } from "../utils/guid";
import { OneStore } from "./index";
import { FileNode } from "./file-node";
import { FileNodeID } from "./file-node-types";
import { FileNodeList } from "./file-node-list";
import { ObjectSpaceObjectPropSet } from "./file-node-types/shared/ObjectSpaceObjectPropSet";
import { decode } from "../utils/reader";
import { parseEncryptionXml, decryptObjectData, type EncryptionInfo } from "../crypto";
export function extendedGuidToKey(id: ExtendedGUID): string {
return id.toString();
}
/**
* The role of a root object reference. See [MS-ONE] 2.1.8.
*/
export enum RootRole {
DefaultContent = 1,
MetadataRoot = 2,
VersionMetadataRoot = 4
}
type IdMapping = Map;
function resolveCompactId(
mapping: IdMapping,
id: CompactID
): ExtendedGUID | undefined {
const guid = mapping.get(id.guidIndex);
if (!guid) return undefined;
return new ExtendedGUID(guid, id.n);
}
export type OneNoteObject = {
contextId: string;
jcid: JCID;
propSet: ObjectSpaceObjectPropSet;
mapping: Map;
fileData?: Uint8Array;
fileDataReference?: string;
fileExtension?: string;
/** Object Data Container State — 0 = not encrypted, 1+ = encrypted. */
odcs: number;
/** When `odcs != 0`, this holds the raw encrypted property set bytes. */
encryptedData?: Uint8Array;
};
type Revision = {
id: string;
parentId: string;
role: number;
context?: string;
idMap: IdMapping;
roots: Map;
objects: Map;
encryptionRef?: { stp: number; cb: number };
};
function cloneIdMap(map: IdMapping): IdMapping {
return new Map(map);
}
function mergeIdMap(target: IdMapping, other: IdMapping) {
for (const [key, value] of other) target.set(key, value);
}
function walkNodes(list: FileNodeList): FileNode[] {
const nodes: FileNode[] = [];
for (const fragment of list.fragments) {
for (const node of fragment.rgFileNodes) {
if (
node.is(FileNodeID.ChunkTerminatorFND) ||
node.is(FileNodeID.UnknownFND)
)
continue;
nodes.push(node);
}
}
return nodes;
}
const ACTIVE_CONTENT_ROLE = 1;
const NIL_EXTENDED_GUID_KEY = ExtendedGUID.nil().toString();
function parseGlobalIdTable(
nodes: FileNode[],
index: number,
parentMapping?: IdMapping
): { mapping: IdMapping; nextIndex: number } {
// nodes[index] is GlobalIdTableStartFNDX or GlobalIdTableStart2FND
let i = index + 1;
const mapping: IdMapping = new Map();
for (; i < nodes.length; ++i) {
const node = nodes[i];
if (node.is(FileNodeID.GlobalIdTableEndFNDX)) {
return { mapping, nextIndex: i + 1 };
} else if (node.is(FileNodeID.GlobalIdTableEntryFNDX)) {
mapping.set(node.data.index, node.data.guid);
} else if (node.is(FileNodeID.GlobalIdTableEntry2FNDX)) {
const guid = parentMapping?.get(node.data.iIndexMapFrom);
if (!guid)
throw new Error(
`GlobalIdTableEntry2FNDX references index ${node.data.iIndexMapFrom} that is not present in the dependency revision's global ID table.`
);
mapping.set(node.data.iIndexMapTo, guid);
} else if (node.is(FileNodeID.GlobalIdTableEntry3FNDX)) {
const { iIndexCopyFromStart, cEntriesToCopy, iIndexCopyToStart } =
node.data;
for (let offset = 0; offset < cEntriesToCopy; ++offset) {
const guid = parentMapping?.get(iIndexCopyFromStart + offset);
if (!guid)
throw new Error(
`GlobalIdTableEntry3FNDX references index ${
iIndexCopyFromStart + offset
} that is not present in the dependency revision's global ID table.`
);
mapping.set(iIndexCopyToStart + offset, guid);
}
} else {
throw new Error(
`Unexpected node (${FileNodeID[node.FileNodeID]}) while parsing global ID table.`
);
}
}
throw new Error("Global ID table was not terminated.");
}
function readPropertySet(
reader: OneNoteReader,
ref: { stp: number }
): ObjectSpaceObjectPropSet {
reader.seek(ref.stp);
return ObjectSpaceObjectPropSet(reader);
}
function readPropertySetAt(
reader: OneNoteReader,
ref: { stp: number; cb: number }
): Uint8Array {
reader.seek(ref.stp);
return reader.deserializeBytes(ref.cb);
}
function parseObjectDeclaration(
node: FileNode,
reader: OneNoteReader,
mapping: IdMapping,
contextId: string
): { key: string; object: OneNoteObject } | undefined {
if (
node.is(FileNodeID.ObjectDeclaration2RefCountFND) ||
node.is(FileNodeID.ObjectDeclaration2LargeRefCountFND)
) {
const declaration = node.data;
const id = resolveCompactId(mapping, declaration.body.oid);
if (!id)
throw new Error(
`Missing mapping for object ID (index: ${declaration.body.oid.guidIndex}).`
);
const odcs = declaration.body.odcs;
if (odcs !== 0) {
// Encrypted object — store raw data for later decryption.
return {
key: extendedGuidToKey(id),
object: {
contextId,
jcid: declaration.body.jcid,
propSet: emptyPropertySet(),
mapping,
odcs,
encryptedData: readPropertySetAt(reader, declaration.BlobRef)
}
};
}
return {
key: extendedGuidToKey(id),
object: {
contextId,
jcid: declaration.body.jcid,
propSet: readPropertySet(reader, declaration.BlobRef),
mapping,
odcs
}
};
} else if (
node.is(FileNodeID.ReadOnlyObjectDeclaration2RefCountFND) ||
node.is(FileNodeID.ReadOnlyObjectDeclaration2LargeRefCountFND)
) {
const declaration = node.data.base;
const id = resolveCompactId(mapping, declaration.body.oid);
if (!id)
throw new Error(
`Missing mapping for object ID (index: ${declaration.body.oid.guidIndex}).`
);
const odcs = declaration.body.odcs;
if (odcs !== 0) {
return {
key: extendedGuidToKey(id),
object: {
contextId,
jcid: declaration.body.jcid,
propSet: emptyPropertySet(),
mapping,
odcs,
encryptedData: readPropertySetAt(reader, declaration.BlobRef)
}
};
}
return {
key: extendedGuidToKey(id),
object: {
contextId,
jcid: declaration.body.jcid,
propSet: readPropertySet(reader, declaration.BlobRef),
mapping,
odcs
}
};
} else if (
node.is(FileNodeID.ObjectDeclarationFileData3RefCountFND) ||
node.is(FileNodeID.ObjectDeclarationFileData3LargeRefCountFND)
) {
const declaration = node.data;
const id = resolveCompactId(mapping, declaration.oid);
if (!id)
throw new Error(
`Missing mapping for object ID (index: ${declaration.oid.guidIndex}).`
);
return {
key: extendedGuidToKey(id),
object: {
contextId,
jcid: declaration.jcid,
propSet: emptyPropertySet(),
mapping,
odcs: 0,
fileDataReference: declaration.FileDataReference.StringData,
fileExtension: declaration.Extension.StringData
}
};
} else if (
node.is(FileNodeID.ObjectDeclarationWithRefCountFNDX) ||
node.is(FileNodeID.ObjectDeclarationWithRefCount2FNDX)
) {
const declaration = node.data;
const id = resolveCompactId(mapping, declaration.body.oid);
if (!id)
throw new Error(
`Missing mapping for object ID (index: ${declaration.body.oid.guidIndex}).`
);
return {
key: extendedGuidToKey(id),
object: {
contextId,
jcid: jcidFromId(declaration.body.jci | 0x20000),
propSet: readPropertySet(reader, declaration.ObjectRef),
mapping,
odcs: 0
}
};
}
return undefined;
}
function jcidFromId(id: number): JCID {
return {
id,
index: id & 0xffff,
isBinary: ((id >> 16) & 0x1) === 1,
isPropertySet: ((id >> 17) & 0x1) === 1,
isGraphNode: ((id >> 18) & 0x1) === 1,
isFileData: ((id >> 19) & 0x1) === 1,
isReadOnly: ((id >> 20) & 0x1) === 1
};
}
function emptyPropertySet(): ObjectSpaceObjectPropSet {
return {
OIDs: {
header: {
count: 0,
ExtendedStreamsPresent: false,
OsidStreamNotPresent: false
},
body: []
},
body: { cProperties: 0, rgData: [], rgPrids: [] }
};
}
function parseRevision(
nodes: FileNode[],
index: number,
reader: OneNoteReader,
revisions: Map,
contextId: string
): { revision: Revision; nextIndex: number } {
const start = nodes[index];
let id: ExtendedGUID;
let parentId: ExtendedGUID;
let role: number;
let context: string | undefined;
if (start.is(FileNodeID.RevisionManifestStart4FND)) {
id = start.data.rid;
parentId = start.data.ridDependent;
role = start.data.revisionRole;
context = undefined;
} else if (start.is(FileNodeID.RevisionManifestStart6FND)) {
id = start.data.rid;
parentId = start.data.ridDependent;
role = start.data.revisionRole;
context = undefined;
} else if (start.is(FileNodeID.RevisionManifestStart7FND)) {
id = start.data.base.rid;
parentId = start.data.base.ridDependent;
role = start.data.base.revisionRole;
context = extendedGuidToKey(start.data.gctxid);
} else {
throw new Error(
`Invalid start node for revision: ${FileNodeID[start.FileNodeID]}`
);
}
const parent = revisions.get(extendedGuidToKey(parentId));
const idMap = parent ? cloneIdMap(parent.idMap) : new Map();
const roots = new Map();
const objects = new Map();
let lastGlobalIdTable: IdMapping | undefined;
let encryptionRef: { stp: number; cb: number } | undefined;
let i = index + 1;
for (; i < nodes.length; ++i) {
const node = nodes[i];
if (node.is(FileNodeID.RevisionManifestEndFND)) {
return {
revision: {
id: extendedGuidToKey(id),
parentId: extendedGuidToKey(parentId),
role,
context,
idMap,
roots,
objects,
encryptionRef
},
nextIndex: i + 1
};
} else if (node.is(FileNodeID.ObjectGroupListReferenceFND)) {
parseObjectGroupList(node.children[0], reader, idMap, contextId, objects);
} else if (
node.is(FileNodeID.GlobalIdTableStartFNDX) ||
node.is(FileNodeID.GlobalIdTableStart2FND)
) {
const parsed = parseGlobalIdTable(nodes, i, parent?.idMap);
lastGlobalIdTable = parsed.mapping;
mergeIdMap(idMap, parsed.mapping);
i = parsed.nextIndex - 1;
} else if (
node.is(FileNodeID.ObjectDeclaration2RefCountFND) ||
node.is(FileNodeID.ObjectDeclaration2LargeRefCountFND) ||
node.is(FileNodeID.ReadOnlyObjectDeclaration2RefCountFND) ||
node.is(FileNodeID.ReadOnlyObjectDeclaration2LargeRefCountFND) ||
node.is(FileNodeID.ObjectDeclarationFileData3RefCountFND) ||
node.is(FileNodeID.ObjectDeclarationFileData3LargeRefCountFND) ||
node.is(FileNodeID.ObjectDeclarationWithRefCountFNDX) ||
node.is(FileNodeID.ObjectDeclarationWithRefCount2FNDX)
) {
// In .onetoc2 files, objects can directly follow GlobalIdTables.
if (!lastGlobalIdTable)
throw new Error("Object declaration without a global ID table.");
const parsed = parseObjectDeclaration(
node,
reader,
lastGlobalIdTable,
contextId
);
if (parsed) objects.set(parsed.key, parsed.object);
else throw new Error("Unexpected object declaration node.");
} else if (
node.is(FileNodeID.ObjectRevisionWithRefCountFNDX) ||
node.is(FileNodeID.ObjectRevisionWithRefCount2FNDX)
) {
// Reference counting does not affect the materialized object state.
} else if (node.is(FileNodeID.RootObjectReference2FNDX)) {
if (!lastGlobalIdTable)
throw new Error(
"Unable to resolve RootObjectReference2FNDX ID: no global ID table found."
);
const id = resolveCompactId(lastGlobalIdTable, node.data.oidRoot);
if (!id)
throw new Error(
`Missing mapping for root object (index: ${node.data.oidRoot.guidIndex}).`
);
roots.set(node.data.RootRole as RootRole, extendedGuidToKey(id));
} else if (node.is(FileNodeID.RootObjectReference3FND)) {
roots.set(
node.data.RootRole as RootRole,
extendedGuidToKey(node.data.oidRoot)
);
} else if (
node.is(FileNodeID.DataSignatureGroupDefinitionFND) ||
node.is(FileNodeID.ObjectInfoDependencyOverridesFND) ||
FileNodeID[node.FileNodeID] === undefined
) {
// Ignored (unknown node types can appear in newer files).
} else if (
node.is(FileNodeID.ObjectDataEncryptionKeyV2FNDX)
) {
// Store the ref to the encryption blob. The blob contains:
// 8-byte header magic, then encryption XML (UTF-16LE), then 8-byte footer magic.
encryptionRef = {
stp: node.data.ref.stp,
cb: node.data.ref.cb
};
} else {
throw new Error(
`Unexpected node (0x${node.FileNodeID.toString(16)}) while parsing revision.`
);
}
}
throw new Error("Revision was not terminated.");
}
function parseObjectGroupList(
list: FileNodeList,
reader: OneNoteReader,
idMap: IdMapping,
contextId: string,
objects: Map
) {
const nodes = walkNodes(list);
if (!nodes[0] || !nodes[0].is(FileNodeID.ObjectGroupStartFND)) {
throw new Error(
"Object group lists must start with an ObjectGroupStartFND node."
);
}
let i = 1;
// Object groups only occur in .one files whose global ID tables never use
// dependency revision references, so no parent table is needed.
if (
nodes[i] &&
(nodes[i].is(FileNodeID.GlobalIdTableStartFNDX) ||
nodes[i].is(FileNodeID.GlobalIdTableStart2FND))
) {
const parsed = parseGlobalIdTable(nodes, i);
i = parsed.nextIndex;
idMap = parsed.mapping;
}
for (; i < nodes.length; ++i) {
const node = nodes[i];
if (node.is(FileNodeID.ObjectGroupEndFND)) break;
if (node.is(FileNodeID.DataSignatureGroupDefinitionFND)) continue;
const parsed = parseObjectDeclaration(node, reader, idMap, contextId);
if (parsed) {
objects.set(parsed.key, parsed.object);
} else {
throw new Error(
`Unexpected node in ObjectGroupList: ${FileNodeID[node.FileNodeID]}`
);
}
}
}
export class ObjectSpace {
readonly id: string;
readonly encryptionXml: string | undefined;
private readonly roots: Map;
private readonly objects: Map;
constructor(
id: string,
roots: Map,
objects: Map,
encryptionXml?: string
) {
this.id = id;
this.roots = roots;
this.objects = objects;
this.encryptionXml = encryptionXml;
}
/** Whether this object space is encrypted (password-protected). */
get isEncrypted(): boolean {
return this.encryptionXml !== undefined;
}
/** Decrypt all encrypted objects in this space using the given data key. */
async decryptObjects(dataKey: Uint8Array, reader: OneNoteReader) {
const encryptedObjects = [...this.objects.entries()].filter(
([, obj]) => obj.odcs !== 0 && obj.encryptedData
);
if (encryptedObjects.length === 0) return;
for (const [key, obj] of encryptedObjects) {
try {
const decrypted = await decryptObjectData(obj.encryptedData!, dataKey);
const tempReader = new (reader.constructor as new (
buf: Uint8Array
) => OneNoteReader)(decrypted);
obj.propSet = ObjectSpaceObjectPropSet(tempReader);
obj.encryptedData = undefined;
} catch (e) {
console.warn(`Failed to decrypt object ${key}:`, e);
}
}
}
getObject(id: string): OneNoteObject | undefined {
return this.objects.get(id);
}
contentRoot(): string | undefined {
return this.roots.get(RootRole.DefaultContent);
}
metadataRoot(): string | undefined {
return this.roots.get(RootRole.MetadataRoot);
}
}
/**
* Provides access to all object spaces contained in a OneStore file and
* mirrors the `OneStore` API of the Rust onenote_parser crate.
*/
export class ObjectSpaceStore {
readonly objectSpaces = new Map();
readonly rootObjectSpaceId: string;
readonly fileDataStore = new Map();
private constructor(
objectSpaces: Map,
rootObjectSpaceId: string,
fileDataStore: Map
) {
this.objectSpaces = objectSpaces;
this.rootObjectSpaceId = rootObjectSpaceId;
this.fileDataStore = fileDataStore;
}
get dataRoot(): ObjectSpace {
const space = this.objectSpaces.get(this.rootObjectSpaceId);
if (!space) throw new Error("Root object space is missing.");
return space;
}
objectSpace(id: string): ObjectSpace | undefined {
return this.objectSpaces.get(id);
}
static parse(store: OneStore): ObjectSpaceStore {
const reader = store.reader;
const fileDataStore = new Map();
const objectSpaces = new Map();
let rootObjectSpaceId: string | undefined;
const nodes = walkNodes(store.fileNodeList);
for (const node of nodes) {
if (node.is(FileNodeID.FileDataStoreListReferenceFND)) {
parseFileDataStore(node, reader, fileDataStore);
}
}
for (const node of nodes) {
if (node.is(FileNodeID.ObjectSpaceManifestListReferenceFND)) {
const space = parseObjectSpace(node, reader, fileDataStore);
objectSpaces.set(space.id, space);
} else if (node.is(FileNodeID.ObjectSpaceManifestRootFND)) {
rootObjectSpaceId = extendedGuidToKey(node.data.gosidRoot);
}
}
if (!rootObjectSpaceId) {
throw new Error(
"Root file node list did not contain a node with the root ID."
);
}
if (!objectSpaces.has(rootObjectSpaceId)) {
throw new Error("Root object space is missing.");
}
return new ObjectSpaceStore(objectSpaces, rootObjectSpaceId, fileDataStore);
}
}
function parseFileDataStore(
node: FileNode,
reader: OneNoteReader,
fileDataStore: Map
) {
for (const list of node.children) {
for (const item of walkNodes(list)) {
if (!item.is(FileNodeID.FileDataStoreObjectReferenceFND)) {
throw new Error(
`Unexpected item in file list: ${FileNodeID[item.FileNodeID]}. Expected FileDataStoreObjectReferenceFND.`
);
}
reader.seek(item.data.ref.stp);
const guidHeader = reader.deserializeGUID();
const cbLength = reader.deserializeLong();
reader.deserializeBytes(4);
reader.deserializeBytes(8);
const fileData = reader.deserializeBytes(cbLength);
fileDataStore.set(item.data.guidReference.toString(), fileData);
void guidHeader;
}
}
}
function parseObjectSpace(
listReference: FileNode,
reader: OneNoteReader,
fileDataByGuid: Map
): ObjectSpace {
if (!listReference.is(FileNodeID.ObjectSpaceManifestListReferenceFND))
throw new Error("Expected an ObjectSpaceManifestListReferenceFND.");
const gosid = extendedGuidToKey(listReference.data.gosid);
const contextId = gosid;
// The object space manifest list contains an ObjectSpaceManifestListStartFND
// followed by one or more RevisionManifestListReferenceFNDs. All but the last
// revision must be ignored.
const manifestNodes = walkNodes(listReference.children[0]);
let lastRevisionList: FileNode | undefined;
for (const node of manifestNodes) {
if (node.is(FileNodeID.RevisionManifestListReferenceFND))
lastRevisionList = node;
}
if (!lastRevisionList) {
throw new Error(
"ObjectSpaceManifestListReferenceFND must point to a list with at least one revision."
);
}
const nodes = walkNodes(lastRevisionList.children[0]);
const revisions = new Map();
const labels = new Map();
let i = 0;
while (i < nodes.length) {
const node = nodes[i];
if (node.is(FileNodeID.RevisionManifestEndFND)) {
i += 1;
break;
} else if (node.is(FileNodeID.RevisionManifestListStartFND)) {
i += 1;
} else if (node.is(FileNodeID.RevisionRoleDeclarationFND)) {
const revisionId = extendedGuidToKey(node.data.rid);
if (!revisions.has(revisionId)) {
throw new Error(
"Revision role declaration points to an undeclared revision."
);
}
labels.set(labelKey(undefined, node.data.RevisionRole), revisionId);
i += 1;
} else if (node.is(FileNodeID.RevisionRoleAndContextDeclarationFND)) {
const revisionId = extendedGuidToKey(node.data.base.rid);
if (!revisions.has(revisionId)) {
throw new Error(
"Revision role declaration points to an undeclared revision."
);
}
labels.set(
labelKey(
extendedGuidToKey(node.data.gctxid),
node.data.base.RevisionRole
),
revisionId
);
i += 1;
} else if (
node.is(FileNodeID.RevisionManifestStart4FND) ||
node.is(FileNodeID.RevisionManifestStart6FND) ||
node.is(FileNodeID.RevisionManifestStart7FND)
) {
const parsed = parseRevision(nodes, i, reader, revisions, contextId);
labels.set(
labelKey(parsed.revision.context, parsed.revision.role),
parsed.revision.id
);
revisions.set(parsed.revision.id, parsed.revision);
i = parsed.nextIndex;
} else {
throw new Error(
`Unexpected node encountered in RevisionManifestList: ${
FileNodeID[node.FileNodeID]
}`
);
}
}
const activeId = labels.get(labelKey(undefined, ACTIVE_CONTENT_ROLE));
if (!activeId) {
throw new Error(
"Revision manifest list has no active revision in the default context."
);
}
// Materialize the active revision's dependency chain.
const roots = new Map();
const objects = new Map();
const chain: Revision[] = [];
let revisionId: string | undefined = activeId;
while (revisionId) {
const revision = revisions.get(revisionId);
if (!revision)
throw new Error(
`Revision chain points to undeclared revision ${revisionId}.`
);
chain.push(revision);
revisionId =
revision.parentId === NIL_EXTENDED_GUID_KEY
? undefined
: revision.parentId;
}
let encryptionRef: { stp: number; cb: number } | undefined;
for (const revision of chain.reverse()) {
for (const [role, id] of revision.roots) roots.set(role, id);
for (const [id, object] of revision.objects) objects.set(id, object);
if (revision.encryptionRef && !encryptionRef) {
encryptionRef = revision.encryptionRef;
}
}
// If the object space is encrypted, read the encryption XML blob.
let encryptionXml: string | undefined;
if (encryptionRef) {
reader.seek(encryptionRef.stp);
const blob = reader.deserializeBytes(encryptionRef.cb);
// The blob has: 8-byte header magic (0xFB6BA385DAD1A067),
// then UTF-16LE XML, then 8-byte footer magic (0x2649294F8E198B3C).
// Skip the 8-byte header and 8-byte footer.
if (blob.length > 16) {
const xmlBytes = blob.slice(8, blob.length - 8);
encryptionXml = decode(xmlBytes, "utf16le");
}
}
// Resolve file data references.
for (const object of objects.values()) {
if (object.fileDataReference) {
if (object.fileDataReference.startsWith("")) {
let guid = object.fileDataReference.slice("".length);
// Strip curly braces if present — the file data store keys
// don't include them but the references may.
if (guid.startsWith("{") && guid.endsWith("}")) {
guid = guid.slice(1, -1);
}
object.fileData = fileDataByGuid.get(guid);
}
}
}
return new ObjectSpace(gosid, roots, objects, encryptionXml);
}
function labelKey(context: string | undefined, role: number): string {
return `${context ?? ""}:${role}`;
}