Obsidian Converter

.github/workflows/node.js.yml

@@ -5,26 +5,25 @@ name: Node.js CI
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ['main']
   pull_request:
-    branches: [ "main" ]
+    branches: ['main']
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        node-version: [14.x, 16.x]
+        node-version: [18.3.X]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
-      with:
-        node-version: ${{ matrix.node-version }}
-        cache: 'yarn'
-    - run: yarn install --frozen-lockfile
-    - run: yarn ci
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'yarn'
+      - run: yarn install --frozen-lockfile
+      - run: yarn ci

.vscode/settings.json

@@ -0,0 +1,3 @@
+{"[markdown]": {
+    "editor.formatOnSave": false,
+  }}

OBSIDIAN_README.md

@@ -0,0 +1,39 @@
+# Vault Converter
+
+The vault converter works like this:
+
+- go through the vault folder recursively
+- create a node for each folder and append it partially to the target file (we leave the children array open)
+- convert each file to a node and append it to the target file (see Markdown Nodes explanation)
+- close each folder-node as soon as its done
+
+Vaults can be quite big, so this append-approach is much more performant.
+We use the VaultContext to store all information that is needed across the whole vault, so that we don't need to go back to the created Tana nodes.
+
+# Concepts
+
+## Folder Nodes
+
+In Obsidian vaults the folder structure is important. For example a PARA approach would have at least four top-level folder and these are critical for orientation (at least until a more Tana-native orientation is established). Therefore we nest all the created nodes under "Folder Nodes" which are created for each folder in the vault.
+
+## Markdown Nodes
+
+Each markdown file is first parsed into a sequential list of Markdown Nodes by breaking up the file according to hierarchy-indicators like headings. These Markdown Nodes are very similar to Tana IF Nodes but don't yet form a graph and contain additional information about the hierarchy level they come from.
+
+This "second" intermediate format is much easier to debug than the finalized graph structure. It also is necessary outside of Developer Experience considerations: the markdown file needs to become a nested list and for this we need the hierarchy information. Just parsing it as a flat structure is not going to cut it because it loses valuable semantic information.
+
+The Markdown Nodes are lightly pre-processed to contain only relevant information.
+
+## Conversion to the Tana IF
+
+Markdown Nodes are converted to Tana IF Nodes, which are then used in the resulting JSON file. We use the hierarchy information in the Markdown Nodes to create the appropriate nesting.
+
+Each Obsidian Link is replaced with a new UID, also each file and folder gets a UID. Via the VaultContext we make sure that the correct UID is used when we replace the Obsidian Links.
+
+### Missing nodes
+
+Missing Links (no file exists for these) are collected and saved in a separate node under "Missing nodes for $YOUR_VAULT_NAME". We could of course not save these links but then we would need to take care how to not lose the Obsidian Link-name (because it was converted to a UID).
+
+## VaultContext
+
+We need some context information like the summary data that needs to be used across the whole converted vault. The place for this is the VaultContext.

README.md

@@ -10,6 +10,7 @@ Supported formats:
 
 - Workflowy (OPML)
 - Roam Research JSON
+- Obsidian Markdown
 
 If you need to do some something special with your data before putting it into Tana you can just fork this project and hack the current converters into doing what you need. As long as the resulting file follows the format you will be able to import it into Tana.
 
@@ -23,6 +24,10 @@ If you are making changes that you think will benefit other users, please create
 
 `yarn convert:workflowy datasets/my_workflowy_export.opml`
 
+#### Converting Obsidian Markdown Vault to Tana JSON
+
+`yarn convert:obsidian datasets/my_obsidian_vault/`
+
 # ✍️ Contributing
 
 We are always looking for new importers and as well as improvements to existing ones! Contributions from open-source developers are greatly appreciated.

docs/index.html

@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Inofficial Obsidian-2-Tana Converter</title>
+    <script type="module" crossorigin src="./assets/index.b5f269bd.js"></script>
+  </head>
+  <body>
+    <h1>Inofficial Obsidian-2-Tana Converter</h1>
+    <p>
+      Select a zip of your Obsidian vault folder and get a TIF file back, ready to be imported in Tana.<br />
+      <em>
+        Zip the central vault folder directly (not just the sub-folders) and don't rename the zip file. Example: my
+        vault is called pkm, this is also the name of its folder and the zip file is called pkm.zip.
+      </em>
+    </p>
+    <b>There is no server involved in the conversion, the zip file will stay on your computer.</b>
+    <p>
+      <label for="vault-zip">Select your vault-zip:</label>
+      <br />
+      <input type="file" id="vault-zip" name="vault-zip" accept="application/zip" />
+    </p>
+
+  </body>
+</html>

index.html

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Inofficial Obsidian-2-Tana Converter</title>
+  </head>
+  <body>
+    <h1>Inofficial Obsidian-2-Tana Converter</h1>
+    <p>
+      Select a zip of your Obsidian vault folder and get a TIF file back, ready to be imported in Tana.<br />
+      <em>
+        Zip the central vault folder directly (not just the sub-folders) and don't rename the zip file. Example: my
+        vault is called pkm, this is also the name of its folder and the zip file is called pkm.zip.
+      </em>
+    </p>
+    <b>There is no server involved in the conversion, the zip file will stay on your computer.</b>
+    <p>
+      <label for="vault-zip">Select your vault-zip:</label>
+      <br />
+      <input type="file" id="vault-zip" name="vault-zip" accept="application/zip" />
+    </p>
+    <script type="module" src="./src/script.ts"></script>
+  </body>
+</html>

package.json

@@ -10,29 +10,34 @@
   ],
   "license": "MIT",
   "scripts": {
+    "dev": "vite",
     "ci": "yarn lint && yarn test",
     "format": "prettier --write \"src/**/*.ts\"",
     "lint": "eslint --ext .ts src/",
     "prepare": "rm -rf dist/ && ./node_modules/.bin/tsc  --project tsconfig.build.json",
     "build": "yarn prepare",
     "convert:roam": "yarn build && node --experimental-modules --es-module-specifier-resolution=node dist/runner.js roam",
     "convert:workflowy": "yarn build && node --experimental-modules --es-module-specifier-resolution=node dist/runner.js workflowy",
-    "test": "jest"
+    "convert:obsidian": "yarn build && node --experimental-modules --es-module-specifier-resolution=node dist/runner.js obsidian",
+    "zip": "cd ./src/converters/obsidian/tests/fixtures/ && zip -r vault.zip vault && cd -",
+    "test": "vitest",
+    "deploy": "vite build && rm -rfv ./docs/* && mv ./dist/* ./docs/"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.18.6",
     "@babel/preset-typescript": "^7.18.6",
-    "@jest/globals": "^28.1.2",
-    "@types/jest": "^28.1.4",
-    "@types/node": "^18.0.3",
+    "@types/node": "^18.8.3",
     "@typescript-eslint/eslint-plugin": "^5.32.0",
     "@typescript-eslint/parser": "^5.32.0",
     "eslint": "^8.21.0",
-    "jest": "^28.1.2",
     "prettier": "^2.7.1",
-    "typescript": "^4.7.4"
+    "typescript": "^4.7.4",
+    "vite": "^3.1.8",
+    "vitest": "^0.24.3"
   },
   "dependencies": {
+    "@zip.js/zip.js": "^2.6.43",
+    "moment": "^2.29.4",
     "opml": "^0.4.24"
   },
   "eslintConfig": {
@@ -74,5 +79,8 @@
     "testMatch": [
       "**/?(*.)+(test).[t]s?(x)"
     ]
+  },
+  "engines": {
+    "node": ">=18.3.0"
   }
 }

src/converters/obsidian/LocalObsidianZipVaultConverter.ts

@@ -0,0 +1,31 @@
+import { readFileSync, unlinkSync, writeFileSync } from 'fs';
+import { TanaIntermediateSummary } from '../../types/types';
+import { basename } from './filesystem/CustomFileSystemAdapter';
+import { WebFileSystemAdapter } from './filesystem/WebFileSystemAdapter';
+import { ObsidianVaultConverter } from './ObsidianVaultConverter';
+import { IdGenerator } from './utils/IdGenerator';
+import { createVaultContext, VaultContext } from './VaultContext';
+
+export async function LocalObsidianZipVaultConverter(
+  zipPath: string,
+  today: number = Date.now(),
+  idGenerator?: IdGenerator,
+): Promise<[TanaIntermediateSummary, VaultContext]> {
+  const zipBlob = readFileSync(zipPath);
+  const adapter = new WebFileSystemAdapter(new Blob([zipBlob]));
+  //removing ".zip"
+  const vaultName = basename(zipPath).slice(0, -4);
+  const context = createVaultContext(vaultName, adapter, idGenerator);
+  return ObsidianVaultConverter(context, today).then((summary) => {
+    const result = adapter.getResult();
+    const targetPath = `${zipPath.slice(0, -4)}.tif.json`;
+
+    try {
+      unlinkSync(targetPath);
+      // eslint-disable-next-line no-empty
+    } catch (e) {}
+    writeFileSync(targetPath, result);
+
+    return [summary, context];
+  });
+}

src/converters/obsidian/ObsidianVaultConverter.ts

@@ -0,0 +1,77 @@
+import { createUnlinkedTanaNodes } from './links/invalidLinks';
+import { addFileNode, addParentNodeEnd, addParentNodeStart, handleVault } from './tanaconversion/vaultConversion';
+import { shiftFromLeafToTop, VaultContext } from './VaultContext';
+import { createSuperTagObjects } from './tanafeatures/supertags';
+import { postProcessTIFFIle } from './links/headingLinks';
+import { basename, readConfig } from './filesystem/CustomFileSystemAdapter';
+
+/**
+ * Converts the vault to the Tana format and incrementally saves it, otherwise it would be to memory intensive on big vaults.
+ * Due to the incremental approach the output-file will be valid JSON but not be formatted perfectly.
+ */
+export async function ObsidianVaultConverter(context: VaultContext, today: number = Date.now()) {
+  await context.adapter.initReadingVault();
+
+  await loadDailyNotesConfig(context);
+
+  const targetPath = `${context.vaultPath}.tif.json`;
+  try {
+    context.adapter.removeFile(targetPath);
+    // eslint-disable-next-line no-empty
+  } catch (e) {}
+
+  context.adapter.appendToResultFile(targetPath, '{\n  "version": "TanaIntermediateFile V0.1",\n  "nodes": [\n');
+
+  await handleVault(
+    context,
+    context.vaultPath,
+    addParentNodeStart(targetPath, today, context),
+    addParentNodeEnd(context, targetPath),
+    addFileNode(targetPath, today, context),
+  );
+  context.adapter.flushResultsFromInitialProcessing(targetPath);
+
+  //the vault-node needs to be counted as a top level node
+  shiftFromLeafToTop(context.summary);
+
+  //post processing can be done before unlinked (it will add unlinked headings)
+  //because the unlinked summary nodes are just created by the converter and have no connection to the rest
+  //20 secs
+  await postProcessTIFFIle(targetPath, context);
+
+  const collectedUnlinkedNodes = createUnlinkedTanaNodes(basename(context.vaultPath), today, context);
+  if (collectedUnlinkedNodes) {
+    //TODO: summary?
+    context.adapter.appendToResultFile(targetPath, ', ' + JSON.stringify(collectedUnlinkedNodes, null, 2));
+  }
+
+  //close vault-node children
+  context.adapter.appendToResultFile(targetPath, '\n  ]');
+
+  const superTags = createSuperTagObjects(context.superTagTracker);
+  if (superTags.length > 0) {
+    context.adapter.appendToResultFile(targetPath, ',\n  "supertags": \n' + JSON.stringify(superTags, null, 2));
+  }
+
+  if (context.attributes.length > 0) {
+    context.adapter.appendToResultFile(
+      targetPath,
+      ',\n  "attributes": \n' + JSON.stringify(context.attributes, null, 2),
+    );
+  }
+
+  context.adapter.appendToResultFile(targetPath, ',\n  "summary": \n' + JSON.stringify(context.summary, null, 2));
+
+  //close target object
+  context.adapter.appendToResultFile(targetPath, '\n}');
+  context.adapter.flushResultsFromInitialProcessing(targetPath);
+  return context.summary;
+}
+
+async function loadDailyNotesConfig(context: VaultContext) {
+  const config = await readConfig<{ format: string }>(context, 'daily-notes', (res) => {
+    return res ? res : { format: 'YYYY-MM-DD' };
+  });
+  context.dailyNoteFormat = config.format;
+  console.log('Using daily notes format ' + config.format + ' for detecting calendar nodes.');
+}

src/converters/obsidian/VaultContext.ts

@@ -0,0 +1,79 @@
+import { TanaIntermediateSummary, TanaIntermediateAttribute } from '../../types/types';
+import { idgenerator as randomGenerator } from '../../utils/utils';
+import { FileDescMap } from './links/FileDescMap';
+import { BlockLinkTracker } from './links/blockLinks';
+import { SuperTagTracker } from './tanafeatures/supertags';
+import { UidRequestType } from './links/internalLinks';
+import { IdGenerator } from './utils/IdGenerator';
+import { HeadingTracker, HeadingDummyUidTracker } from './links/headingLinks';
+import { CustomFileSystemAdapter, SEPARATOR } from './filesystem/CustomFileSystemAdapter';
+
+export type UidTracker = FileDescMap<UidData>;
+
+interface UidData {
+  type: UidRequestType;
+  uid: string;
+  obsidianLink: string;
+}
+
+/**
+ * Contains all information that is used across the whole vault, like which UIDs have already been used.
+ */
+export interface VaultContext {
+  summary: TanaIntermediateSummary;
+  defaultLinkTracker: UidTracker;
+  headingTracker: HeadingTracker;
+  dummyHeadingLinkTracker: HeadingDummyUidTracker;
+  blockLinkTracker: BlockLinkTracker;
+  invalidLinks: { uid: string; link: string }[];
+  superTagTracker: SuperTagTracker;
+  attributes: TanaIntermediateAttribute[];
+  vaultPath: string;
+  idGenerator: IdGenerator;
+  adapter: CustomFileSystemAdapter;
+  dailyNoteFormat: string;
+}
+
+export function incrementSummary(summary: TanaIntermediateSummary) {
+  summary.totalNodes++;
+  summary.leafNodes++;
+}
+
+export function shiftFromLeafToTop(summary: TanaIntermediateSummary) {
+  summary.leafNodes--;
+  summary.topLevelNodes++;
+}
+
+export function createVaultContext(
+  vaultPath: string,
+  fileSystemAdapter: CustomFileSystemAdapter,
+  idGenerator: () => string = randomGenerator,
+  dailyNoteFormat = 'YYYY-MM-DD',
+): VaultContext {
+  if (vaultPath.endsWith(SEPARATOR)) {
+    vaultPath = vaultPath.slice(0, -1);
+  }
+  vaultPath = fileSystemAdapter.resolve(vaultPath);
+
+  return {
+    summary: {
+      leafNodes: 0,
+      topLevelNodes: 0,
+      totalNodes: 0,
+      calendarNodes: 0,
+      fields: 0,
+      brokenRefs: 0,
+    },
+    adapter: fileSystemAdapter,
+    idGenerator,
+    vaultPath,
+    defaultLinkTracker: new FileDescMap(),
+    headingTracker: new FileDescMap(),
+    dummyHeadingLinkTracker: new Map(),
+    blockLinkTracker: new FileDescMap(),
+    invalidLinks: [],
+    superTagTracker: new Map(),
+    attributes: [],
+    dailyNoteFormat,
+  };
+}