feat: Improved ergonomics for import/require

dljsjr · dljsjr · commit 2d8eb1d7438d · 2025-02-04T22:28:04.000-06:00
This commit improves the ergonomics aroud importing/requiring scripts from
other files or blocks, while remaining backwards compatible with the existing
functionality in Datacore. The specifics of the changes are:

- Paths and wikilink-like strings passed to `dc.require()` will be checked
against all elements in the `scriptRoots` Setting if the path fails to resolve
against the vault root
- The `imports` transform is now enabled on the Sucrase `transform` operations. This
causes `import { foo } from "bar"` syntax to be transformed in to `const _foo = require("bar")` during
transpiling.
- The script transpilation pass will convert any `foo = require("bar")` statements in to the form
`foo = await dc.require("bar")` before evaluation. This covers both explicitly authored CommonJS require
syntax, as well as the output of the Sucrase transformation for ES Module imports.
- The script transpilation pass will resolve relative paths (e.g. `./foo`) against the script that called
`dc.require()`, allowing for relative imports to now be utilized. This happens as part of the same new
transform sa the one above that converts all `require` statements in to `dc.require` statements.
- The `imports` transform also allows for the use of the `export` keyword in scripts, which gets
transformed in to keys on an `exports` object.
  - The `exports` object is injected via the context used to construct the `Function` evaluator.
  - Scripts can now choose to either `return { ... }`, as is the current pattern, *or* they can use
    `export`.
  - Returned values take precedence over the exports object.
- Datacore code blocks that return a View to be rendered can now use `export default` by leveraging
the same effects introduced by the `imports` transform.
  - Similarly to scripts loaded with `dc.require()`, the `return function ...` form is still supported.
  - Also similarly to the above, the `return function ...` form is given precedence over the `export default` form.
diff --git a/package.json b/package.json
@@ -1,61 +1,63 @@
 {
-  "name": "datacore",
-  "version": "0.1.19",
-  "description": "Reactive data engine for Obsidian.md.",
-  "main": "lib/index.js",
-  "types": "lib/index.d.ts",
-  "files": [
-    "lib/**/*"
-  ],
-  "scripts": {
-    "dev": "node esbuild.config.mjs",
-    "build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
-    "test": "yarn run jest",
-    "test-watch": "yarn run jest -i --watch --no-cache",
-    "check-format": "yarn run prettier --check src",
-    "format": "yarn run prettier --write src"
-  },
-  "keywords": [
-    "obsidian",
-    "datacore",
-    "dataview",
-    "pkm"
-  ],
-  "author": "Michael Brenan",
-  "license": "MIT",
-  "devDependencies": {
-    "@codemirror/language": "https://github.com/lishid/cm-language",
-    "@codemirror/state": "^6.0.1",
-    "@codemirror/view": "^6.0.1",
-    "@types/jest": "^27.0.1",
-    "@types/luxon": "^2.3.2",
-    "@types/node": "^16.7.13",
-    "@types/parsimmon": "^1.10.6",
-    "builtin-modules": "3.3.0",
-    "esbuild": "^0.16.11",
-    "esbuild-plugin-inline-worker": "https://github.com/mitschabaude/esbuild-plugin-inline-worker",
-    "jest": "^27.1.0",
-    "obsidian": "^1.6.6",
-    "prettier": "2.3.2",
-    "ts-jest": "^27.0.5",
-    "ts-node": "^10.9.2",
-    "tslib": "^2.3.1",
-    "typescript": "^5.4.2"
-  },
-  "dependencies": {
-    "@datastructures-js/queue": "^4.2.3",
-    "@fortawesome/fontawesome-svg-core": "^6.4.0",
-    "@fortawesome/free-solid-svg-icons": "^6.4.0",
-    "@fortawesome/react-fontawesome": "^0.2.0",
-    "emoji-regex": "^10.2.1",
-    "flatqueue": "^2.0.3",
-    "localforage": "1.10.0",
-    "luxon": "^2.4.0",
-    "parsimmon": "^1.18.0",
-    "preact": "^10.17.1",
-    "react-select": "^5.8.0",
-    "sorted-btree": "^1.8.1",
-    "sucrase": "3.35.0",
-    "yaml": "^2.3.3"
-  }
+    "name": "datacore",
+    "version": "0.1.19",
+    "description": "Reactive data engine for Obsidian.md.",
+    "main": "lib/index.js",
+    "types": "lib/index.d.ts",
+    "files": [
+        "lib/**/*"
+    ],
+    "scripts": {
+        "dev": "node esbuild.config.mjs",
+        "build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
+        "test": "yarn run jest",
+        "test-watch": "yarn run jest -i --watch --no-cache",
+        "check-format": "yarn run prettier --check src",
+        "format": "yarn run prettier --write src"
+    },
+    "keywords": [
+        "obsidian",
+        "datacore",
+        "dataview",
+        "pkm"
+    ],
+    "author": "Michael Brenan",
+    "license": "MIT",
+    "devDependencies": {
+        "@codemirror/language": "https://github.com/lishid/cm-language",
+        "@codemirror/state": "^6.0.1",
+        "@codemirror/view": "^6.0.1",
+        "@types/jest": "^27.0.1",
+        "@types/luxon": "^2.3.2",
+        "@types/node": "^16.7.13",
+        "@types/parsimmon": "^1.10.6",
+        "@types/path-browserify": "^1.0.3",
+        "builtin-modules": "3.3.0",
+        "esbuild": "^0.16.11",
+        "esbuild-plugin-inline-worker": "https://github.com/mitschabaude/esbuild-plugin-inline-worker",
+        "jest": "^27.1.0",
+        "obsidian": "^1.6.6",
+        "prettier": "2.3.2",
+        "ts-jest": "^27.0.5",
+        "ts-node": "^10.9.2",
+        "tslib": "^2.3.1",
+        "typescript": "^5.4.2"
+    },
+    "dependencies": {
+        "@datastructures-js/queue": "^4.2.3",
+        "@fortawesome/fontawesome-svg-core": "^6.4.0",
+        "@fortawesome/free-solid-svg-icons": "^6.4.0",
+        "@fortawesome/react-fontawesome": "^0.2.0",
+        "emoji-regex": "^10.2.1",
+        "flatqueue": "^2.0.3",
+        "localforage": "1.10.0",
+        "luxon": "^2.4.0",
+        "parsimmon": "^1.18.0",
+        "path-browserify": "^1.0.1",
+        "preact": "^10.17.1",
+        "react-select": "^5.8.0",
+        "sorted-btree": "^1.8.1",
+        "sucrase": "3.35.0",
+        "yaml": "^2.3.3"
+    }
 }
diff --git a/src/api/local-api.tsx b/src/api/local-api.tsx
@@ -90,7 +90,7 @@ export class DatacoreLocalApi {
      * ```
      */
     public async require(path: string | Link): Promise<any> {
-        const result = await this.scriptCache.load(path, { dc: this });
+        const result = await this.scriptCache.load(path, this);
         return result.orElseThrow();
     }
 
diff --git a/src/api/script-cache.ts b/src/api/script-cache.ts
@@ -3,10 +3,16 @@ import { Datastore } from "index/datastore";
 import { Result } from "./result";
 import { MarkdownCodeblock, MarkdownSection } from "index/types/markdown";
 import { Deferred, deferred } from "utils/deferred";
-import { ScriptLanguage, asyncEvalInContext, transpile } from "utils/javascript";
+import {
+    ScriptDefinition,
+    ScriptLanguage,
+    asyncEvalInContext,
+    defaultScriptLoadingContext,
+    transpile,
+} from "utils/javascript";
 import { lineRange } from "utils/normalizers";
-import { TFile } from "obsidian";
-import { Fragment, h } from "preact";
+import { normalizePath, TFile } from "obsidian";
+import { DatacoreLocalApi } from "./local-api";
 
 /** A script that is currently being loaded. */
 export interface LoadingScript {
@@ -56,20 +62,33 @@ export class ScriptCache {
     public constructor(private store: Datastore) {}
 
     /** Load the given script at the given path, recursively loading any subscripts as well.  */
-    public async load(path: string | Link, context: Record<string, any>): Promise<Result<any, string>> {
+    public async load(path: string | Link, api: DatacoreLocalApi): Promise<Result<any, string>> {
+        // First, attempt to resolve the script against the script roots so we cache a canonical script path as the key.
+        var resolvedPath = undefined;
+        const roots = ["", ...api.core.settings.scriptRoots];
+        for (var i = 0; i < roots.length; i++) {
+            resolvedPath = this.store.resolveLink(path, normalizePath(roots[i]));
+            if (resolvedPath) {
+                break;
+            }
+        }
+
+        resolvedPath = resolvedPath?.$file ?? path;
         // Always check the cache first.
-        const key = this.pathkey(path);
+        const key = this.pathkey(resolvedPath);
         const currentScript = this.scripts.get(key);
         if (currentScript) {
-            if (currentScript.type === "loaded") return Result.success(currentScript.object);
+            if (currentScript.type === "loaded") {
+                return Result.success(currentScript.object);
+            }
 
             // TODO: If we try to load an already-loading script, we are almost certainly doing something
             // weird. Either the caller is not `await`-ing the load and loading multiple times, OR
             // we are in a `require()` loop. Either way, we'll error out for now since we can't handle
             // either case currently.
             return Result.failure(
-                `Failed to import script "${path.toString()}", as it is in the middle of being loaded. Do you have
-                 a circular dependency in your require() calls? The currently loaded or loading scripts are: 
+                `Failed to import script "${resolvedPath.toString()}", as it is in the middle of being loaded. Do you have
+                 a circular dependency in your require() calls? The currently loaded or loading scripts are:
                  ${Array.from(this.scripts.values())
                      .map((sc) => "\t" + sc.path)
                      .join("\n")}`
@@ -80,7 +99,7 @@ export class ScriptCache {
         const deferral = deferred<Result<any, string>>();
         this.scripts.set(key, { type: "loading", promise: deferral, path: key });
 
-        const result = await this.loadUncached(path, context);
+        const result = await this.loadUncached(resolvedPath, api);
         deferral.resolve(result);
 
         if (result.successful) {
@@ -93,25 +112,30 @@ export class ScriptCache {
     }
 
     /** Load a script, directly bypassing the cache. */
-    private async loadUncached(path: string | Link, context: Record<string, any>): Promise<Result<any, string>> {
-        const maybeSource = await this.resolveSource(path);
+    private async loadUncached(scriptPath: string | Link, api: DatacoreLocalApi): Promise<Result<any, string>> {
+        var maybeSource = await this.resolveSource(scriptPath);
         if (!maybeSource.successful) return maybeSource;
 
         // Transpile to vanilla javascript first...
-        const { code, language } = maybeSource.value;
+        const scriptDefinition = maybeSource.value;
         let basic;
         try {
-            basic = transpile(code, language);
+            basic = transpile(scriptDefinition);
         } catch (error) {
-            return Result.failure(`Failed to import ${path.toString()} while transpiling from ${language}: ${error}`);
+            return Result.failure(
+                `Failed to import ${scriptPath.toString()} while transpiling from ${
+                    scriptDefinition.scriptLanguage
+                }: ${error}`
+            );
         }
 
         // Then finally execute the script to 'load' it.
-        const finalContext = Object.assign({ h: h, Fragment: Fragment }, context);
+        const scriptContext = defaultScriptLoadingContext(api);
         try {
-            return Result.success(await asyncEvalInContext(basic, finalContext));
+            const loadRet = (await asyncEvalInContext(basic, scriptContext)) ?? scriptContext.exports;
+            return Result.success(loadRet);
         } catch (error) {
-            return Result.failure(`Failed to execute script '${path.toString()}': ${error}`);
+            return Result.failure(`Failed to execute script '${scriptPath.toString()}': ${error}`);
         }
     }
 
@@ -122,10 +146,8 @@ export class ScriptCache {
     }
 
     /** Attempts to resolve the source to load given a path or link to a markdown section. */
-    private async resolveSource(
-        path: string | Link
-    ): Promise<Result<{ code: string; language: ScriptLanguage }, string>> {
-        const object = this.store.resolveLink(path);
+    private async resolveSource(path: string | Link, sourcePath?: string): Promise<Result<ScriptDefinition, string>> {
+        const object = this.store.resolveLink(path, sourcePath);
         if (!object) return Result.failure("Could not find a script at the given path: " + path.toString());
 
         const tfile = this.store.vault.getFileByPath(object.$file!);
@@ -137,7 +159,7 @@ export class ScriptCache {
 
             try {
                 const code = await this.store.vault.cachedRead(tfile);
-                return Result.success({ code, language });
+                return Result.success({ scriptFile: tfile, scriptLanguage: language, scriptSource: code });
             } catch (error) {
                 return Result.failure("Failed to load javascript/typescript source file: " + error);
             }
@@ -158,7 +180,11 @@ export class ScriptCache {
                 ScriptCache.SCRIPT_LANGUAGES[
                     maybeBlock.$languages.find((lang) => lang.toLocaleLowerCase() in ScriptCache.SCRIPT_LANGUAGES)!
                 ];
-            return (await this.readCodeblock(tfile, maybeBlock)).map((code) => ({ code, language }));
+            return (await this.readCodeblock(tfile, maybeBlock)).map((code) => ({
+                scriptFile: tfile,
+                scriptSource: code,
+                scriptLanguage: language,
+            }));
         } else if (object instanceof MarkdownCodeblock) {
             const maybeLanguage = object.$languages.find(
                 (lang) => lang.toLocaleLowerCase() in ScriptCache.SCRIPT_LANGUAGES
@@ -167,7 +193,11 @@ export class ScriptCache {
                 return Result.failure(`The codeblock referenced by '${path}' is not a JS/TS codeblock.`);
 
             const language = ScriptCache.SCRIPT_LANGUAGES[maybeLanguage];
-            return (await this.readCodeblock(tfile, object)).map((code) => ({ code, language }));
+            return (await this.readCodeblock(tfile, object)).map((code) => ({
+                scriptFile: tfile,
+                scriptSource: code,
+                scriptLanguage: language,
+            }));
         }
 
         return Result.failure(`Cannot import '${path.toString()}: not a JS/TS file or codeblock reference.`);
diff --git a/src/index/datastore.ts b/src/index/datastore.ts
@@ -4,7 +4,7 @@ import { FolderIndex } from "index/storage/folder";
 import { InvertedIndex } from "index/storage/inverted";
 import { IndexPrimitive, IndexQuery, IndexSource } from "index/types/index-query";
 import { Indexable, LINKABLE_TYPE, LINKBEARING_TYPE, TAGGABLE_TYPE } from "index/types/indexable";
-import { MetadataCache, Vault } from "obsidian";
+import { MetadataCache, normalizePath, Vault } from "obsidian";
 import { MarkdownPage } from "./types/markdown";
 import { extractSubtags, normalizeHeaderForLink } from "utils/normalizers";
 import FlatQueue from "flatqueue";
@@ -14,6 +14,7 @@ import { IndexResolver, execute, optimizeQuery } from "index/storage/query-execu
 import { Result } from "api/result";
 import { Evaluator } from "expression/evaluator";
 import { Settings } from "settings";
+import path from "path-browserify";
 
 /** Central, index storage for datacore values. */
 export class Datastore {
@@ -269,12 +270,25 @@ export class Datastore {
     /** Find the corresponding object for a given link. */
     public resolveLink(rawLink: string | Link, sourcePath?: string): Indexable | undefined {
         let link = typeof rawLink === "string" ? Link.parseInner(rawLink) : rawLink;
-
         if (sourcePath) {
             const linkdest = this.metadataCache.getFirstLinkpathDest(link.path, sourcePath);
             if (linkdest) link = link.withPath(linkdest.path);
         }
 
+        if (!this.objects.has(link.path)) {
+            const normalizedSourcePath = normalizePath(sourcePath ?? "/");
+            const resolvedModuleFile = this.folder
+                .getExact(normalizePath(path.join(normalizedSourcePath, path.dirname(link.path))), (childPath) => {
+                    const moduleBasename = path.basename(link.path);
+                    const childFileBasename = path.basename(childPath);
+                    return childFileBasename === moduleBasename || childFileBasename.startsWith(`${moduleBasename}.`);
+                })
+                .values()
+                .next().value;
+
+            link.path = resolvedModuleFile;
+        }
+
         const file = this.objects.get(link.path);
         if (!file) return undefined;
 
diff --git a/src/ui/javascript.tsx b/src/ui/javascript.tsx
@@ -1,9 +1,9 @@
 import { ErrorMessage, SimpleErrorBoundary, CURRENT_FILE_CONTEXT, DatacoreContextProvider } from "ui/markdown";
 import { App, MarkdownRenderChild } from "obsidian";
 import { DatacoreLocalApi } from "api/local-api";
-import { h, render, Fragment, VNode } from "preact";
+import { render, VNode } from "preact";
 import { unmountComponentAtNode } from "preact/compat";
-import { ScriptLanguage, asyncEvalInContext, transpile } from "utils/javascript";
+import { ScriptLanguage, asyncEvalInContext, defaultScriptLoadingContext, transpile } from "utils/javascript";
 import { LoadingBoundary, ScriptContainer } from "./loading-boundary";
 import { Datacore } from "index/datacore";
 
@@ -29,13 +29,16 @@ export class DatacoreJSRenderer extends MarkdownRenderChild {
 
         // Attempt to parse and evaluate the script to produce either a renderable JSX object or a function.
         try {
-            const primitiveScript = transpile(this.script, this.language);
+            const primitiveScript = transpile({
+                scriptSource: this.script,
+                scriptLanguage: this.language,
+                scriptFile: this.api.app.vault.getFileByPath(this.path)!,
+            });
+            const scriptContext = defaultScriptLoadingContext(this.api);
             const renderer = async () => {
-                return await asyncEvalInContext(primitiveScript, {
-                    dc: this.api,
-                    h: h,
-                    Fragment: Fragment,
-                });
+                const loadedScript =
+                    (await asyncEvalInContext(primitiveScript, scriptContext)) ?? scriptContext.exports;
+                return loadedScript;
             };
 
             render(
diff --git a/src/ui/loading-boundary.tsx b/src/ui/loading-boundary.tsx
@@ -7,6 +7,8 @@ import { ErrorMessage, Lit } from "./markdown";
 
 import "./errors.css";
 
+type Renderable = Literal | VNode | Function;
+
 function LoadingProgress({ datacore }: { datacore: Datacore }) {
     useIndexUpdates(datacore, { debounce: 250 });
 
@@ -53,7 +55,7 @@ export function ScriptContainer({
     executor,
     sourcePath,
 }: {
-    executor: () => Promise<Literal | VNode | Function>;
+    executor: () => Promise<Renderable | { default: () => Renderable }>;
     sourcePath: string;
 }) {
     const [element, setElement] = useState<JSX.Element | undefined>(undefined);
@@ -64,7 +66,12 @@ export function ScriptContainer({
         setError(undefined);
 
         executor()
-            .then((result) => setElement(makeRenderableElement(result, sourcePath)))
+            .then((result) => {
+                if (result && result.hasOwnProperty("default")) {
+                    return setElement(makeRenderableElement((result as any).default, sourcePath));
+                }
+                return setElement(makeRenderableElement(result, sourcePath));
+            })
             .catch((error) => setError(error));
     }, [executor]);
 
diff --git a/src/utils/javascript.ts b/src/utils/javascript.ts
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -90,7 +90,7 @@ export class DatacoreLocalApi {`
`90`	`90`	* ```
`91`	`91`	`*/`
`92`	`92`	`public async require(path: string \| Link): Promise<any> {`
`93`		`- const result = await this.scriptCache.load(path, { dc: this });`
	`93`	`+ const result = await this.scriptCache.load(path, this);`
`94`	`94`	`return result.orElseThrow();`
`95`	`95`	`}`
`96`	`96`