Skip to content

Commit 23be4b0

Browse files
dljsjrdljsjr
committed
feat: Improved ergonomics for import/require
This commit improves the ergonomics aroud importing/requiring scripts from other files or blocks, while remaining backwards compatible with the existing functionality in Datacore. The specifics of the changes are: - Paths and wikilink-like strings passed to `dc.require()` will be checked against all elements in the `scriptRoots` Setting if the path fails to resolve against the vault root - The `imports` transform is now enabled on the Sucrase `transform` operations. This causes `import { foo } from "bar"` syntax to be transformed in to `const _foo = require("bar")` during transpiling. - The script transpilation pass will convert any `foo = require("bar")` statements in to the form `foo = await dc.require("bar")` before evaluation. This covers both explicitly authored CommonJS require syntax, as well as the output of the Sucrase transformation for ES Module imports. - The script transpilation pass will resolve relative paths (e.g. `./foo`) against the script that called `dc.require()`, allowing for relative imports to now be utilized. This happens as part of the same new transform sa the one above that converts all `require` statements in to `dc.require` statements. - The `imports` transform also allows for the use of the `export` keyword in scripts, which gets transformed in to keys on an `exports` object. - The `exports` object is injected via the context used to construct the `Function` evaluator. - Scripts can now choose to either `return { ... }`, as is the current pattern, *or* they can use `export`. - Returned values take precedence over the exports object. - Datacore code blocks that return a View to be rendered can now use `export default` by leveraging the same effects introduced by the `imports` transform. - Similarly to scripts loaded with `dc.require()`, the `return function ...` form is still supported. - Also similarly to the above, the `return function ...` form is given precedence over the `export default` form.
1 parent 63f96c0 commit 23be4b0

File tree

8 files changed

+251
-112
lines changed

8 files changed

+251
-112
lines changed

‎package.json‎

Lines changed: 61 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,63 @@
11
{
2-
"name": "datacore",
3-
"version": "0.1.19",
4-
"description": "Reactive data engine for Obsidian.md.",
5-
"main": "lib/index.js",
6-
"types": "lib/index.d.ts",
7-
"files": [
8-
"lib/**/*"
9-
],
10-
"scripts": {
11-
"dev": "node esbuild.config.mjs",
12-
"build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
13-
"test": "yarn run jest",
14-
"test-watch": "yarn run jest -i --watch --no-cache",
15-
"check-format": "yarn run prettier --check src",
16-
"format": "yarn run prettier --write src"
17-
},
18-
"keywords": [
19-
"obsidian",
20-
"datacore",
21-
"dataview",
22-
"pkm"
23-
],
24-
"author": "Michael Brenan",
25-
"license": "MIT",
26-
"devDependencies": {
27-
"@codemirror/language": "https://github.com/lishid/cm-language",
28-
"@codemirror/state": "^6.0.1",
29-
"@codemirror/view": "^6.0.1",
30-
"@types/jest": "^27.0.1",
31-
"@types/luxon": "^2.3.2",
32-
"@types/node": "^16.7.13",
33-
"@types/parsimmon": "^1.10.6",
34-
"builtin-modules": "3.3.0",
35-
"esbuild": "^0.16.11",
36-
"esbuild-plugin-inline-worker": "https://github.com/mitschabaude/esbuild-plugin-inline-worker",
37-
"jest": "^27.1.0",
38-
"obsidian": "^1.6.6",
39-
"prettier": "2.3.2",
40-
"ts-jest": "^27.0.5",
41-
"ts-node": "^10.9.2",
42-
"tslib": "^2.3.1",
43-
"typescript": "^5.4.2"
44-
},
45-
"dependencies": {
46-
"@datastructures-js/queue": "^4.2.3",
47-
"@fortawesome/fontawesome-svg-core": "^6.4.0",
48-
"@fortawesome/free-solid-svg-icons": "^6.4.0",
49-
"@fortawesome/react-fontawesome": "^0.2.0",
50-
"emoji-regex": "^10.2.1",
51-
"flatqueue": "^2.0.3",
52-
"localforage": "1.10.0",
53-
"luxon": "^2.4.0",
54-
"parsimmon": "^1.18.0",
55-
"preact": "^10.17.1",
56-
"react-select": "^5.8.0",
57-
"sorted-btree": "^1.8.1",
58-
"sucrase": "3.35.0",
59-
"yaml": "^2.3.3"
60-
}
2+
"name": "datacore",
3+
"version": "0.1.19",
4+
"description": "Reactive data engine for Obsidian.md.",
5+
"main": "lib/index.js",
6+
"types": "lib/index.d.ts",
7+
"files": [
8+
"lib/**/*"
9+
],
10+
"scripts": {
11+
"dev": "node esbuild.config.mjs",
12+
"build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
13+
"test": "yarn run jest",
14+
"test-watch": "yarn run jest -i --watch --no-cache",
15+
"check-format": "yarn run prettier --check src",
16+
"format": "yarn run prettier --write src"
17+
},
18+
"keywords": [
19+
"obsidian",
20+
"datacore",
21+
"dataview",
22+
"pkm"
23+
],
24+
"author": "Michael Brenan",
25+
"license": "MIT",
26+
"devDependencies": {
27+
"@codemirror/language": "https://github.com/lishid/cm-language",
28+
"@codemirror/state": "^6.0.1",
29+
"@codemirror/view": "^6.0.1",
30+
"@types/jest": "^27.0.1",
31+
"@types/luxon": "^2.3.2",
32+
"@types/node": "^16.7.13",
33+
"@types/parsimmon": "^1.10.6",
34+
"@types/path-browserify": "^1.0.3",
35+
"builtin-modules": "3.3.0",
36+
"esbuild": "^0.16.11",
37+
"esbuild-plugin-inline-worker": "https://github.com/mitschabaude/esbuild-plugin-inline-worker",
38+
"jest": "^27.1.0",
39+
"obsidian": "^1.6.6",
40+
"prettier": "2.3.2",
41+
"ts-jest": "^27.0.5",
42+
"ts-node": "^10.9.2",
43+
"tslib": "^2.3.1",
44+
"typescript": "^5.4.2"
45+
},
46+
"dependencies": {
47+
"@datastructures-js/queue": "^4.2.3",
48+
"@fortawesome/fontawesome-svg-core": "^6.4.0",
49+
"@fortawesome/free-solid-svg-icons": "^6.4.0",
50+
"@fortawesome/react-fontawesome": "^0.2.0",
51+
"emoji-regex": "^10.2.1",
52+
"flatqueue": "^2.0.3",
53+
"localforage": "1.10.0",
54+
"luxon": "^2.4.0",
55+
"parsimmon": "^1.18.0",
56+
"path-browserify": "^1.0.1",
57+
"preact": "^10.17.1",
58+
"react-select": "^5.8.0",
59+
"sorted-btree": "^1.8.1",
60+
"sucrase": "3.35.0",
61+
"yaml": "^2.3.3"
62+
}
6163
}

‎src/api/local-api.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ export class DatacoreLocalApi {
9090
* ```
9191
*/
9292
public async require(path: string | Link): Promise<any> {
93-
const result = await this.scriptCache.load(path, { dc: this });
93+
const result = await this.scriptCache.load(path, this);
9494
return result.orElseThrow();
9595
}
9696

‎src/api/script-cache.ts‎

Lines changed: 55 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,16 @@ import { Datastore } from "index/datastore";
33
import { Result } from "./result";
44
import { MarkdownCodeblock, MarkdownSection } from "index/types/markdown";
55
import { Deferred, deferred } from "utils/deferred";
6-
import { ScriptLanguage, asyncEvalInContext, transpile } from "utils/javascript";
6+
import {
7+
ScriptDefinition,
8+
ScriptLanguage,
9+
asyncEvalInContext,
10+
defaultScriptLoadingContext,
11+
transpile,
12+
} from "utils/javascript";
713
import { lineRange } from "utils/normalizers";
8-
import { TFile } from "obsidian";
9-
import { Fragment, h } from "preact";
14+
import { normalizePath, TFile } from "obsidian";
15+
import { DatacoreLocalApi } from "./local-api";
1016

1117
/** A script that is currently being loaded. */
1218
export interface LoadingScript {
@@ -56,20 +62,34 @@ export class ScriptCache {
5662
public constructor(private store: Datastore) {}
5763

5864
/** Load the given script at the given path, recursively loading any subscripts as well. */
59-
public async load(path: string | Link, context: Record<string, any>): Promise<Result<any, string>> {
65+
public async load(path: string | Link, api: DatacoreLocalApi): Promise<Result<any, string>> {
66+
// First, attempt to resolve the script against the script roots so we cache a canonical script path as the key.
67+
var linkToLoad = undefined;
68+
const roots = ["", ...api.core.settings.scriptRoots];
69+
for (var i = 0; i < roots.length; i++) {
70+
linkToLoad = this.store.tryNormalizeLink(path, normalizePath(roots[i]));
71+
if (linkToLoad) {
72+
break;
73+
}
74+
}
75+
76+
const resolvedPath = linkToLoad ?? path;
77+
6078
// Always check the cache first.
61-
const key = this.pathkey(path);
79+
const key = this.pathkey(resolvedPath);
6280
const currentScript = this.scripts.get(key);
6381
if (currentScript) {
64-
if (currentScript.type === "loaded") return Result.success(currentScript.object);
82+
if (currentScript.type === "loaded") {
83+
return Result.success(currentScript.object);
84+
}
6585

6686
// TODO: If we try to load an already-loading script, we are almost certainly doing something
6787
// weird. Either the caller is not `await`-ing the load and loading multiple times, OR
6888
// we are in a `require()` loop. Either way, we'll error out for now since we can't handle
6989
// either case currently.
7090
return Result.failure(
71-
`Failed to import script "${path.toString()}", as it is in the middle of being loaded. Do you have
72-
a circular dependency in your require() calls? The currently loaded or loading scripts are:
91+
`Failed to import script "${resolvedPath.toString()}", as it is in the middle of being loaded. Do you have
92+
a circular dependency in your require() calls? The currently loaded or loading scripts are:
7393
${Array.from(this.scripts.values())
7494
.map((sc) => "\t" + sc.path)
7595
.join("\n")}`
@@ -80,7 +100,7 @@ export class ScriptCache {
80100
const deferral = deferred<Result<any, string>>();
81101
this.scripts.set(key, { type: "loading", promise: deferral, path: key });
82102

83-
const result = await this.loadUncached(path, context);
103+
const result = await this.loadUncached(resolvedPath, api);
84104
deferral.resolve(result);
85105

86106
if (result.successful) {
@@ -93,25 +113,30 @@ export class ScriptCache {
93113
}
94114

95115
/** Load a script, directly bypassing the cache. */
96-
private async loadUncached(path: string | Link, context: Record<string, any>): Promise<Result<any, string>> {
97-
const maybeSource = await this.resolveSource(path);
116+
private async loadUncached(scriptPath: string | Link, api: DatacoreLocalApi): Promise<Result<any, string>> {
117+
var maybeSource = await this.resolveSource(scriptPath);
98118
if (!maybeSource.successful) return maybeSource;
99119

100120
// Transpile to vanilla javascript first...
101-
const { code, language } = maybeSource.value;
121+
const scriptDefinition = maybeSource.value;
102122
let basic;
103123
try {
104-
basic = transpile(code, language);
124+
basic = transpile(scriptDefinition);
105125
} catch (error) {
106-
return Result.failure(`Failed to import ${path.toString()} while transpiling from ${language}: ${error}`);
126+
return Result.failure(
127+
`Failed to import ${scriptPath.toString()} while transpiling from ${
128+
scriptDefinition.scriptLanguage
129+
}: ${error}`
130+
);
107131
}
108132

109133
// Then finally execute the script to 'load' it.
110-
const finalContext = Object.assign({ h: h, Fragment: Fragment }, context);
134+
const scriptContext = defaultScriptLoadingContext(api);
111135
try {
112-
return Result.success(await asyncEvalInContext(basic, finalContext));
136+
const loadRet = (await asyncEvalInContext(basic, scriptContext)) ?? scriptContext.exports;
137+
return Result.success(loadRet);
113138
} catch (error) {
114-
return Result.failure(`Failed to execute script '${path.toString()}': ${error}`);
139+
return Result.failure(`Failed to execute script '${scriptPath.toString()}': ${error}`);
115140
}
116141
}
117142

@@ -122,10 +147,8 @@ export class ScriptCache {
122147
}
123148

124149
/** Attempts to resolve the source to load given a path or link to a markdown section. */
125-
private async resolveSource(
126-
path: string | Link
127-
): Promise<Result<{ code: string; language: ScriptLanguage }, string>> {
128-
const object = this.store.resolveLink(path);
150+
private async resolveSource(path: string | Link, sourcePath?: string): Promise<Result<ScriptDefinition, string>> {
151+
const object = this.store.resolveLink(path, sourcePath);
129152
if (!object) return Result.failure("Could not find a script at the given path: " + path.toString());
130153

131154
const tfile = this.store.vault.getFileByPath(object.$file!);
@@ -137,7 +160,7 @@ export class ScriptCache {
137160

138161
try {
139162
const code = await this.store.vault.cachedRead(tfile);
140-
return Result.success({ code, language });
163+
return Result.success({ scriptFile: tfile, scriptLanguage: language, scriptSource: code });
141164
} catch (error) {
142165
return Result.failure("Failed to load javascript/typescript source file: " + error);
143166
}
@@ -158,7 +181,11 @@ export class ScriptCache {
158181
ScriptCache.SCRIPT_LANGUAGES[
159182
maybeBlock.$languages.find((lang) => lang.toLocaleLowerCase() in ScriptCache.SCRIPT_LANGUAGES)!
160183
];
161-
return (await this.readCodeblock(tfile, maybeBlock)).map((code) => ({ code, language }));
184+
return (await this.readCodeblock(tfile, maybeBlock)).map((code) => ({
185+
scriptFile: tfile,
186+
scriptSource: code,
187+
scriptLanguage: language,
188+
}));
162189
} else if (object instanceof MarkdownCodeblock) {
163190
const maybeLanguage = object.$languages.find(
164191
(lang) => lang.toLocaleLowerCase() in ScriptCache.SCRIPT_LANGUAGES
@@ -167,7 +194,11 @@ export class ScriptCache {
167194
return Result.failure(`The codeblock referenced by '${path}' is not a JS/TS codeblock.`);
168195

169196
const language = ScriptCache.SCRIPT_LANGUAGES[maybeLanguage];
170-
return (await this.readCodeblock(tfile, object)).map((code) => ({ code, language }));
197+
return (await this.readCodeblock(tfile, object)).map((code) => ({
198+
scriptFile: tfile,
199+
scriptSource: code,
200+
scriptLanguage: language,
201+
}));
171202
}
172203

173204
return Result.failure(`Cannot import '${path.toString()}: not a JS/TS file or codeblock reference.`);

‎src/index/datastore.ts‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { FolderIndex } from "index/storage/folder";
44
import { InvertedIndex } from "index/storage/inverted";
55
import { IndexPrimitive, IndexQuery, IndexSource } from "index/types/index-query";
66
import { Indexable, LINKABLE_TYPE, LINKBEARING_TYPE, TAGGABLE_TYPE } from "index/types/indexable";
7-
import { MetadataCache, Vault } from "obsidian";
7+
import { MetadataCache, normalizePath, Vault } from "obsidian";
88
import { MarkdownPage } from "./types/markdown";
99
import { extractSubtags, normalizeHeaderForLink } from "utils/normalizers";
1010
import FlatQueue from "flatqueue";
@@ -14,6 +14,7 @@ import { IndexResolver, execute, optimizeQuery } from "index/storage/query-execu
1414
import { Result } from "api/result";
1515
import { Evaluator } from "expression/evaluator";
1616
import { Settings } from "settings";
17+
import path from "path-browserify";
1718

1819
/** Central, index storage for datacore values. */
1920
export class Datastore {
@@ -266,15 +267,44 @@ export class Datastore {
266267
this.revision++;
267268
}
268269

269-
/** Find the corresponding object for a given link. */
270-
public resolveLink(rawLink: string | Link, sourcePath?: string): Indexable | undefined {
270+
public tryNormalizeLink(rawLink: string | Link, sourcePath?: string): Link | undefined {
271271
let link = typeof rawLink === "string" ? Link.parseInner(rawLink) : rawLink;
272-
273272
if (sourcePath) {
274273
const linkdest = this.metadataCache.getFirstLinkpathDest(link.path, sourcePath);
275274
if (linkdest) link = link.withPath(linkdest.path);
276275
}
277276

277+
if (this.objects.has(link.path)) {
278+
return link;
279+
}
280+
281+
const normalizedSourcePath = normalizePath(sourcePath ?? "/");
282+
const normalizedModuleParentDir = normalizePath(path.join(normalizedSourcePath, path.dirname(link.path)));
283+
const resolvedModuleFile = this.folder
284+
.getExact(normalizedModuleParentDir, (childPath) => {
285+
const moduleBasename = path.basename(link.path);
286+
const childFileBasename = path.basename(childPath);
287+
return childFileBasename === moduleBasename || childFileBasename.startsWith(`${moduleBasename}.`);
288+
})
289+
.values()
290+
.next()?.value;
291+
if (resolvedModuleFile) {
292+
return link.withPath(resolvedModuleFile);
293+
}
294+
295+
return undefined;
296+
}
297+
298+
public normalizeLink(rawLink: string | Link, sourcePath?: string): Link {
299+
return (
300+
this.tryNormalizeLink(rawLink, sourcePath) ??
301+
(typeof rawLink === "string" ? Link.parseInner(rawLink) : rawLink)
302+
);
303+
}
304+
305+
/** Find the corresponding object for a given link. */
306+
public resolveLink(rawLink: string | Link, sourcePath?: string): Indexable | undefined {
307+
const link = this.normalizeLink(rawLink, sourcePath);
278308
const file = this.objects.get(link.path);
279309
if (!file) return undefined;
280310

@@ -288,8 +318,9 @@ export class Datastore {
288318
(sec) => normalizeHeaderForLink(sec.$title) == link.subpath || sec.$title == link.subpath
289319
);
290320

291-
if (section) return section;
292-
else return undefined;
321+
if (section) {
322+
return section;
323+
} else return undefined;
293324
} else if (link.type === "block") {
294325
for (const section of file.$sections) {
295326
const block = section.$blocks.find((bl) => bl.$blockId === link.subpath);

‎src/ui/javascript.tsx‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import { ErrorMessage, SimpleErrorBoundary, CURRENT_FILE_CONTEXT, DatacoreContextProvider } from "ui/markdown";
22
import { App, MarkdownRenderChild } from "obsidian";
33
import { DatacoreLocalApi } from "api/local-api";
4-
import { h, render, Fragment, VNode } from "preact";
4+
import { render, VNode } from "preact";
55
import { unmountComponentAtNode } from "preact/compat";
6-
import { ScriptLanguage, asyncEvalInContext, transpile } from "utils/javascript";
6+
import { ScriptLanguage, asyncEvalInContext, defaultScriptLoadingContext, transpile } from "utils/javascript";
77
import { LoadingBoundary, ScriptContainer } from "./loading-boundary";
88
import { Datacore } from "index/datacore";
99

@@ -29,13 +29,16 @@ export class DatacoreJSRenderer extends MarkdownRenderChild {
2929

3030
// Attempt to parse and evaluate the script to produce either a renderable JSX object or a function.
3131
try {
32-
const primitiveScript = transpile(this.script, this.language);
32+
const primitiveScript = transpile({
33+
scriptSource: this.script,
34+
scriptLanguage: this.language,
35+
scriptFile: this.api.app.vault.getFileByPath(this.path)!,
36+
});
37+
const scriptContext = defaultScriptLoadingContext(this.api);
3338
const renderer = async () => {
34-
return await asyncEvalInContext(primitiveScript, {
35-
dc: this.api,
36-
h: h,
37-
Fragment: Fragment,
38-
});
39+
const loadedScript =
40+
(await asyncEvalInContext(primitiveScript, scriptContext)) ?? scriptContext.exports;
41+
return loadedScript;
3942
};
4043

4144
render(

0 commit comments

Comments
 (0)