diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 1eaec529..ef480b88 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -57,6 +57,53 @@ npm run build-ws # 5-10 minutes npm run lint # 2-3 minutes ``` +## CMake Configuration + +### **CRITICAL**: ALWAYS Use VS Code CMake Extension + +**YOU MUST use the VS Code CMake extension for ALL CMake operations.** Manual cmake commands will fail because they don't properly set up the Emscripten environment. + +**REQUIRED WORKFLOW** when modifying C++ code or vcpkg packages (patches, portfile.cmake): + +1. **Remove vcpkg cache**: `rm -rf vcpkg/buildtrees/` +2. **Use VS Code Command Palette** (Ctrl/Cmd+Shift+P): + - **`CMake: Configure`** - Reconfigures CMake and reinstalls vcpkg packages with new patches + - **`CMake: Build`** - Builds the configured targets + - **`CMake: Clean Rebuild`** - Cleans and rebuilds everything + +**WHY THIS IS REQUIRED**: +- Sets up Emscripten SDK environment variables correctly +- Configures vcpkg toolchain properly +- Handles WASM-specific build flags +- Provides IntelliSense and debugging integration + +**DO NOT use manual cmake commands like:** +- ❌ `cmake --preset vcpkg-emscripten-*` (will fail with "emcc compiler not found") +- ❌ `cmake --build build --target ` (won't work without proper environment) +- ❌ `ninja ` (missing environment and configuration) + +**EXCEPTION**: `npm run build-cpp` scripts are pre-configured with the correct environment and are safe to use for full builds. + +## Embind (C++ <-> JS Interop) + +This repo uses Emscripten Embind to expose C++ APIs to JavaScript/TypeScript. + +Reference: https://emscripten.org/docs/porting/connecting_cpp_and_javascript/embind.html + +### Guidelines +- Prefer Embind over ad-hoc JS glue when exposing C++ to JS. +- Avoid throwing across the WASM boundary; catch exceptions in C++ and translate into explicit error results. +- Be explicit about ownership/lifetime: + - If you return raw pointers (often created with `new`) via Embind, JS must eventually free them (e.g. via the generated `.delete()` method or `Module.destroy(obj)` depending on how the wrapper is used). + - If practical, bind smart pointers (e.g. `std::unique_ptr` / `std::shared_ptr`) to make ownership explicit. +- For JS values, prefer `emscripten::val` and convert at the boundary (e.g. arrays -> `val.isArray()` + `val["length"]`). Validate types and reject non-finite numbers. +- Keep interop-friendly result types: materialize streaming results where needed for JS access patterns. + +### Common Patterns Used Here +- `EMSCRIPTEN_BINDINGS(...)` with `emscripten::class_` and `.function(...)`. +- `allow_raw_pointers()` is used for functions that accept/return pointer types. +- Provide small helper conversions (JS `val` -> C++ types) and centralize error messages for predictable JS behavior. + ## Working Without Full Build ### What Works with npm ci + npm run lint Only diff --git a/.gitignore b/.gitignore index a6d68ed4..7cc31ca0 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ types/ /wasi-sdk* /wit-bindgen /vcpkg +__screenshots__/ .coveralls.yml *.tsbuildinfo diff --git a/.vscode/settings.json b/.vscode/settings.json index 7623d70e..bb6b3c91 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,40 +1,5 @@ { "cmake.cmakePath": "${workspaceFolder}/scripts/cmake.sh", "vitest.maximumConfigs": 20, - "vitest.configSearchPatternExclude": "{emsdk/**,node_modules/**}", - - // AI Assistant Configuration - "github.copilot.enable": { - "*": true, - "markdown": true, - "typescript": true, - "javascript": true - }, - - // File associations for better AI context - "files.associations": { - "*.idl": "webidl", - "*.wasm": "wasm", - ".copilot-*": "markdown" - }, - - // Exclude build artifacts from AI context - "files.exclude": { - "**/build/**": true, - "**/dist/**": true, - "**/node_modules/**": true, - "**/emsdk/**": true, - "**/vcpkg/**": true, - "**/.nyc_output/**": true - }, - - // Include important files for AI context - "search.exclude": { - "**/node_modules": true, - "**/emsdk": true, - "**/vcpkg": true, - "**/build": true, - "**/dist": false, - "**/.copilot-*": false - } + "vitest.configSearchPatternExclude": "{emsdk/**,node_modules/**}" } \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index bfc2c657..c3a839d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,6 +82,7 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug") endif () add_subdirectory(packages/base91/src-cpp) +add_subdirectory(packages/duckdb/src-cpp) add_subdirectory(packages/expat/src-cpp) add_subdirectory(packages/graphviz/src-cpp) add_subdirectory(packages/llama/src-cpp) diff --git a/README.md b/README.md index 08103001..d63e13b3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This repository contains a collection of useful c++ libraries compiled to WASM for (re)use in Node JS, Web Browsers and JavaScript Libraries: - [base91](https://base91.sourceforge.net/) - v0.6.0 -- [duckdb](https://github.com/duckdb/duckdb) - v1.4.0 +- [duckdb](https://github.com/duckdb/duckdb) - v1.4.3 - [expat](https://libexpat.github.io/) - v2.7.1 - [graphviz](https://www.graphviz.org/) - 14.1.0 - [llama.cpp](https://github.com/ggerganov/llama.cpp) - b3718 diff --git a/package-lock.json b/package-lock.json index 49279e12..20a3a8b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,9 +13,9 @@ "tests/bundlers" ], "devDependencies": { - "@eslint/js": "9.39.1", + "@eslint/js": "9.39.2", "@types/emscripten": "1.41.5", - "@types/node": "24.10.2", + "@types/node": "24.10.4", "@types/yargs": "17.0.35", "@typescript-eslint/parser": "8.49.0", "@vitest/browser": "4.0.15", @@ -24,7 +24,7 @@ "@vitest/coverage-v8": "4.0.15", "assemblyscript": "0.28.9", "chokidar-cli": "3.0.0", - "eslint": "9.39.1", + "eslint": "9.39.2", "globals": "16.5.0", "happy-dom": "20.0.11", "lerna": "9.0.3", @@ -649,16 +649,6 @@ } } }, - "node_modules/@duckdb/duckdb-wasm": { - "version": "1.31.0", - "resolved": "https://registry.npmjs.org/@duckdb/duckdb-wasm/-/duckdb-wasm-1.31.0.tgz", - "integrity": "sha512-HcR5XFmNpi7ua0IFohO2dFDNh0LkZgnkfkiD5BDlP8SAmhhcHe+zfOPD1izpmuBxLel25LHO608NY/3RolwztQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "apache-arrow": "^17.0.0" - } - }, "node_modules/@emnapi/core": { "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.7.1.tgz", @@ -1301,9 +1291,9 @@ } }, "node_modules/@eslint/js": { - "version": "9.39.1", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.1.tgz", - "integrity": "sha512-S26Stp4zCy88tH94QbBv3XCuzRQiZ9yXofEILmglYTh/Ug/a9/umqvgFtYBAo3Lp0nsI/5/qH1CCrbdK3AP1Tw==", + "version": "9.39.2", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz", + "integrity": "sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==", "dev": true, "license": "MIT", "engines": { @@ -4180,16 +4170,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@swc/helpers": { - "version": "0.5.17", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.17.tgz", - "integrity": "sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.8.0" - } - }, "node_modules/@tufjs/canonical-json": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tufjs/canonical-json/-/canonical-json-2.0.0.tgz", @@ -4235,20 +4215,6 @@ "assertion-error": "^2.0.1" } }, - "node_modules/@types/command-line-args": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz", - "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/command-line-usage": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.4.tgz", - "integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/deep-eql": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", @@ -4359,9 +4325,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "24.10.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.2.tgz", - "integrity": "sha512-WOhQTZ4G8xZ1tjJTvKOpyEVSGgOTvJAfDK3FNFgELyaTpzhdgHVHeqW8V+UJvzF5BT+/B54T/1S2K6gd9c7bbA==", + "version": "24.10.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.4.tgz", + "integrity": "sha512-vnDVpYPMzs4wunl27jHrfmwojOGKya0xyM3sH+UE5iv5uPS6vX7UIoh6m+vQc5LGBq52HBKPIn/zcSZVzeDEZg==", "dev": true, "license": "MIT", "dependencies": { @@ -5647,44 +5613,6 @@ "node": ">= 8" } }, - "node_modules/apache-arrow": { - "version": "17.0.0", - "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-17.0.0.tgz", - "integrity": "sha512-X0p7auzdnGuhYMVKYINdQssS4EcKec9TCXyez/qtJt32DrIMGbzqiaMiQ0X6fQlQpw8Fl0Qygcv4dfRAr5Gu9Q==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@swc/helpers": "^0.5.11", - "@types/command-line-args": "^5.2.3", - "@types/command-line-usage": "^5.0.4", - "@types/node": "^20.13.0", - "command-line-args": "^5.2.1", - "command-line-usage": "^7.0.1", - "flatbuffers": "^24.3.25", - "json-bignum": "^0.0.3", - "tslib": "^2.6.2" - }, - "bin": { - "arrow2csv": "bin/arrow2csv.cjs" - } - }, - "node_modules/apache-arrow/node_modules/@types/node": { - "version": "20.19.26", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.26.tgz", - "integrity": "sha512-0l6cjgF0XnihUpndDhk+nyD3exio3iKaYROSgvh/qSevPXax3L8p5DBRFjbvalnwatGgHEQn2R88y2fA3g4irg==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/apache-arrow/node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, - "license": "MIT" - }, "node_modules/aproba": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", @@ -5699,16 +5627,6 @@ "dev": true, "license": "Python-2.0" }, - "node_modules/array-back": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", - "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/array-buffer-byte-length": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.2.tgz", @@ -6389,22 +6307,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/chalk-template": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", - "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.1.2" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/chalk-template?sponsor=1" - } - }, "node_modules/character-entities-html4": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", @@ -6860,58 +6762,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/command-line-args": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", - "integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==", - "dev": true, - "license": "MIT", - "dependencies": { - "array-back": "^3.1.0", - "find-replace": "^3.0.0", - "lodash.camelcase": "^4.3.0", - "typical": "^4.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/command-line-usage": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.3.tgz", - "integrity": "sha512-PqMLy5+YGwhMh1wS04mVG44oqDsgyLRSKJBdOo1bnYhMKBW65gZF1dRp2OZRhiTjgUHljy99qkO7bsctLaw35Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "array-back": "^6.2.2", - "chalk-template": "^0.4.0", - "table-layout": "^4.1.0", - "typical": "^7.1.1" - }, - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/command-line-usage/node_modules/array-back": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", - "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.17" - } - }, - "node_modules/command-line-usage/node_modules/typical": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-7.3.0.tgz", - "integrity": "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.17" - } - }, "node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", @@ -8034,9 +7884,9 @@ } }, "node_modules/eslint": { - "version": "9.39.1", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.1.tgz", - "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", + "version": "9.39.2", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", + "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", "dependencies": { @@ -8046,7 +7896,7 @@ "@eslint/config-helpers": "^0.4.2", "@eslint/core": "^0.17.0", "@eslint/eslintrc": "^3.3.1", - "@eslint/js": "9.39.1", + "@eslint/js": "9.39.2", "@eslint/plugin-kit": "^0.4.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", @@ -8429,19 +8279,6 @@ "node": ">=8" } }, - "node_modules/find-replace": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz", - "integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "array-back": "^3.0.1" - }, - "engines": { - "node": ">=4.0.0" - } - }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -8483,13 +8320,6 @@ "node": ">=16" } }, - "node_modules/flatbuffers": { - "version": "24.12.23", - "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-24.12.23.tgz", - "integrity": "sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==", - "dev": true, - "license": "Apache-2.0" - }, "node_modules/flatted": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", @@ -10518,15 +10348,6 @@ "node": ">=6" } }, - "node_modules/json-bignum": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz", - "integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==", - "dev": true, - "engines": { - "node": ">=0.8" - } - }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -11194,13 +11015,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", - "dev": true, - "license": "MIT" - }, "node_modules/lodash.debounce": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", @@ -15765,30 +15579,6 @@ "dev": true, "license": "MIT" }, - "node_modules/table-layout": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-4.1.1.tgz", - "integrity": "sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==", - "dev": true, - "license": "MIT", - "dependencies": { - "array-back": "^6.2.2", - "wordwrapjs": "^5.1.0" - }, - "engines": { - "node": ">=12.17" - } - }, - "node_modules/table-layout/node_modules/array-back": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", - "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.17" - } - }, "node_modules/tapable": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.0.tgz", @@ -16432,16 +16222,6 @@ "typescript": ">=4.8.4 <6.0.0" } }, - "node_modules/typical": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz", - "integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/uc.micro": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", @@ -18437,16 +18217,6 @@ "dev": true, "license": "MIT" }, - "node_modules/wordwrapjs": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.1.tgz", - "integrity": "sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.17" - } - }, "node_modules/wrap-ansi": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", @@ -18954,25 +18724,7 @@ "version": "1.12.0", "license": "Apache-2.0", "devDependencies": { - "@duckdb/duckdb-wasm": "1.31.0", - "@hpcc-js/esbuild-plugins": "1.7.0", - "mkdirp": "3.0.1" - } - }, - "packages/duckdb/node_modules/mkdirp": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", - "dev": true, - "license": "MIT", - "bin": { - "mkdirp": "dist/cjs/src/bin.js" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" + "@hpcc-js/esbuild-plugins": "1.7.0" } }, "packages/expat": { diff --git a/package.json b/package.json index 7868549c..6b740e87 100644 --- a/package.json +++ b/package.json @@ -85,9 +85,9 @@ "update-major": "npm run update-major-root && lerna run update-major" }, "devDependencies": { - "@eslint/js": "9.39.1", + "@eslint/js": "9.39.2", "@types/emscripten": "1.41.5", - "@types/node": "24.10.2", + "@types/node": "24.10.4", "@types/yargs": "17.0.35", "@typescript-eslint/parser": "8.49.0", "@vitest/browser": "4.0.15", @@ -96,7 +96,7 @@ "@vitest/coverage-v8": "4.0.15", "assemblyscript": "0.28.9", "chokidar-cli": "3.0.0", - "eslint": "9.39.1", + "eslint": "9.39.2", "globals": "16.5.0", "happy-dom": "20.0.11", "lerna": "9.0.3", diff --git a/packages/duckdb/esbuild.js b/packages/duckdb/esbuild.js index 03a23f4d..40a6eb68 100644 --- a/packages/duckdb/esbuild.js +++ b/packages/duckdb/esbuild.js @@ -1,9 +1,10 @@ -import { browserTpl } from "@hpcc-js/esbuild-plugins"; +import { neutralTpl } from "@hpcc-js/esbuild-plugins"; import { sfxWasm } from "@hpcc-js/esbuild-plugins/sfx-wrapper"; import { replaceFunction, replaceString } from "../../utils/esbuild-plugins.js"; // config --- -await browserTpl("src/index.ts", "dist/index", { +await neutralTpl("src/index.ts", "dist/index", { + plugins: [ replaceFunction({ 'findWasmBinary': 'const findWasmBinary=()=>"";' diff --git a/packages/duckdb/package.json b/packages/duckdb/package.json index c9a87f96..23a69ddd 100644 --- a/packages/duckdb/package.json +++ b/packages/duckdb/package.json @@ -5,34 +5,33 @@ "type": "module", "exports": { ".": { - "types": "./types/src/index.d.ts", + "types": "./types/index.d.ts", "default": "./dist/index.js" } }, "main": "./dist/index.js", - "types": "./types/src/index.d.ts", + "types": "./types/index.d.ts", "files": [ "dist/**/*", "src/**/*", "types/**/*" ], "scripts": { - "clean": "rimraf coverage build dist dist-test types", - "pack-duckdb-eh-worker-node": "npx -y mkdirp build && node ./utils/sfx-wasm.js ../../node_modules/@duckdb/duckdb-wasm/dist/duckdb-node-eh.worker.cjs > ./build/duckdb-node-eh.worker.ts", - "pack-duckdb-eh-worker": "npx -y mkdirp build && node ./utils/sfx-wasm.js ../../node_modules/@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js > ./build/duckdb-browser-eh.worker.ts", - "pack-duckdb-eh": "npx -y mkdirp build && node ./utils/sfx-wasm.js ../../node_modules/@duckdb/duckdb-wasm/dist/duckdb-eh.wasm > ./build/duckdb-eh.wasm.ts", - "pack-duckdb": "run-p pack-duckdb-eh pack-duckdb-eh-worker pack-duckdb-eh-worker-node", + "clean": "rimraf coverage dist dist-test types .nyc_output", + "build-cpp": "cmake --build ../../build --target duckdblib", + "build-cpp-watch": "chokidar 'src-cpp/**.*' -c 'npm run build-cpp'", "gen-types": "tsc --project tsconfig.json --emitDeclarationOnly", "gen-types-watch": "npm run gen-types -- --watch", "bundle": "node esbuild.js", "bundle-dev": "npm run bundle -- --development", "bundle-watch": "npm run bundle-dev -- --watch", "build-dev": "run-p gen-types bundle-dev", - "build": "npm-run-all --serial pack-duckdb --parallel gen-types bundle", + "build": "run-p gen-types bundle", "lint-skypack": "npx -y @skypack/package-check", "lint-eslint": "eslint src/**/*.ts tests/**/*.ts", "lint": "run-p lint-eslint", "test-browser": "vitest run --project browser", + "test-node": "vitest run --project node", "test": "vitest run", "coverage": "vitest run --coverage", "update": "npx -y npm-check-updates -u -t minor", @@ -40,9 +39,7 @@ }, "dependencies": {}, "devDependencies": { - "@duckdb/duckdb-wasm": "1.31.0", - "@hpcc-js/esbuild-plugins": "1.7.0", - "mkdirp": "3.0.1" + "@hpcc-js/esbuild-plugins": "1.7.0" }, "keywords": [ "DuckDB", diff --git a/packages/duckdb/src-cpp/CMakeLists.txt b/packages/duckdb/src-cpp/CMakeLists.txt new file mode 100644 index 00000000..3d6b5601 --- /dev/null +++ b/packages/duckdb/src-cpp/CMakeLists.txt @@ -0,0 +1,44 @@ +project(duckdblib) + +find_package(DuckDB CONFIG REQUIRED) + +# See: https://github.com/emscripten-core/emscripten/blob/main/src/settings.js + +set(EM_CPP_FLAGS + ${EM_CPP_FLAGS} + "-msimd128" + "-fwasm-exceptions" +) +string(REPLACE ";" " " CPP_FLAGS "${EM_CPP_FLAGS}") + +set(EM_LINK_FLAGS + ${EM_LINK_FLAGS} + "-sEXPORT_NAME='${CMAKE_PROJECT_NAME}'" + "-sFILESYSTEM=1" + "-sFORCE_FILESYSTEM=1" + "-sWASMFS=1" + "-sWASM_BIGINT=1" + "-lembind" + "-fwasm-exceptions" + "--emit-tsd ${CMAKE_CURRENT_BINARY_DIR}/duckdblib.d.ts" +) +string(REPLACE ";" " " LINK_FLAGS "${EM_LINK_FLAGS}") + +set(SRCS + main.cpp +) + +include_directories( + ${VCPKG_INCLUDE_DIR} +) + +add_executable(duckdblib + ${SRCS} +) + +set_target_properties(duckdblib PROPERTIES COMPILE_FLAGS "${CPP_FLAGS}") +set_target_properties(duckdblib PROPERTIES LINK_FLAGS "${LINK_FLAGS}") + +target_link_libraries(duckdblib + PRIVATE $,duckdb,duckdb_static> +) diff --git a/packages/duckdb/src-cpp/main.cpp b/packages/duckdb/src-cpp/main.cpp new file mode 100644 index 00000000..1a5ed517 --- /dev/null +++ b/packages/duckdb/src-cpp/main.cpp @@ -0,0 +1,762 @@ +#include +#include +#include +#include +#include + +#include +#include + +#include "duckdb/main/stream_query_result.hpp" + +// Statically initialize extensions without requiring SQL LOAD. +#include "duckdb/main/extension/extension_loader.hpp" + +using namespace duckdb; + +class ColumnarQueryResult +{ +public: + enum class Kind + { + F64 = 0, + BOOL = 1, + STRING = 2 + }; + + ColumnarQueryResult() = default; + + explicit ColumnarQueryResult(const std::string &error) + { + _has_error = true; + _error = error; + } + + explicit ColumnarQueryResult(std::unique_ptr result) + { + if (!result) + { + return; + } + + _column_count = (uint32_t)result->ColumnCount(); + _row_count = (uint32_t)result->RowCount(); + + if (result->HasError()) + { + _has_error = true; + // Best-effort: this includes the error message in current DuckDB builds. + _error = result->ToString(); + } + + _names.reserve(_column_count); + _kinds.resize(_column_count, Kind::STRING); + _types.reserve(_column_count); + _validity.resize(_column_count); + _f64.resize(_column_count); + _u8.resize(_column_count); + _str.resize(_column_count); + + for (uint32_t c = 0; c < _column_count; ++c) + { + _names.push_back(result->ColumnName(c)); + _types.push_back(std::string("")); + _validity[c].resize(_row_count); + + // Infer a display type + storage kind from first non-null value. + Kind inferred = Kind::STRING; + std::string inferred_type; + for (uint32_t r = 0; r < _row_count; ++r) + { + Value v = result->GetValue(c, r); + if (!v.IsNull()) + { + inferred_type = v.type().ToString(); + switch (v.type().id()) + { + case LogicalTypeId::BOOLEAN: + inferred = Kind::BOOL; + break; + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + inferred = Kind::F64; + break; + default: + inferred = Kind::STRING; + break; + } + break; + } + } + _kinds[c] = inferred; + _types[c] = inferred_type; + + switch (inferred) + { + case Kind::F64: + _f64[c].resize(_row_count); + break; + case Kind::BOOL: + _u8[c].resize(_row_count); + break; + default: + _str[c].resize(_row_count); + break; + } + } + + for (uint32_t r = 0; r < _row_count; ++r) + { + for (uint32_t c = 0; c < _column_count; ++c) + { + Value v = result->GetValue(c, r); + if (v.IsNull()) + { + _validity[c][r] = 0; + // Leave default value. + continue; + } + _validity[c][r] = 1; + + switch (_kinds[c]) + { + case Kind::BOOL: + _u8[c][r] = v.GetValue() ? 1 : 0; + break; + case Kind::F64: + switch (v.type().id()) + { + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + _f64[c][r] = v.GetValue(); + break; + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + _f64[c][r] = (double)v.GetValue(); + break; + default: + // Fallback: if conversion isn't supported, keep a default value. + _validity[c][r] = 0; + break; + } + break; + default: + _str[c][r] = v.ToString(); + break; + } + } + } + } + + bool hasError() const { return _has_error; } + std::string error() const { return _error; } + + uint32_t rowCount() const { return _row_count; } + uint32_t columnCount() const { return _column_count; } + + std::string columnName(uint32_t column) const + { + if (column >= _names.size()) + return std::string(""); + return _names[column]; + } + + std::string columnType(uint32_t column) const + { + if (column >= _types.size()) + return std::string(""); + return _types[column]; + } + + uint32_t columnKind(uint32_t column) const + { + if (column >= _kinds.size()) + return (uint32_t)Kind::STRING; + return (uint32_t)_kinds[column]; + } + + emscripten::val validity(uint32_t column) + { + if (column >= _validity.size()) + return emscripten::val::null(); + return emscripten::val(emscripten::typed_memory_view(_row_count, _validity[column].data())); + } + + emscripten::val f64Column(uint32_t column) + { + if (column >= _f64.size() || _kinds[column] != Kind::F64) + return emscripten::val::null(); + return emscripten::val(emscripten::typed_memory_view(_row_count, _f64[column].data())); + } + + emscripten::val boolColumn(uint32_t column) + { + if (column >= _u8.size() || _kinds[column] != Kind::BOOL) + return emscripten::val::null(); + return emscripten::val(emscripten::typed_memory_view(_row_count, _u8[column].data())); + } + + emscripten::val stringColumn(uint32_t column) + { + if (column >= _str.size() || _kinds[column] != Kind::STRING) + return emscripten::val::null(); + emscripten::val arr = emscripten::val::array(); + for (uint32_t r = 0; r < _row_count; ++r) + { + if (_validity[column][r] == 0) + { + arr.call("push", emscripten::val::null()); + } + else + { + arr.call("push", emscripten::val(_str[column][r])); + } + } + return arr; + } + +private: + bool _has_error = false; + std::string _error; + uint32_t _row_count = 0; + uint32_t _column_count = 0; + + std::vector _names; + std::vector _types; + std::vector _kinds; + + std::vector> _validity; + std::vector> _f64; + std::vector> _u8; + std::vector> _str; +}; + +extern "C" +{ + // Defined by DUCKDB_CPP_EXTENSION_ENTRY(core_functions, loader) inside DuckDB's core_functions extension. + void core_functions_duckdb_cpp_init(duckdb::ExtensionLoader &loader); +} + +class CoreFunctionsStaticExtension +{ +public: + void Load(duckdb::ExtensionLoader &loader) + { + core_functions_duckdb_cpp_init(loader); + } + + std::string Name() + { + return "core_functions"; + } + + std::string Version() const + { + return ""; + } +}; + +bool try_emval_to_duckdb_value(const emscripten::val &v, Value &out, std::string &error) +{ + if (v.isNull() || v.isUndefined()) + { + out = Value(LogicalType::SQLNULL); + return true; + } + + const std::string type = v.typeOf().as(); + if (type == "string") + { + out = Value::CreateValue(v.as()); + return true; + } + if (type == "boolean") + { + out = Value::CreateValue(v.as()); + return true; + } + if (type == "number") + { + const double d = v.as(); + if (!std::isfinite(d)) + { + error = "Invalid number (non-finite)"; + return false; + } + const double i = std::floor(d); + if (i == d && i >= (double)std::numeric_limits::min() && i <= (double)std::numeric_limits::max()) + { + out = Value::CreateValue((int64_t)i); + } + else + { + out = Value::CreateValue(d); + } + return true; + } + + error = "Unsupported parameter type: " + type; + return false; +} + +namespace MaterializedQueryResultHelper +{ + emscripten::val GetValue(MaterializedQueryResult &obj, uint32_t column, uint32_t index) + { + Value val = obj.GetValue(column, index); + if (val.IsNull()) + { + return emscripten::val::null(); + } + + switch (val.type().id()) + { + case LogicalTypeId::BOOLEAN: + return emscripten::val(val.GetValue()); + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + return emscripten::val((double)val.GetValue()); + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + return emscripten::val(val.GetValue()); + case LogicalTypeId::VARCHAR: + return emscripten::val(val.GetValue()); + default: + return emscripten::val(val.ToString()); + } + } +} + +namespace ConnectionHelper +{ + PreparedStatement *prepare(Connection &obj, const string &query) + { + return obj.Prepare(query).release(); + } + MaterializedQueryResult *query(Connection &obj, const string &query) + { + return obj.Query(query).release(); + } + + ColumnarQueryResult *queryColumnar(Connection &obj, const string &query) + { + try + { + auto result = obj.Query(query); + return new ColumnarQueryResult(std::move(result)); + } + catch (const std::exception &e) + { + return new ColumnarQueryResult(std::string(e.what())); + } + } + + void interrupt(Connection &obj) + { + obj.Interrupt(); + } + + double queryProgress(Connection &obj) + { + return obj.GetQueryProgress(); + } + + void enableProfiling(Connection &obj) + { + obj.EnableProfiling(); + } + + void disableProfiling(Connection &obj) + { + obj.DisableProfiling(); + } + + std::string profilingInformation(Connection &obj) + { + return obj.GetProfilingInformation(); + } + + void beginTransaction(Connection &obj) + { + obj.BeginTransaction(); + } + + void commit(Connection &obj) + { + obj.Commit(); + } + + void rollback(Connection &obj) + { + obj.Rollback(); + } + + void setAutoCommit(Connection &obj, bool auto_commit) + { + obj.SetAutoCommit(auto_commit); + } + + bool isAutoCommit(Connection &obj) + { + return obj.IsAutoCommit(); + } + + bool hasActiveTransaction(Connection &obj) + { + return obj.HasActiveTransaction(); + } +} + +namespace DuckDBHelper +{ + DuckDB *create() + { + auto retVal = new DuckDB(nullptr); + retVal->LoadStaticExtension(); + return retVal; + } + + void close(DuckDB *db) + { + delete db; + } + + std::string libraryVersion() + { + return std::string(DuckDB::LibraryVersion()); + } + + std::string sourceID() + { + return std::string(DuckDB::SourceID()); + } + + std::string releaseCodename() + { + return std::string(DuckDB::ReleaseCodename()); + } + + std::string platform() + { + return DuckDB::Platform(); + } + + uint32_t standardVectorSize() + { + return (uint32_t)DuckDB::StandardVectorSize(); + } + + Connection *connect(DuckDB &obj) + { + return new Connection(obj); + } +} + +namespace LogicalTypeHelper +{ + uint32_t id(const LogicalType &obj) + { + return (uint32_t)obj.id(); + } + + uint32_t physicalType(const LogicalType &obj) + { + return (uint32_t)obj.InternalType(); + } + + bool isValid(const LogicalType &obj) + { + return obj.IsValid(); + } + + bool isComplete(const LogicalType &obj) + { + return obj.IsComplete(); + } + + bool isSigned(const LogicalType &obj) + { + return obj.IsSigned(); + } + + bool isUnsigned(const LogicalType &obj) + { + return obj.IsUnsigned(); + } + + void setAlias(LogicalType &obj, const std::string &alias) + { + obj.SetAlias(alias); + } + + bool hasAlias(const LogicalType &obj) + { + return obj.HasAlias(); + } + + std::string getAlias(const LogicalType &obj) + { + return obj.GetAlias(); + } +} + +namespace ValueHelper +{ + Value copy(const Value &obj) + { + return obj.Copy(); + } + + std::string toSQLString(const Value &obj) + { + return obj.ToSQLString(); + } +} + +namespace ErrorDataHelper +{ + std::string message(ErrorData &obj) + { + return obj.Message(); + } + + std::string rawMessage(ErrorData &obj) + { + return obj.RawMessage(); + } +} + +namespace BaseQueryResultHelper +{ + uint32_t resultType(const BaseQueryResult &obj) + { + return (uint32_t)obj.type; + } + + uint32_t statementType(const BaseQueryResult &obj) + { + return (uint32_t)obj.statement_type; + } + + emscripten::val columnNames(const BaseQueryResult &obj) + { + emscripten::val arr = emscripten::val::array(); + for (auto &n : obj.names) + { + arr.call("push", emscripten::val(n)); + } + return arr; + } + + emscripten::val columnTypes(const BaseQueryResult &obj) + { + emscripten::val arr = emscripten::val::array(); + for (auto &t : obj.types) + { + arr.call("push", emscripten::val(t.ToString())); + } + return arr; + } +} + +namespace PreparedStatementHelper +{ + QueryResult *executeValues(PreparedStatement &obj, vector &values) + { + return obj.Execute(values, false).release(); + } + + QueryResult *execute(PreparedStatement &obj, const emscripten::val &args) + { + const uint32_t len = args["length"].as(); + vector values; + values.reserve(len); + for (uint32_t i = 0; i < len; ++i) + { + Value v; + std::string error; + if (!try_emval_to_duckdb_value(args[i], v, error)) + { + return new MaterializedQueryResult(ErrorData(error)); + } + values.push_back(std::move(v)); + } + return executeValues(obj, values); + } + + emscripten::val names(PreparedStatement &obj) + { + emscripten::val arr = emscripten::val::array(); + for (auto &n : obj.GetNames()) + { + arr.call("push", emscripten::val(n)); + } + return arr; + } + + emscripten::val types(PreparedStatement &obj) + { + emscripten::val arr = emscripten::val::array(); + for (auto &t : obj.GetTypes()) + { + arr.call("push", emscripten::val(t.ToString())); + } + return arr; + } + + uint32_t statementType(PreparedStatement &obj) + { + return (uint32_t)obj.GetStatementType(); + } +} + +EMSCRIPTEN_BINDINGS(duckdblib_bindings) +{ + using namespace emscripten; + + enum_("ColumnarQueryResultKind") + .value("F64", ColumnarQueryResult::Kind::F64) + .value("BOOL", ColumnarQueryResult::Kind::BOOL) + .value("STRING", ColumnarQueryResult::Kind::STRING); + + class_("ColumnarQueryResult") + .constructor<>() + .function("hasError", &ColumnarQueryResult::hasError) + .function("error", &ColumnarQueryResult::error) + .function("rowCount", &ColumnarQueryResult::rowCount) + .function("columnCount", &ColumnarQueryResult::columnCount) + .function("columnName", &ColumnarQueryResult::columnName) + .function("columnType", &ColumnarQueryResult::columnType) + .function("columnKind", &ColumnarQueryResult::columnKind) + .function("validity", &ColumnarQueryResult::validity) + .function("f64Column", &ColumnarQueryResult::f64Column) + .function("boolColumn", &ColumnarQueryResult::boolColumn) + .function("stringColumn", &ColumnarQueryResult::stringColumn); + + class_("LogicalType") + .constructor<>() + .function("toString", &LogicalType::ToString) + .function("id", &LogicalTypeHelper::id) + .function("physicalType", &LogicalTypeHelper::physicalType) + .function("isIntegral", &LogicalType::IsIntegral) + .function("isFloating", &LogicalType::IsFloating) + .function("isNumeric", select_overload(&LogicalType::IsNumeric)) + .function("isTemporal", &LogicalType::IsTemporal) + .function("isValid", &LogicalTypeHelper::isValid) + .function("isComplete", &LogicalTypeHelper::isComplete) + .function("isSigned", &LogicalTypeHelper::isSigned) + .function("isUnsigned", &LogicalTypeHelper::isUnsigned) + .function("setAlias", &LogicalTypeHelper::setAlias) + .function("hasAlias", &LogicalTypeHelper::hasAlias) + .function("getAlias", &LogicalTypeHelper::getAlias) + + ; + + class_("Value") + .constructor<>() + .function("type", &Value::type) + .function("isNull", &Value::IsNull) + .function("toString", &Value::ToString) + .function("toSQLString", &ValueHelper::toSQLString) + .function("copy", &ValueHelper::copy) + .function("getBoolean", &Value::GetValue) + .function("getInt32", &Value::GetValue) + .function("getFloat", &Value::GetValue) + .function("getDouble", &Value::GetValue) + .function("getString", &Value::GetValue) + + ; + + class_("ErrorData") + .constructor<>() + .function("hasError", &ErrorData::HasError) + .function("message", &ErrorDataHelper::message) + .function("rawMessage", &ErrorDataHelper::rawMessage) + + ; + + class_("BaseQueryResult") + .function("throwError", &BaseQueryResult::ThrowError) + .function("setError", &BaseQueryResult::SetError) + .function("hasError", &BaseQueryResult::HasError) + .function("getError", &BaseQueryResult::GetError) + .function("columnCount", &BaseQueryResult::ColumnCount) + .function("resultType", &BaseQueryResultHelper::resultType) + .function("statementType", &BaseQueryResultHelper::statementType) + .function("columnNames", &BaseQueryResultHelper::columnNames) + .function("columnTypes", &BaseQueryResultHelper::columnTypes) + + ; + + class_>("QueryResult") + .function("toString_", &QueryResult::ToString) + .function("print", &QueryResult::Print) + .function("columnCount", &QueryResult::ColumnCount) + .function("columnName", &QueryResult::ColumnName) + + ; + + class_("ColumnDataCollection") + .function("count", &ColumnDataCollection::Count) + .function("columnCount", &ColumnDataCollection::ColumnCount) + + ; + + class_>("MaterializedQueryResult") + .function("rowCount", &MaterializedQueryResult::RowCount) + .function("collection", &MaterializedQueryResult::Collection) + .function("getValue", &MaterializedQueryResultHelper::GetValue) + + ; + + class_("PreparedStatement") + .function("execute", &PreparedStatementHelper::execute, return_value_policy::take_ownership()) + .function("hasError", &PreparedStatement::HasError) + .function("getError", &PreparedStatement::GetError) + .function("columnCount", &PreparedStatement::ColumnCount) + .function("statementType", &PreparedStatementHelper::statementType) + .function("names", &PreparedStatementHelper::names) + .function("types", &PreparedStatementHelper::types) + + ; + + class_("Connection") + .function("prepare", &ConnectionHelper::prepare, return_value_policy::take_ownership()) + .function("query", &ConnectionHelper::query, return_value_policy::take_ownership()) + .function("queryColumnar", &ConnectionHelper::queryColumnar, return_value_policy::take_ownership()) + .function("interrupt", &ConnectionHelper::interrupt) + .function("queryProgress", &ConnectionHelper::queryProgress) + .function("enableProfiling", &ConnectionHelper::enableProfiling) + .function("disableProfiling", &ConnectionHelper::disableProfiling) + .function("profilingInformation", &ConnectionHelper::profilingInformation) + .function("beginTransaction", &ConnectionHelper::beginTransaction) + .function("commit", &ConnectionHelper::commit) + .function("rollback", &ConnectionHelper::rollback) + .function("setAutoCommit", &ConnectionHelper::setAutoCommit) + .function("isAutoCommit", &ConnectionHelper::isAutoCommit) + .function("hasActiveTransaction", &ConnectionHelper::hasActiveTransaction) + + ; + + class_("DuckDB") + .function("connect", &DuckDBHelper::connect, return_value_policy::take_ownership()) + .function("numberOfThreads", &DuckDB::NumberOfThreads) + .function("close", &DuckDBHelper::close, allow_raw_pointers()) + + ; + + function("create", &DuckDBHelper::create, return_value_policy::take_ownership()); + function("libraryVersion", &DuckDBHelper::libraryVersion); + function("sourceID", &DuckDBHelper::sourceID); + function("releaseCodename", &DuckDBHelper::releaseCodename); + function("platform", &DuckDBHelper::platform); + function("standardVectorSize", &DuckDBHelper::standardVectorSize); +} diff --git a/packages/duckdb/src/duckdb.ts b/packages/duckdb/src/duckdb.ts index 67e2f468..3f68c829 100644 --- a/packages/duckdb/src/duckdb.ts +++ b/packages/duckdb/src/duckdb.ts @@ -1,6 +1,10 @@ -import load, { reset } from "../build/duckdb-eh.wasm.ts"; -import loadWasmWorker, { reset as resetWasmWorker } from "../build/duckdb-browser-eh.worker.ts"; -import { AsyncDuckDB, ConsoleLogger } from "@duckdb/duckdb-wasm"; +// @ts-expect-error importing from a wasm file is resolved via a custom esbuild plugin +import load, { reset } from "../../../build/packages/duckdb/src-cpp/duckdblib.wasm"; +import type { MainModule, DuckDB as DuckDBGlobals, Connection, PreparedStatement, MaterializedQueryResult, QueryResult, ColumnarQueryResult } from "../../../build/packages/duckdb/src-cpp/duckdblib.js"; +import { WasmLibrary } from "./wasm-library.ts"; + +let g_duckdb: Promise; +const textEncoder = new TextEncoder(); /** * DuckDB WASM library, a in-process SQL OLAP Database Management System.. @@ -11,13 +15,13 @@ import { AsyncDuckDB, ConsoleLogger } from "@duckdb/duckdb-wasm"; * import { DuckDB } from "@hpcc-js/wasm-duckdb"; * * let duckdb = await DuckDB.load(); - * const c = await duckdb.db.connect(); + * const c = await duckdb.connect(); * * const data = [ * { "col1": 1, "col2": "foo" }, * { "col1": 2, "col2": "bar" }, * ]; - * await duckdb.db.registerFileText("rows.json", JSON.stringify(data)); + * await duckdb.registerFileText("rows.json", JSON.stringify(data)); * await c.insertJSONFromPath('rows.json', { name: 'rows' }); * * const arrowResult = await c.query("SELECT * FROM read_json_auto('rows.json')"); @@ -30,12 +34,282 @@ import { AsyncDuckDB, ConsoleLogger } from "@duckdb/duckdb-wasm"; * c.close(); * ``` */ -export class DuckDB { - db: AsyncDuckDB; +export class DuckDBResult extends WasmLibrary { + + constructor(_module: MainModule, result: MaterializedQueryResult) { + super(_module, result); + } + + toString(): string { + return this._exports.toString_(); + } + + print(): void { + // eslint-disable-next-line no-console + console.log(this.toString()); + } + + toArray(): any[] { + const rowCount = this._exports.rowCount(); + const colCount = this._exports.columnCount(); + const columns: string[] = []; + for (let i = 0; i < colCount; ++i) { + columns.push(this._exports.columnName(BigInt(i))); + } + + const retVal: any[] = []; + for (let index = 0; index < rowCount; ++index) { + const row: any = { + }; + for (let column = 0; column < colCount; ++column) { + row[columns[column]] = this._exports.getValue(column, index); + } + retVal.push(row); + } + return retVal; + } +} + +export class DuckDBQueryResult extends WasmLibrary { + + constructor(_module: MainModule, result: QueryResult) { + super(_module, result); + } + + toString(): string { + return this._exports.toString_(); + } + + print(): void { + // eslint-disable-next-line no-console + console.log(this.toString()); + } +} + +export class DuckDBPreparedStatement extends WasmLibrary { + + constructor(_module: MainModule, stmt: PreparedStatement) { + super(_module, stmt); + } + + toString(): string { + return "[DuckDBPreparedStatement]"; + } + + execute(args: Array): DuckDBQueryResult { + return new DuckDBQueryResult(this._module, this._exports.execute(args)!); + } +} + +export type DuckDBColumnKind = "f64" | "bool" | "string"; + +export class DuckDBColumnarResult extends WasmLibrary { + + constructor(_module: MainModule, result: ColumnarQueryResult) { + super(_module, result); + } + + hasError(): boolean { + return this._exports.hasError(); + } + + error(): string { + return this._exports.error(); + } + + rowCount(): number { + return Number(this._exports.rowCount()); + } + + columnCount(): number { + return Number(this._exports.columnCount()); + } + + columnName(column: number): string { + return this._exports.columnName(column); + } + + columnType(column: number): string { + return this._exports.columnType(column); + } + + columnKind(column: number): DuckDBColumnKind { + const k = Number(this._exports.columnKind(column)); + switch (k) { + case 0: return "f64"; + case 1: return "bool"; + default: return "string"; + } + } + + validity(column: number): Uint8Array { + return this._exports.validity(column) as any; + } + + f64Column(column: number): Float64Array | null { + return (this._exports.f64Column(column) as any) ?? null; + } + + boolColumn(column: number): Uint8Array | null { + return (this._exports.boolColumn(column) as any) ?? null; + } + + stringColumn(column: number): Array | null { + return (this._exports.stringColumn(column) as any) ?? null; + } + + toArray(): any[] { + if (this.hasError()) { + throw new Error(this.error() || "DuckDB columnar query failed"); + } + + const rowCount = this.rowCount(); + const colCount = this.columnCount(); + const columns: string[] = []; + const kinds: DuckDBColumnKind[] = []; + const validity: Uint8Array[] = []; + const f64Cols: Array = []; + const boolCols: Array = []; + const strCols: Array | null> = []; + + for (let c = 0; c < colCount; ++c) { + columns.push(this.columnName(c)); + const kind = this.columnKind(c); + kinds.push(kind); + validity.push(this.validity(c)); + f64Cols.push(kind === "f64" ? this.f64Column(c) : null); + boolCols.push(kind === "bool" ? this.boolColumn(c) : null); + strCols.push(kind === "string" ? this.stringColumn(c) : null); + } + + const retVal: any[] = new Array(rowCount); + for (let r = 0; r < rowCount; ++r) { + retVal[r] = {}; + } + + for (let c = 0; c < colCount; ++c) { + const name = columns[c]; + const kind = kinds[c]; + const valid = validity[c]; + + if (kind === "f64") { + const col = f64Cols[c]; + for (let r = 0; r < rowCount; ++r) { + retVal[r][name] = valid[r] ? (col ? col[r] : null) : null; + } + } else if (kind === "bool") { + const col = boolCols[c]; + for (let r = 0; r < rowCount; ++r) { + retVal[r][name] = valid[r] ? Boolean(col && col[r]) : null; + } + } else { + const col = strCols[c]; + for (let r = 0; r < rowCount; ++r) { + retVal[r][name] = valid[r] ? (col ? col[r] : null) : null; + } + } + } + + return retVal; + } +} + +export class DuckDBConnection extends WasmLibrary { + + constructor(_module: MainModule, connection: Connection) { + super(_module, connection); + } + + prepare(sql: string): DuckDBPreparedStatement { + return new DuckDBPreparedStatement(this._module, this._exports.prepare(sql)!); + } - private constructor(db: AsyncDuckDB, protected _version: string) { - this.db = db; + query(sql: string): DuckDBResult { + return new DuckDBResult(this._module, this._exports.query(sql)!); + } + + /** + * Executes a query and materializes the result in WASM memory, exposing columns + * as typed-array views (plus a per-column validity mask). + * + * This is substantially faster than row-wise iteration (`getValue` per cell). + * The returned typed arrays are views into WASM memory; clone them if you need + * to keep data after disposing the result. + */ + queryColumnar(sql: string): DuckDBColumnarResult { + return new DuckDBColumnarResult(this._module, (this._exports as any).queryColumnar(sql)!); + } + + /** + * Convenience alias for `queryColumnar`. + */ + runQueryColumnar(sql: string): DuckDBColumnarResult { + return this.queryColumnar(sql); + } + + /** + * Execute a query and return the result as a binary payload. + * + * Note: unlike `DuckDB-Wasm`'s `runQuery` (which returns Arrow IPC bytes), this + * implementation returns UTF-8 bytes of DuckDB's textual result representation. + */ + runQuery(sql: string): Uint8Array { + const result = this._exports.query(sql); + if (!result) return new Uint8Array(); + try { + const resultAny = result as any; + const text = typeof resultAny.toString_ === "function" ? resultAny.toString_() : String(resultAny); + return textEncoder.encode(text); + } finally { + const resultAny = result as any; + if (typeof resultAny.delete === "function") { + resultAny.delete(); + } else { + const moduleAny = this._module as any; + if (typeof moduleAny.destroy === "function") { + moduleAny.destroy(result); + } + } + } + } + + /** + * Validate a SQL identifier (e.g., table name) to prevent injection. + * Only allows unquoted identifiers like: letters/underscore followed by letters/digits/underscore. + */ + private sanitizeIdentifier(name: string): string { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) { + throw new Error("Invalid SQL identifier"); + } + return name; + } + + /** + * Escape a string literal for inclusion in SQL single-quoted strings. + * Escapes single quotes by doubling them, per standard SQL rules. + */ + private escapeLiteral(value: string): string { + return value.replace(/'/g, "''"); + } + + insertJSONFromPath(fileName: string, options: { name: string }): void { + const tableName = this.sanitizeIdentifier(options.name); + const escapedFileName = this.escapeLiteral(fileName); + this.query(`CREATE TABLE ${tableName} AS SELECT * FROM read_json('${escapedFileName}')`); + } + + close() { + this.dispose(); + } +} + +export class DuckDB extends WasmLibrary { + + private constructor(_module: MainModule) { + super(_module, _module.create()!); + const { FS_createPath } = this._module; + FS_createPath("/", "home/web_user", true, true); } /** @@ -48,27 +322,18 @@ export class DuckDB { * @returns A promise to an instance of the DuckDB class. */ static load(): Promise { - const workerUrl = URL.createObjectURL( - new Blob([new Uint8Array(loadWasmWorker())], { type: "text/javascript" }) - ); - const worker = new Worker(workerUrl); - URL.revokeObjectURL(workerUrl); - const logger = new ConsoleLogger(); - const db = new AsyncDuckDB(logger, worker); - const wasmUrl = URL.createObjectURL( - new Blob([new Uint8Array(load())], { "type": "application/wasm" }) - ); - return db.instantiate(wasmUrl, null).then(async () => { - URL.revokeObjectURL(wasmUrl); - return new DuckDB(db, await db.getVersion()); - }); + if (!g_duckdb) { + g_duckdb = load().then((module: any) => { + return new DuckDB(module) + }); + } + return g_duckdb; } /** * Unloades the compiled wasm instance. */ static unload() { - resetWasmWorker(); reset(); } @@ -76,6 +341,27 @@ export class DuckDB { * @returns The DuckDB version */ version(): string { - return this._version; + return this._module.libraryVersion(); + } + + connect(): DuckDBConnection { + return new DuckDBConnection(this._module, this._exports.connect()!); + } + + registerFile(path: string, content: Uint8Array): void { + const { FS_createDataFile, FS_createPath } = this._module; + const normalizedPath = path.startsWith("/") ? path : `/${path}`; + const split = normalizedPath.lastIndexOf("/"); + const dir = split > 0 ? normalizedPath.substring(0, split) : "/"; + if (dir.length > 1 && FS_createPath) { + FS_createPath("/", dir.slice(1), true, true); + } + FS_createDataFile(normalizedPath, undefined, content, true, true, true); } + + registerFileString(fileName: string, content: string): void { + const encoded = textEncoder.encode(content); + this.registerFile(fileName, encoded); + } + } diff --git a/packages/duckdb/src/wasm-library.ts b/packages/duckdb/src/wasm-library.ts new file mode 100644 index 00000000..3bf85eb0 --- /dev/null +++ b/packages/duckdb/src/wasm-library.ts @@ -0,0 +1,82 @@ +export type PTR = number; +export interface HeapU8 { + ptr: PTR; + size: number; +} + +interface WasmLibraryModule { + _malloc(size: number): PTR; + _free(ptr: PTR): void; + destroy?: (instance: U) => void; + HEAPU8: Uint8Array; +} + +/** + * Base class to simplify moving data into and out of Wasm memory. + */ +export class WasmLibrary { + + protected _module: T & WasmLibraryModule; + protected _exports: U; + + protected constructor(_module: T, _exports: U) { + this._module = _module as any; + this._exports = _exports as any; + } + + dispose() { + const exportsAny = this._exports as any; + if (exportsAny && typeof exportsAny.delete === "function") { + exportsAny.delete(); + return; + } + const moduleAny = this._module as any; + if (moduleAny && typeof moduleAny.destroy === "function") { + moduleAny.destroy(this._exports); + } + } + + protected malloc_heapu8(size: number): HeapU8 { + const ptr: PTR = this._module._malloc(size) as PTR; + return { + ptr, + size + }; + } + + protected free_heapu8(data: HeapU8) { + this._module._free(data.ptr); + } + + protected uint8_heapu8(data: Uint8Array): HeapU8 { + const retVal = this.malloc_heapu8(data.byteLength); + this._module.HEAPU8.set(data, retVal.ptr); + return retVal; + } + + protected heapu8_view(data: HeapU8): Uint8Array { + return this._module.HEAPU8.subarray(data.ptr, data.ptr + data.size); + } + + protected heapu8_uint8(data: HeapU8): Uint8Array { + return new Uint8Array([...this.heapu8_view(data)]); + } + + protected string_heapu8(str: string): HeapU8 { + const data = Uint8Array.from(str, x => x.charCodeAt(0)); + return this.uint8_heapu8(data); + } + + protected string_uint8array(str: string): Uint8Array { + return Uint8Array.from(str, x => x.charCodeAt(0)); + } + + protected heapu8_string(data: HeapU8): string { + const retVal = Array.from({ length: data.size }); + const submodule = this._module.HEAPU8.subarray(data.ptr, data.ptr + data.size); + submodule.forEach((c: number, i: number) => { + retVal[i] = String.fromCharCode(c); + }); + return retVal.join(""); + } +} diff --git a/packages/duckdb/tests/duckdb.browser.spec.ts b/packages/duckdb/tests/duckdb.browser.spec.ts deleted file mode 100644 index 38153e99..00000000 --- a/packages/duckdb/tests/duckdb.browser.spec.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { DuckDB } from "@hpcc-js/wasm-duckdb"; - -describe("duckdb", function () { - - it("version", async function () { - const duckdb = await DuckDB.load(); - const v = duckdb.version(); - expect(v).to.be.a.string; - expect(v).to.equal("v1.4.0"); // Update README.md if this changes - console.log("duckdb version: " + v); - }); - - it("simple", async function () { - const duckdb = await DuckDB.load(); - const db = duckdb.db; - const conn = await db.connect(); - const stmt = await conn.prepare("SELECT v + ? FROM generate_series(0, 10000) AS t(v);"); - await stmt.query(234); - for await (const batch of await stmt.send(234)) { - expect(batch).to.exist; - } - await stmt.close(); - await conn.close(); - }); - - it("json", async function () { - const duckdb = await DuckDB.load(); - const c = await duckdb.db.connect(); - - const data = [ - { "col1": 1, "col2": "foo" }, - { "col1": 2, "col2": "bar" }, - ]; - await duckdb.db.registerFileText("rows.json", JSON.stringify(data)); - await c.insertJSONFromPath("rows.json", { name: "rows" }); - - const arrowResult = await c.query("SELECT * FROM read_json_auto('rows.json')"); - const result = arrowResult.toArray().map((row: any) => row.toJSON()); - expect(result.length).to.equal(data.length); - for (let i = 0; i < result.length; i++) { - expect(result[i].col2).to.equal(data[i].col2); - } - - c.close(); - }); -}); diff --git a/packages/duckdb/tests/duckdb.spec.ts b/packages/duckdb/tests/duckdb.spec.ts new file mode 100644 index 00000000..bf21276e --- /dev/null +++ b/packages/duckdb/tests/duckdb.spec.ts @@ -0,0 +1,158 @@ +import { describe, expect, it } from "vitest"; +import { DuckDB } from "@hpcc-js/wasm-duckdb"; + +describe("duckdb", function () { + + it("version", async function () { + const duckdb = await DuckDB.load(); + const v = duckdb.version(); + expect(v).to.be.a("string"); + expect(v).to.equal("v1.4.3"); // Update README.md if this changes + console.log("duckdb version: " + v); + }); + + it("simple", async function () { + const duckdb = await DuckDB.load(); + const con = duckdb.connect(); + con.query("CREATE TABLE integers(i INTEGER)").dispose(); + con.query("INSERT INTO integers VALUES (3)").dispose(); + const result = con.query("SELECT * FROM integers"); + const resultStr = result.toString(); + result.print(); + expect(resultStr).to.be.a("string"); + expect(resultStr).to.contain("3"); + result.dispose(); + con.close(); + }); + + it("runQuery", async function () { + const duckdb = await DuckDB.load(); + const con = duckdb.connect(); + const bytes = con.runQuery("SELECT 42 AS answer"); + expect(bytes).to.be.instanceOf(Uint8Array); + expect(bytes.length).to.be.gt(0); + const text = new TextDecoder().decode(bytes); + expect(text).to.contain("42"); + con.close(); + }); + + it("queryColumnar", async function () { + const duckdb = await DuckDB.load(); + const con = duckdb.connect(); + const result = con.queryColumnar("SELECT 42 AS answer"); + expect(result.hasError()).to.equal(false); + expect(result.rowCount()).to.equal(1); + expect(result.columnCount()).to.equal(1); + expect(result.columnName(0).toLowerCase()).to.contain("answer"); + expect(result.validity(0)).to.be.instanceOf(Uint8Array); + const col = result.f64Column(0); + expect(col).to.not.equal(null); + if (!(col instanceof Float64Array)) { + throw new Error("Expected f64Column(0) to return a Float64Array"); + } + expect(col[0]).to.equal(42); + result.dispose(); + con.close(); + }); + + it("queryColumnar_toArray", async function () { + const duckdb = await DuckDB.load(); + const con = duckdb.connect(); + + const result = con.queryColumnar( + "SELECT * FROM (VALUES (1, true, 'x'), (2, false, NULL)) t(a, b, c)" + ); + expect(result.hasError()).to.equal(false); + expect(result.toArray()).to.deep.equal([ + { a: 1, b: true, c: "x" }, + { a: 2, b: false, c: null } + ]); + + result.dispose(); + con.close(); + }); + + it("prepare", async function () { + const duckdb = await DuckDB.load(); + const conn = duckdb.connect(); + conn.query("CREATE TABLE person (name VARCHAR, age BIGINT);").dispose(); + conn.query("INSERT INTO person VALUES ('Alice', 37), ('Ana', 35), ('Bob', 41), ('Bea', 25);").dispose(); + const stmt = conn.prepare("SELECT * FROM person WHERE starts_with(name, CAST(? AS VARCHAR))"); + const result = stmt.execute(["B"]); + const resultStr = result.toString(); + expect(resultStr).to.contain("name"); + expect(resultStr).to.contain("age"); + expect(resultStr.toLowerCase()).to.not.contain("error"); + result.dispose(); + stmt.dispose(); + conn.close(); + }); + + it("prepare_values", async function () { + const duckdb = await DuckDB.load(); + const conn = duckdb.connect(); + const stmt = conn.prepare( + "SELECT CAST(? AS VARCHAR) AS s, CAST(? AS BIGINT) AS n, CAST(? AS BOOLEAN) AS b, CAST(? AS VARCHAR) AS z" + ); + const result = stmt.execute(["x", 42, true, null]); + const resultStr = result.toString(); + expect(resultStr.toLowerCase()).to.not.contain("error"); + expect(resultStr).to.contain("s"); + expect(resultStr).to.contain("n"); + expect(resultStr).to.contain("b"); + expect(resultStr).to.contain("z"); + expect(resultStr).to.contain("42"); + result.dispose(); + stmt.dispose(); + conn.close(); + }); + + it("json", async function () { + const duckdb = await DuckDB.load(); + const c = duckdb.connect(); + + const data = [ + { "col1": 1, "col2": "foo" }, + { "col1": 2, "col2": "bar" }, + ]; + duckdb.registerFileString("/rows.json", JSON.stringify(data)); + c.insertJSONFromPath("/rows.json", { name: "rows" }); + + const resultObj = c.query("SELECT * FROM rows"); + const resultArr = resultObj.toArray(); + resultObj.dispose(); + expect(resultArr.length).to.equal(data.length); + for (let i = 0; i < resultArr.length; i++) { + expect(resultArr[i].col2).to.equal(data[i].col2); + } + + c.close(); + }); + + it("core_functions", async function () { + const duckdb = await DuckDB.load(); + const c = duckdb.connect(); + + // generate_series is provided by core_functions; should work without calling SQL LOAD. + const result = c.query("SELECT * FROM generate_series(0, 3) AS t(v)"); + const rows = result.toArray(); + result.dispose(); + expect(rows.map((r: any) => r.v)).to.deep.equal([0, 1, 2, 3]); + + c.close(); + }); + + it("extensions", async function () { + const duckdb = await DuckDB.load(); + const con = duckdb.connect(); + const result = con.query(`SELECT extension_name, loaded, installed FROM duckdb_extensions() WHERE extension_name IN ('json', 'parquet')`); + const extensions = result.toArray(); + result.dispose(); + + expect(extensions.length).to.be.gt(0); + const loadedExtensions = extensions.filter((ext: any) => ext.loaded); + expect(loadedExtensions.length).to.be.gt(0); + + con.close(); + }); +}); diff --git a/packages/duckdb/tsconfig.json b/packages/duckdb/tsconfig.json index f27852bb..20a93433 100644 --- a/packages/duckdb/tsconfig.json +++ b/packages/duckdb/tsconfig.json @@ -1,8 +1,8 @@ { "extends": "../tsconfig.json", "compilerOptions": { - "rootDir": ".", - "declarationDir": "./types", + "rootDir": "./src", + "declarationDir": "./types" }, "include": [ "./src/**/*" diff --git a/packages/tsconfig.json b/packages/tsconfig.json index 7484236f..09ca1332 100644 --- a/packages/tsconfig.json +++ b/packages/tsconfig.json @@ -10,7 +10,8 @@ "allowImportingTsExtensions": true, "lib": [ "DOM", - "ES2020" + "ES2020", + "ES2020.BigInt" ], "types": [] } diff --git a/scripts/cpp-build.sh b/scripts/cpp-build.sh index 49acdf84..22893fb4 100755 --- a/scripts/cpp-build.sh +++ b/scripts/cpp-build.sh @@ -5,6 +5,6 @@ if [ ! -d "./build" ] then cmake -S . -B ./build --preset vcpkg-emscripten-MinSizeRel fi -# cmake -S . -B ./build --preset vcpkg-emscripten-Debug +# cmake -S . -B ./build --preset vcpkg-emscripten-RelWithDebInfo cmake --build ./build --parallel diff --git a/scripts/cpp-install-emsdk.sh b/scripts/cpp-install-emsdk.sh index e9ab54b5..080bbc7c 100755 --- a/scripts/cpp-install-emsdk.sh +++ b/scripts/cpp-install-emsdk.sh @@ -14,3 +14,8 @@ git pull ./emsdk install $VERSION-upstream ./emsdk activate $VERSION-upstream cd .. + +( + cd ./emsdk/upstream/emscripten + npm install --include=dev +) diff --git a/tests/bundlers/src/browser-test-utils.js b/tests/bundlers/src/browser-test-utils.js index 4c2aea91..baf01a4f 100644 --- a/tests/bundlers/src/browser-test-utils.js +++ b/tests/bundlers/src/browser-test-utils.js @@ -122,37 +122,19 @@ export async function testExpat() { export async function testDuckDB() { try { - // DuckDB should work in browser environments with Worker support const { DuckDB } = await import('@hpcc-js/wasm-duckdb'); // Load the WASM module const duckdb = await DuckDB.load(); // Test basic SQL query - const db = duckdb.db; - const conn = await db.connect(); - - // Simple query test - const result = await conn.query("SELECT 42 as answer"); - - // DuckDB in browser returns events, let's check if query executed successfully - let querySuccessful = false; - const batches = []; - - for await (const batch of result) { - console.log(JSON.stringify(batch, null, 2)); - - if (batch && (batch.toColumns || batch.value === "SELECT 42 as answer" || typeof batch === 'object')) { - querySuccessful = true; - batches.push(batch); - } - } - - if (!querySuccessful && batches.length === 0) { - throw new Error('DuckDB query did not execute successfully'); + const conn = duckdb.connect(); + const result = conn.query("SELECT 42 as answer"); + const resultStr = result.toString(); + if (!resultStr.includes("42")) { + throw new Error('DuckDB query did not return expected result'); } - - await conn.close(); + conn.close(); console.log('✓ DuckDB test passed'); DuckDB.unload(); diff --git a/vcpkg-overlays/duckdb/extensions.patch b/vcpkg-overlays/duckdb/extensions.patch new file mode 100644 index 00000000..d5fa227d --- /dev/null +++ b/vcpkg-overlays/duckdb/extensions.patch @@ -0,0 +1,51 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -1289,7 +1289,19 @@ + include("${EXTENSION_CONFIG_BASE_DIR}/${EXT}.cmake") + else() + # in-tree or non-existent extension: load it +- duckdb_extension_load(${EXT}) ++ if("${EXT}" STREQUAL "httpfs") ++ duckdb_extension_load(${EXT} ++ SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/extension/httpfs ++ INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/extension/httpfs/extension/httpfs/include ++ ) ++ elseif("${EXT}" STREQUAL "excel") ++ duckdb_extension_load(${EXT} ++ SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/extension/excel ++ INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/extension/excel/src/excel/include ++ ) ++ else() ++ duckdb_extension_load(${EXT}) ++ endif() + endif() + endif() + endforeach() + +diff --git a/DuckDBConfig.cmake.in b/DuckDBConfig.cmake.in +--- a/DuckDBConfig.cmake.in ++++ b/DuckDBConfig.cmake.in +@@ -6,10 +6,21 @@ + + include(CMakeFindDependencyMacro) + find_dependency(Threads) +-if(NOT @WITH_INTERNAL_ICU@) ++set(DUCKDB_EXTENSION_LIST "@BUILD_EXTENSIONS@") ++if(DUCKDB_EXTENSION_LIST) ++ list(FIND DUCKDB_EXTENSION_LIST "icu" DUCKDB_ICU_INDEX) ++ list(FIND DUCKDB_EXTENSION_LIST "excel" DUCKDB_EXCEL_INDEX) ++endif() ++ ++if(DEFINED DUCKDB_ICU_INDEX AND DUCKDB_ICU_INDEX GREATER_EQUAL 0) + find_dependency(ICU COMPONENTS i18n uc data) + endif() + ++if(DEFINED DUCKDB_EXCEL_INDEX AND DUCKDB_EXCEL_INDEX GREATER_EQUAL 0) ++ find_dependency(expat CONFIG) ++ find_dependency(minizip-ng CONFIG) ++endif() ++ + # Compute paths + get_filename_component(DuckDB_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) + set(DuckDB_INCLUDE_DIRS "@CONF_INCLUDE_DIRS@") diff --git a/vcpkg-overlays/duckdb/fix-platform-binary-esm.patch b/vcpkg-overlays/duckdb/fix-platform-binary-esm.patch new file mode 100644 index 00000000..999a010d --- /dev/null +++ b/vcpkg-overlays/duckdb/fix-platform-binary-esm.patch @@ -0,0 +1,21 @@ +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -1507,7 +1507,17 @@ if(NOT DUCKDB_EXPLICIT_PLATFORM) + set_target_properties(duckdb_platform_binary PROPERTIES RUNTIME_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}) + add_custom_command( + OUTPUT ${PROJECT_BINARY_DIR}/duckdb_platform_out + DEPENDS duckdb_platform_binary +- COMMAND duckdb_platform_binary > ${PROJECT_BINARY_DIR}/duckdb_platform_out || ( echo "Provide explicit DUCKDB_PLATFORM=your_target_arch to avoid build-type detection of the platform" && exit 1 ) ++ COMMAND ${CMAKE_COMMAND} -E rename ++ ${PROJECT_BINARY_DIR}/duckdb_platform_binary.js ++ ${PROJECT_BINARY_DIR}/duckdb_platform_binary.cjs || echo "No .js file to rename (not Emscripten build)" ++ COMMAND ${CMAKE_COMMAND} -E env ++ ${CMAKE_COMMAND} -E echo "Running platform detection..." ++ COMMAND $, ++ node ${PROJECT_BINARY_DIR}/duckdb_platform_binary.cjs, ++ duckdb_platform_binary ++ > > ${PROJECT_BINARY_DIR}/duckdb_platform_out || ( echo "Provide explicit DUCKDB_PLATFORM=your_target_arch to avoid build-type detection of the platform" && exit 1 ) + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + ) + else() diff --git a/vcpkg-overlays/duckdb/portfile.cmake b/vcpkg-overlays/duckdb/portfile.cmake new file mode 100644 index 00000000..909b3bf2 --- /dev/null +++ b/vcpkg-overlays/duckdb/portfile.cmake @@ -0,0 +1,99 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO duckdb/duckdb + REF v${VERSION} + SHA512 058218e4551867dc231cae682e835fb76b2d02b655f889753fde6745b9895b81a7161c7eb3104c9f9e8a7a33fed460fc0028d0b94a1e36834100aa597b97a877 + HEAD_REF main + PATCHES + extensions.patch +) + +# Remove vendored dependencies which are not properly namespaced +file(REMOVE_RECURSE + "${SOURCE_PATH}/third_party/catch" + "${SOURCE_PATH}/third_party/imdb" + "${SOURCE_PATH}/third_party/snowball" + "${SOURCE_PATH}/third_party/tpce-tool" +) + +if("excel" IN_LIST FEATURES) + vcpkg_from_github( + OUT_SOURCE_PATH DUCKDB_EXCEL_SOURCE_PATH + REPO duckdb/duckdb-excel + REF 6c7a0270608d18053d23359834b775d40804a052 + SHA512 442b4dc9405f34a9b624e5c4e874ebf2cffd1f5c477257b090613f987d83fcc02bc2293b8d163fffe018aa250e90bcadc9ac345e84dc4c96f4092c19c769f924 + HEAD_REF main + ) + file(RENAME "${DUCKDB_EXCEL_SOURCE_PATH}" "${SOURCE_PATH}/extension/excel") +endif() + +if("httpfs" IN_LIST FEATURES) + vcpkg_from_github( + OUT_SOURCE_PATH DUCKDB_HTTPFS_SOURCE_PATH + REPO duckdb/duckdb_httpfs + REF a4a014d4fc232c3087ee44a804959b5d67a0f8c5 + SHA512 7e774a0714b863ecd49ad6ff07b8ecf780614f8e81d097dc01def37b48efb140efba003a5caa2deec9c83c636906fbcb44f5d74813da31f162d9d8b06016afe8 + HEAD_REF main + ) + file(RENAME "${DUCKDB_HTTPFS_SOURCE_PATH}" "${SOURCE_PATH}/extension/httpfs") +endif() + +string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "static" DUCKDB_BUILD_STATIC) +string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" DUCKDB_BUILD_DYNAMIC) + +set(EXTENSION_LIST "autocomplete;core-functions;excel;httpfs;icu;json;tpcds;tpch") +set(BUILD_EXTENSIONS "") +foreach(EXT ${EXTENSION_LIST}) + if(${EXT} IN_LIST FEATURES) + # Special case: convert core-functions to core_functions + if(${EXT} STREQUAL "core-functions") + list(APPEND BUILD_EXTENSIONS "core_functions") + else() + list(APPEND BUILD_EXTENSIONS ${EXT}) + endif() + endif() +endforeach() +if(NOT "${BUILD_EXTENSIONS}" STREQUAL "") + set(BUILD_EXTENSIONS_FLAG "-DBUILD_EXTENSIONS='${BUILD_EXTENSIONS}'") +endif() + +vcpkg_cmake_configure( + SOURCE_PATH ${SOURCE_PATH} + OPTIONS + -DOVERRIDE_GIT_DESCRIBE=v${VERSION} + -DDUCKDB_EXPLICIT_VERSION=v${VERSION} + -DBUILD_UNITTESTS=OFF + -DBUILD_SHELL=FALSE + "${BUILD_EXTENSIONS_FLAG}" + -DENABLE_EXTENSION_AUTOLOADING=1 + -DENABLE_EXTENSION_AUTOINSTALL=1 + -DENABLE_SANITIZER=OFF + -DENABLE_THREAD_SANITIZER=OFF + -DENABLE_UBSAN=OFF + -DDUCKDB_EXPLICIT_PLATFORM=wasm_mvp + -DDISABLE_THREADS=ON +) + +vcpkg_cmake_install() + +if(EXISTS "${CURRENT_PACKAGES_DIR}/CMake") + vcpkg_cmake_config_fixup(CONFIG_PATH CMake) +elseif(EXISTS "${CURRENT_PACKAGES_DIR}/lib/cmake/DuckDB") + vcpkg_cmake_config_fixup(CONFIG_PATH "lib/cmake/DuckDB") +elseif(EXISTS "${CURRENT_PACKAGES_DIR}/lib/cmake/${PORT}") + vcpkg_cmake_config_fixup(CONFIG_PATH "lib/cmake/${PORT}") +endif() + +if(VCPKG_LIBRARY_LINKAGE STREQUAL static) + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin" "${CURRENT_PACKAGES_DIR}/debug/bin") +endif() + +file(REMOVE_RECURSE + "${CURRENT_PACKAGES_DIR}/include/duckdb/main/capi/header_generation" +) + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/include/duckdb/storage/serialization") +file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/vcpkg-overlays/duckdb/usage b/vcpkg-overlays/duckdb/usage new file mode 100644 index 00000000..97870d4d --- /dev/null +++ b/vcpkg-overlays/duckdb/usage @@ -0,0 +1,4 @@ +The package DuckDB provides CMake targets: + + find_package(DuckDB CONFIG REQUIRED) + target_link_libraries(main PRIVATE $,duckdb,duckdb_static>) diff --git a/vcpkg-overlays/duckdb/vcpkg.json b/vcpkg-overlays/duckdb/vcpkg.json new file mode 100644 index 00000000..9957452d --- /dev/null +++ b/vcpkg-overlays/duckdb/vcpkg.json @@ -0,0 +1,57 @@ +{ + "name": "duckdb", + "version": "1.4.3", + "description": "High-performance in-process analytical database system", + "homepage": "https://duckdb.org", + "license": "MIT", + "supports": "!(uwp | android | (windows & arm64))", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ], + "features": { + "autocomplete": { + "description": "Statically link the autocomplete extension into DuckDB" + }, + "excel": { + "description": "Statically link the excel extension into DuckDB", + "dependencies": [ + "expat", + "minizip-ng" + ] + }, + "httpfs": { + "description": "Statically link the httpfs extension into DuckDB", + "dependencies": [ + "openssl" + ] + }, + "icu": { + "description": "Statically link the icu extension into DuckDB", + "dependencies": [ + { + "name": "icu", + "default-features": false + } + ] + }, + "json": { + "description": "Statically link the json extension into DuckDB" + }, + "core-functions": { + "description": "Statically link the core_functions extension into DuckDB" + }, + "tpcds": { + "description": "Statically link the tpcds extension into DuckDB" + }, + "tpch": { + "description": "Statically link the tpch extension into DuckDB" + } + } +} \ No newline at end of file diff --git a/vcpkg.json b/vcpkg.json index 5dd83a56..96153820 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -3,6 +3,14 @@ { "name": "base91" }, + { + "name": "duckdb", + "default-features": false, + "features": [ + "json", + "core-functions" + ] + }, { "name": "expat" },