Skip to content

Commit 55aec40

Browse files
committed
wasm: add WebAssembly (WASM) support
Add support for running bbolt in WebAssembly environments (js/wasm and wasip1): - Implement memory management for WASM platforms without mmap - Add platform-specific transaction initialization - Configure smaller page size and memory limits for WASM - Add GitHub workflow for WASM testing (js and wasip1) - Adjust tests to handle WASM memory constraints - Support both browser (js) and WASI environments Signed-off-by: Travis Cline <[email protected]>
1 parent 3e91289 commit 55aec40

File tree

13 files changed

+302
-5
lines changed

13 files changed

+302
-5
lines changed

.github/workflows/tests_wasm.yaml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
---
2+
name: Tests WASM
3+
permissions: read-all
4+
on: [push, pull_request]
5+
jobs:
6+
test-wasm-js:
7+
runs-on: ubuntu-latest
8+
steps:
9+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
10+
- id: goversion
11+
run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT"
12+
- uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
13+
with:
14+
go-version: ${{ steps.goversion.outputs.goversion }}
15+
- name: Setup Node.js
16+
uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
17+
with:
18+
node-version: '20'
19+
- name: Run WASM JS tests
20+
run: |
21+
env -i \
22+
PATH="$PATH" \
23+
HOME="$HOME" \
24+
GOCACHE="$GOCACHE" \
25+
GOPATH="$GOPATH" \
26+
GOROOT="$(go env GOROOT)" \
27+
make test-wasm
28+
29+
test-wasm-wasip1:
30+
runs-on: ubuntu-latest
31+
steps:
32+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
33+
- id: goversion
34+
run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT"
35+
- uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
36+
with:
37+
go-version: ${{ steps.goversion.outputs.goversion }}
38+
- name: Install wazero
39+
run: go install github.com/tetratelabs/wazero/cmd/wazero@latest
40+
- name: Run WASM wasip1 tests
41+
run: |
42+
make test-wasip1

Makefile

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,32 @@ test:
5151
BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./internal/...
5252
BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./cmd/bbolt
5353

54+
.PHONY: test-wasm
55+
test-wasm:
56+
@echo "WASM js test"
57+
export PATH="$$PATH:$$(go env GOROOT)/lib/wasm" && \
58+
GOOS=js GOARCH=wasm go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT}
59+
60+
.PHONY: test-wasip1
61+
test-wasip1:
62+
@echo "WASM wasip1 test"
63+
if command -v wazero >/dev/null 2>&1; then \
64+
echo "Using wazero runtime"; \
65+
GOOS=wasip1 GOARCH=wasm go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} \
66+
-exec="wazero run -mount=$(shell mktemp -d):/tmp -mount=.:/test"; \
67+
elif command -v wasmtime >/dev/null 2>&1; then \
68+
WASI_TEMP=$$(mktemp -d); \
69+
echo "Using wasmtime runtime with temp dir: $$WASI_TEMP"; \
70+
GOOS=wasip1 GOARCH=wasm go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} \
71+
-exec="wasmtime --dir=. --dir=$$WASI_TEMP --env=TMPDIR=$$WASI_TEMP"; \
72+
else \
73+
echo "No WASI runtime found - install wazero (recommended) or wasmtime"; \
74+
echo " go install github.com/tetratelabs/wazero/cmd/wazero@latest"; \
75+
echo " brew install wasmtime"; \
76+
echo "Building wasip1 binary to verify compilation..."; \
77+
GOOS=wasip1 GOARCH=wasm go build .; \
78+
fi
79+
5480
.PHONY: coverage
5581
coverage:
5682
@echo "hashmap freelist test"

bolt_unix.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
//go:build !windows && !plan9 && !solaris && !aix && !android
1+
//go:build !windows && !plan9 && !solaris && !aix && !android && !js && !wasip1
22

33
package bbolt
44

bolt_wasm.go

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
//go:build js || wasip1
2+
3+
package bbolt
4+
5+
import (
6+
"fmt"
7+
"io"
8+
"time"
9+
"unsafe"
10+
11+
berrors "go.etcd.io/bbolt/errors"
12+
"go.etcd.io/bbolt/internal/common"
13+
)
14+
15+
// mmap memory maps a DB's data file.
16+
func mmap(db *DB, sz int) error {
17+
// Check MaxSize constraint for WASM platforms
18+
if !db.readOnly && db.MaxSize > 0 && sz > db.MaxSize {
19+
// The max size only limits future writes; however, we don't block opening
20+
// and mapping the database if it already exceeds the limit.
21+
fileSize, err := db.fileSize()
22+
if err != nil {
23+
return fmt.Errorf("could not check existing db file size: %s", err)
24+
}
25+
26+
if sz > fileSize {
27+
return berrors.ErrMaxSizeReached
28+
}
29+
}
30+
31+
// Truncate and fsync to ensure file size metadata is flushed.
32+
// https://github.com/boltdb/bolt/issues/284
33+
if !db.NoGrowSync && !db.readOnly {
34+
if err := db.file.Truncate(int64(sz)); err != nil {
35+
return fmt.Errorf("file resize error: %s", err)
36+
}
37+
if err := db.file.Sync(); err != nil {
38+
return fmt.Errorf("file sync error: %s", err)
39+
}
40+
}
41+
42+
// Map the data file to memory.
43+
b := make([]byte, sz)
44+
if sz > 0 {
45+
// Read the data file.
46+
if _, err := db.file.ReadAt(b, 0); err != nil && err != io.EOF {
47+
return err
48+
}
49+
}
50+
51+
// Save the original byte slice and convert to a byte array pointer.
52+
db.dataref = b
53+
db.datasz = sz
54+
if sz > 0 {
55+
db.data = (*[common.MaxMapSize]byte)(unsafe.Pointer(&b[0]))
56+
}
57+
58+
return nil
59+
}
60+
61+
// munmap unmaps a DB's data file from memory.
62+
func munmap(db *DB) error {
63+
// In WASM, we just clear the references
64+
db.dataref = nil
65+
db.data = nil
66+
db.datasz = 0
67+
return nil
68+
}
69+
70+
// madvise is not supported in WASM.
71+
func madvise(b []byte, advice int) error {
72+
// Not implemented - no memory advice in WASM
73+
return nil
74+
}
75+
76+
// mlock is not supported in WASM.
77+
func mlock(db *DB, fileSize int) error {
78+
// Not implemented - no memory locking in WASM
79+
return nil
80+
}
81+
82+
// munlock is not supported in WASM.
83+
func munlock(db *DB, fileSize int) error {
84+
// Not implemented - no memory unlocking in WASM
85+
return nil
86+
}
87+
88+
// flock acquires an advisory lock on a file descriptor.
89+
func flock(db *DB, exclusive bool, timeout time.Duration) error {
90+
// Not implemented - no file locking in WASM
91+
return nil
92+
}
93+
94+
// funlock releases an advisory lock on a file descriptor.
95+
func funlock(db *DB) error {
96+
// Not implemented - no file unlocking in WASM
97+
return nil
98+
}
99+
100+
// fdatasync flushes written data to a file descriptor.
101+
func fdatasync(db *DB) error {
102+
if db.file == nil {
103+
return nil
104+
}
105+
return db.file.Sync()
106+
}
107+
108+
// txInit refreshes the memory buffer from the file for WASM platforms.
109+
// This is needed because WASM doesn't have real mmap, so we need to manually
110+
// sync the memory buffer with the file to see changes from previous transactions.
111+
func (db *DB) txInit() error {
112+
// For read-only databases or initial state, skip refresh
113+
if db.file == nil {
114+
return nil
115+
}
116+
117+
// Check if the file has grown
118+
fileInfo, err := db.file.Stat()
119+
if err != nil {
120+
return err
121+
}
122+
fileSize := int(fileInfo.Size())
123+
124+
// If file has grown or we need to initialize, refresh memory
125+
if fileSize > db.datasz || db.datasz == 0 {
126+
// Re-mmap with the new size
127+
if err := mmap(db, fileSize); err != nil {
128+
return err
129+
}
130+
} else if db.datasz > 0 {
131+
// Refresh the existing buffer
132+
b := make([]byte, db.datasz)
133+
if _, err := db.file.ReadAt(b, 0); err != nil && err != io.EOF {
134+
return err
135+
}
136+
db.dataref = b
137+
db.data = (*[common.MaxMapSize]byte)(unsafe.Pointer(&b[0]))
138+
}
139+
140+
// Update meta page pointers
141+
if db.pageSize > 0 && db.datasz >= db.pageSize*2 {
142+
db.meta0 = db.page(0).Meta()
143+
db.meta1 = db.page(1).Meta()
144+
}
145+
146+
return nil
147+
}

boltsync_unix.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
//go:build !windows && !plan9 && !linux && !openbsd
1+
//go:build !windows && !plan9 && !linux && !openbsd && !js && !wasip1
22

33
package bbolt
44

bucket_test.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"log"
99
"math/rand"
1010
"os"
11+
"runtime"
1112
"strconv"
1213
"strings"
1314
"testing"
@@ -208,6 +209,9 @@ func TestDB_Put_VeryLarge(t *testing.T) {
208209
if testing.Short() {
209210
t.Skip("skipping test in short mode.")
210211
}
212+
if runtime.GOOS == "js" || runtime.GOOS == "wasip1" {
213+
t.Skip("skipping test on WASM due to memory constraints")
214+
}
211215

212216
n, batchN := 400000, 200000
213217
ksize, vsize := 8, 500
@@ -377,6 +381,9 @@ func TestBucket_Delete_FreelistOverflow(t *testing.T) {
377381
if testing.Short() {
378382
t.Skip("skipping test in short mode.")
379383
}
384+
if runtime.GOOS == "js" || runtime.GOOS == "wasip1" {
385+
t.Skip("skipping test on WASM due to memory constraints")
386+
}
380387

381388
db := btesting.MustCreateDB(t)
382389

@@ -1207,6 +1214,9 @@ func TestBucket_Put_KeyTooLarge(t *testing.T) {
12071214

12081215
// Ensure that an error is returned when inserting a value that's too large.
12091216
func TestBucket_Put_ValueTooLarge(t *testing.T) {
1217+
if runtime.GOARCH == "wasm" {
1218+
t.Skip("skipping test on wasm")
1219+
}
12101220
// Skip this test on DroneCI because the machine is resource constrained.
12111221
if os.Getenv("DRONE") == "true" {
12121222
t.Skip("not enough RAM for test")

db.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -772,12 +772,26 @@ func (db *DB) Logger() Logger {
772772
return db.logger
773773
}
774774

775+
// txIniter is an interface that allows for platform-specific transaction
776+
// initialization.
777+
type txIniter interface {
778+
txInit() error
779+
}
780+
775781
func (db *DB) beginTx() (*Tx, error) {
776782
// Lock the meta pages while we initialize the transaction. We obtain
777783
// the meta lock before the mmap lock because that's the order that the
778784
// write transaction will obtain them.
779785
db.metalock.Lock()
780786

787+
// Allow platform-specific transaction initialization
788+
if initer, ok := any(db).(txIniter); ok {
789+
if err := initer.txInit(); err != nil {
790+
db.metalock.Unlock()
791+
return nil, err
792+
}
793+
}
794+
781795
// Obtain a read-only lock on the mmap. When the mmap is remapped it will
782796
// obtain a write lock so all transactions must finish before it can be
783797
// remapped.
@@ -835,6 +849,14 @@ func (db *DB) beginRWTx() (*Tx, error) {
835849
db.metalock.Lock()
836850
defer db.metalock.Unlock()
837851

852+
// Allow platform-specific transaction initialization
853+
if initer, ok := any(db).(txIniter); ok {
854+
if err := initer.txInit(); err != nil {
855+
db.rwlock.Unlock()
856+
return nil, err
857+
}
858+
}
859+
838860
// Exit if the database is not open yet.
839861
if !db.opened {
840862
db.rwlock.Unlock()

db_test.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
bolt "go.etcd.io/bbolt"
2525
berrors "go.etcd.io/bbolt/errors"
2626
"go.etcd.io/bbolt/internal/btesting"
27+
"go.etcd.io/bbolt/internal/common"
2728
)
2829

2930
// pageSize is the size of one page in the data file.
@@ -240,12 +241,15 @@ func TestOpen_ReadPageSize_FromMeta1_OS(t *testing.T) {
240241

241242
// Reopen data file.
242243
db = btesting.MustOpenDBWithOption(t, path, nil)
243-
require.Equalf(t, os.Getpagesize(), db.Info().PageSize, "check page size failed")
244+
require.Equalf(t, common.GetPagesize(), db.Info().PageSize, "check page size failed")
244245
}
245246

246247
// Ensure that it can read the page size from the second meta page if the first one is invalid.
247248
// The page size is expected to be the given page size in this case.
248249
func TestOpen_ReadPageSize_FromMeta1_Given(t *testing.T) {
250+
if runtime.GOOS == "js" || runtime.GOOS == "wasip1" {
251+
t.Skip("skipping test on WASM due to memory constraints")
252+
}
249253
// test page size from 1KB (1024<<0) to 16MB(1024<<14)
250254
for i := 0; i <= 14; i++ {
251255
givenPageSize := 1024 << uint(i)
@@ -331,6 +335,9 @@ func TestOpen_Size_Large(t *testing.T) {
331335
if testing.Short() {
332336
t.Skip("short mode")
333337
}
338+
if runtime.GOOS == "js" || runtime.GOOS == "wasip1" {
339+
t.Skip("skipping large file test in WASM")
340+
}
334341

335342
// Open a data file.
336343
db := btesting.MustCreateDB(t)
@@ -455,6 +462,10 @@ func TestOpen_FileTooSmall(t *testing.T) {
455462
// read transaction blocks the write transaction and causes deadlock.
456463
// This is a very hacky test since the mmap size is not exposed.
457464
func TestDB_Open_InitialMmapSize(t *testing.T) {
465+
t.Parallel()
466+
if runtime.GOOS == "js" || runtime.GOOS == "wasip1" {
467+
t.Skip("skipping test on WASM due to memory constraints")
468+
}
458469
path := tempfile()
459470
defer os.Remove(path)
460471

internal/common/bolt_wasm.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
//go:build js || wasip1
2+
3+
package common
4+
5+
// MaxMapSize represents the largest mmap size supported by Bolt.
6+
// Reduced for WASM due to memory constraints.
7+
const MaxMapSize = 0x10000000 // 256MB
8+
9+
// MaxAllocSize is the size used when creating array pointers.
10+
// Slightly larger than MaxMapSize to accommodate metadata overhead.
11+
const MaxAllocSize = 0x10100000 // 257MB
12+
13+
func init() {
14+
// Override the default page size for WASM to use 4KB instead of 64KB.
15+
// This reduces memory usage and makes behavior more consistent with other platforms.
16+
// Note: Databases created with different page sizes are not compatible.
17+
DefaultPageSize = 4096
18+
}
19+
20+
// GetPagesize returns the system page size.
21+
// On WASM platforms, we always return 4KB as the page size.
22+
func GetPagesize() int {
23+
return 4096
24+
}

internal/common/pagesize.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
//go:build !js && !wasip1
2+
3+
package common
4+
5+
import "os"
6+
7+
// GetPagesize returns the system page size.
8+
// On most platforms, this returns the OS page size.
9+
func GetPagesize() int {
10+
return os.Getpagesize()
11+
}

0 commit comments

Comments
 (0)