From 44f071655312b097eb10073dd6e11a6e8399df97 Mon Sep 17 00:00:00 2001 From: seelain <251264045+viridale@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:14:26 -0700 Subject: [PATCH 1/4] A non-UTF-8 argument reaches the program as bytes and opens its file (#1798) IO.args decodes argv as UTF-8, so a Latin-1 name such as "caf" E9 arrived as U+FFFD, could not be opened, and could not be told from "caf" E8. Two byte twins, as TCP, UDP and File.read have: IO.args_bytes gives each argument's bytes, and File.open_bytes opens a path given as bytes (a value past 255 fails with EINVAL, a 0 with EILSEQ). Each lives in its sibling's effect file under its own #ifdef CID. The JS lane reads the raw arguments from /proc/self/cmdline on Linux or _NSGetArgv under Bun on macOS when an argument holds a U+FFFD; node on macOS still sees EF BF BD. --- bend2/base.bend | 17 +++++++ bend2/effs/args.c | 26 +++++++++- bend2/effs/args.js | 50 +++++++++++++++++++ bend2/effs/file_open.c | 31 ++++++++++-- bend2/effs/file_open.js | 16 +++++- tests/io/args_bytes.bend | 28 +++++++++++ tests/io/file_open_bytes.bend | 91 +++++++++++++++++++++++++++++++++++ 7 files changed, 251 insertions(+), 8 deletions(-) create mode 100644 tests/io/args_bytes.bend create mode 100644 tests/io/file_open_bytes.bend diff --git a/bend2/base.bend b/bend2/base.bend index 5ab38471e5..48f117257a 100644 --- a/bend2/base.bend +++ b/bend2/base.bend @@ -192,6 +192,15 @@ def IO.args() -> IO(List): import "./effs/args.c" import "./effs/args.js" +# IO.args as the OS hands it over: each entry's bytes (0..255), one List +# cell each, with no UTF-8 decoding, so an argument that isn't UTF-8 (a +# Latin-1 file name) keeps every byte, and File.open_bytes opens it. Only +# node on macOS hides the bytes: there, a byte that isn't UTF-8 arrives as +# U+FFFD's UTF-8, EF BF BD. +def IO.args_bytes() -> IO(List>): + import "./effs/args.c" + import "./effs/args.js" + def Process.run(program: String, args: List, input: String, max_output: U32, timeout_ms: U32) -> IO(Result<&1, &1, U32 & String, U32 & (String & String)>): @@ -295,6 +304,14 @@ def File.open(path: String, mode: String) -> import "./effs/file_open.c" import "./effs/file_open.js" +# The path as its bytes (0..255), with no UTF-8 encoding, so a name that +# isn't UTF-8, as IO.args_bytes gives it, opens. A value past 255 fails +# with EINVAL, and a 0 with EILSEQ, as a NUL in File.open's path does. +def File.open_bytes(path: List<&2, U32>, mode: String) -> + IO(Result<&1, &1, U32 & String, File>): + import "./effs/file_open.c" + import "./effs/file_open.js" + def File.read(file: File, max: U32) -> IO(File & Result<&1, &1, U32 & String, String>): import "./effs/file_read.c" diff --git a/bend2/effs/args.c b/bend2/effs/args.c index eee598e461..7d9cc03305 100644 --- a/bend2/effs/args.c +++ b/bend2/effs/args.c @@ -1,15 +1,37 @@ // IO // == -Term io_args_run(Env e, Term* f, IoWork* w) { +// str makes an argument's value from its bytes: io_str's String, or +// io_list's bytes for IO.args_bytes. +static Term io_args_with(Env e, Term (*str)(Env e, const char* p, u64 n)) { Term xs = term_pak(CID(Nil), 0); for (int i = io_argc; i > 0; i -= 1) { const char* a = io_argv[i - 1]; - xs = io_node(e, CID(Con), io_str(e, a, strlen(a)), xs); + xs = io_node(e, CID(Con), str(e, a, strlen(a)), xs); } return xs; } +#ifdef CID(IO.args) + +Term io_args_run(Env e, Term* f, IoWork* w) { + return io_args_with(e, io_str); +} + static void __attribute__((constructor)) io_args_use(void) { io_eff(CID(IO.args), io_args_run); } + +#endif + +#ifdef CID(IO.args_bytes) + +Term io_args_bytes_run(Env e, Term* f, IoWork* w) { + return io_args_with(e, io_list); +} + +static void __attribute__((constructor)) io_args_bytes_use(void) { + io_eff(CID(IO.args_bytes), io_args_bytes_run); +} + +#endif diff --git a/bend2/effs/args.js b/bend2/effs/args.js index c1f3a94c9a..1d53842be8 100644 --- a/bend2/effs/args.js +++ b/bend2/effs/args.js @@ -9,4 +9,54 @@ function io_args() { return xs; } +// Node and Bun decode the arguments before a program sees them, so a byte +// that isn't UTF-8 arrives as U+FFFD. Each argument takes the last raw one, +// before the one the next argument took, that decodes to it; with no such +// one (no raw ones, or the head, which the host makes absolute), its UTF-8. +function io_args_bytes() { + const raw = cli_args.some((a) => a.includes("�")) ? io_args_raw() : []; + let xs = { $: CID(Nil) }; + for (let i = cli_args.length, at = raw.length; i > 0; i -= 1) { + let k = at - 1; + while (k >= 0 && io_text(raw[k], raw[k].length) !== cli_args[i - 1]) { + k -= 1; + } + at = k < 0 ? at : k; + const b = k < 0 ? io_bytes(cli_args[i - 1]) : raw[k]; + xs = { $: CID(Con), head: io_list(b, b.length), tail: xs }; + } + return xs; +} + +// The process's arguments as the OS holds them: Linux's /proc/self/cmdline, +// or macOS's _NSGetArgv through bun:ffi; none under node on macOS. +function io_args_raw() { + try { + if (process.platform === "linux") { + const b = require("fs").readFileSync("/proc/self/cmdline"); + const xs = []; + for (let at = 0, k; (k = b.indexOf(0, at)) >= 0; at = k + 1) { + xs.push(b.subarray(at, k)); + } + return xs; + } + const ffi = require("bun:ffi"); + const get = { args: [], returns: "ptr" }; + const sys = ffi.dlopen("libSystem.dylib", + { _NSGetArgc: get, _NSGetArgv: get }).symbols; + const argv = ffi.read.ptr(sys._NSGetArgv(), 0); + return Array.from({ length: ffi.read.i32(sys._NSGetArgc(), 0) }, (_, i) => { + const p = ffi.read.ptr(argv, 8 * i); + let n = 0; + while (ffi.read.u8(p, n) !== 0) { + n += 1; + } + return Uint8Array.from({ length: n }, (_, j) => ffi.read.u8(p, j)); + }); + } catch (_) { + return []; + } +} + io_eff(CID(IO.args), io_args); +io_eff(CID(IO.args_bytes), io_args_bytes); diff --git a/bend2/effs/file_open.c b/bend2/effs/file_open.c index f463831405..4fcd0eaef0 100644 --- a/bend2/effs/file_open.c +++ b/bend2/effs/file_open.c @@ -23,20 +23,43 @@ static Term file_open_pack(Env e, IoWork* w) { return io_res(e, w, io_hand(w->made)); } -Term file_open_run(Env e, Term* f, IoWork* w) { +// cons is CID(SCon) for a String path, opened as its UTF-8, or CID(Con) +// for a List, opened as its bytes (NULL past 255). +static Term file_open_start(Env e, Term* f, IoWork* w, u64 cons) { uint64_t mn = 0; - w->data = io_cstr(e, f[0], &w->size); + w->data = io_cbuf(e, f[0], &w->size, cons); char* mode = io_cstr(e, f[1], &mn); int flags = io_nul(mode, mn) ? -1 : file_open_mode(mode); free(mode); w->word = (uint32_t)flags; - if (io_nul(w->data, w->size) || flags < 0) { - w->code = io_nul(w->data, w->size) ? EILSEQ : EINVAL; + w->code = w->data == NULL ? EINVAL : io_nul(w->data, w->size) ? EILSEQ + : flags < 0 ? EINVAL : 0; + if (w->code != 0) { return file_open_pack(e, w); } return io_work(w, file_open_call, file_open_pack); } +#ifdef CID(File.open) + +Term file_open_run(Env e, Term* f, IoWork* w) { + return file_open_start(e, f, w, CID(SCon)); +} + static void __attribute__((constructor)) file_open_use(void) { io_eff(CID(File.open), file_open_run); } + +#endif + +#ifdef CID(File.open_bytes) + +Term file_open_bytes_run(Env e, Term* f, IoWork* w) { + return file_open_start(e, f, w, CID(Con)); +} + +static void __attribute__((constructor)) file_open_bytes_use(void) { + io_eff(CID(File.open_bytes), file_open_bytes_run); +} + +#endif diff --git a/bend2/effs/file_open.js b/bend2/effs/file_open.js index 2b98f8fb4c..2f0862416c 100644 --- a/bend2/effs/file_open.js +++ b/bend2/effs/file_open.js @@ -1,8 +1,11 @@ // File // ==== -function file_open(path, mode) { - const name = io_bytes(path); +// name is the path's bytes, or null if a value is past 255. +function file_open_with(name, mode) { + if (name === null) { + return io_fail(22); + } if (name.includes(0)) { return io_fail(process.platform === "darwin" ? 92 : 84); } @@ -18,4 +21,13 @@ function file_open(path, mode) { } } +function file_open(path, mode) { + return file_open_with(io_bytes(path), mode); +} + +function file_open_bytes(path, mode) { + return file_open_with(io_unlist(path), mode); +} + io_eff(CID(File.open), file_open); +io_eff(CID(File.open_bytes), file_open_bytes); diff --git a/tests/io/args_bytes.bend b/tests/io/args_bytes.bend new file mode 100644 index 0000000000..1d9ff512af --- /dev/null +++ b/tests/io/args_bytes.bend @@ -0,0 +1,28 @@ +# io (io_args_bytes): IO.args_bytes is IO.args as the OS hands it over, +# with no UTF-8 decoding (#1798): a Latin-1 argument "caf" E9 arrives as +# 99 97 102 233, where IO.args gives "caf" U+FFFD, the same String as for +# "caf" E8. The harness passes no argument, so every lane sees its program, +# named after this test, and no argument. +import Base + +def latin1(xs: List<&2, U32>) -> String: + match xs: + case Nil{}: + SNil{} + case x <> t: + SCon{Char.from_u32(x), latin1(t)} + +def program(xs: List>) -> String: + match xs: + case Nil{}: + "no program" + case p <> t: + Bool.show(String.contains(latin1(p), "args_bytes")) ++ " " + ++ Nat.show(List.length(&1, List<&2, U32>, t)) + +def main() -> IO(Unit): + do IO: + xs : List> <- IO.args_bytes() + IO.print(program(xs)) + +#|True 0 diff --git a/tests/io/file_open_bytes.bend b/tests/io/file_open_bytes.bend new file mode 100644 index 0000000000..bdee8d724c --- /dev/null +++ b/tests/io/file_open_bytes.bend @@ -0,0 +1,91 @@ +# io (io_file_open_bytes): File.open_bytes opens a path given as bytes, +# with no UTF-8 encoding (#1798). UTF-8 bytes name the file File.open's +# String names; a value past 255 fails with EINVAL and a 0 with EILSEQ. A +# Latin-1 E9 reaches the OS as itself: Linux makes that name, and "caf" +# U+FFFD, all IO.args could give for it, names no file; macOS file systems +# take UTF-8 names only, and refuse it with EILSEQ. +import Base + +def ascii(s: String, tail: List<&2, U32>) -> List<&2, U32>: + match s: + case SNil{}: + tail + case SCon{c, t}: + Char.to_u32(c) <> ascii(t, tail) + +def path(tail: List<&2, U32>) -> List<&2, U32>: + ascii("/tmp/bend_io_open_bytes_", tail) + +def pick(ok: Bool, yes: String, no: String) -> String: + match ok: + case True{}: + yes + case False{}: + no + +def kept(+got: String, want: String) -> String: + pick(String.eq(got, want), "E9 kept", got) + +# EILSEQ is 92 on macOS and 84 on Linux. +def code(+c: U32) -> String: + pick(U32.is_eq(c, 92) || U32.is_eq(c, 84), "EILSEQ", U32.show(c)) + +def shown(r: Result<&1, &1, U32 & String, File>) -> IO(String): + match r: + case Done{file}: + do IO: + File.close(file) + return "opened" + case Fail{(c, message)}: + IO.pure(String, code(c)) + +def written(pair: File & Result<&1, &1, U32 & String, Unit>) -> IO(Unit): + (file, result) = pair + do IO: + IO.pass(Unit, result) + File.close(file) + +def read(pair: File & Result<&1, &1, U32 & String, String>) -> IO(String): + (file, result) = pair + do IO: + text : String <- IO.pass(String, result) + File.close(file) + return text + +def latin1(r: Result<&1, &1, U32 & String, File>) -> IO(String): + match r: + case Done{file}: + do IO: + File.close(file) + a : Result<&1, &1, U32 & String, File> <- File.open_bytes(path([233]), "r") + a : String <- shown(a) + b : Result<&1, &1, U32 & String, File> <- + File.open("/tmp/bend_io_open_bytes_\u{fffd}", "r") + b : String <- shown(b) + return kept(a ++ " " ++ b, "opened 2") + case Fail{(c, message)}: + IO.pure(String, kept(code(c), "EILSEQ")) + +def main() -> IO(Unit): + do IO: + f : File <- IO.try(File, File.open_bytes(path([195, 169]), "w")) + p : File & Result<&1, &1, U32 & String, Unit> <- File.write(f, "hi") + written(p) + f : File <- IO.try(File, File.open("/tmp/bend_io_open_bytes_é", "r")) + p : File & Result<&1, &1, U32 & String, String> <- File.read(f, 16) + t : String <- read(p) + IO.print("utf-8 bytes: " ++ t) + r : Result<&1, &1, U32 & String, File> <- File.open_bytes(path([256]), "r") + t : String <- shown(r) + IO.print("past 255: " ++ t) + r : Result<&1, &1, U32 & String, File> <- File.open_bytes(path([0, 120]), "w") + t : String <- shown(r) + IO.print("nul: " ++ t) + r : Result<&1, &1, U32 & String, File> <- File.open_bytes(path([233]), "w") + t : String <- latin1(r) + IO.print("latin-1: " ++ t) + +#|utf-8 bytes: hi +#|past 255: 22 +#|nul: EILSEQ +#|latin-1: E9 kept From 2ac26c96a5a5af3b711c19e886bd4331d3cdc705 Mon Sep 17 00:00:00 2001 From: Lorenzobattistela Date: Wed, 21 Oct 2026 06:59:44 -0300 Subject: [PATCH 2/4] The JS lane's IO.args_bytes is IO.args's entries as UTF-8 Drop io_args_raw (/proc/self/cmdline, _NSGetArgv through bun:ffi) and the matching of decoded arguments against it: platform-specific, never reached by a gate, and hiding its failures behind a catch. Node and Bun decode argv before the program runs, so in JS a byte that isn't UTF-8 arrives as EF BF BD; the C lane keeps every byte. Co-Authored-By: Claude Opus 5.5 (1M context) --- bend2/effs/args.js | 46 ++++------------------------------------------ 1 file changed, 4 insertions(+), 42 deletions(-) diff --git a/bend2/effs/args.js b/bend2/effs/args.js index 1d53842be8..58297b3408 100644 --- a/bend2/effs/args.js +++ b/bend2/effs/args.js @@ -9,54 +9,16 @@ function io_args() { return xs; } -// Node and Bun decode the arguments before a program sees them, so a byte -// that isn't UTF-8 arrives as U+FFFD. Each argument takes the last raw one, -// before the one the next argument took, that decodes to it; with no such -// one (no raw ones, or the head, which the host makes absolute), its UTF-8. +// Node and Bun decode the arguments before a program sees them, so here a +// byte that isn't UTF-8 arrives as U+FFFD's UTF-8, EF BF BD. function io_args_bytes() { - const raw = cli_args.some((a) => a.includes("�")) ? io_args_raw() : []; let xs = { $: CID(Nil) }; - for (let i = cli_args.length, at = raw.length; i > 0; i -= 1) { - let k = at - 1; - while (k >= 0 && io_text(raw[k], raw[k].length) !== cli_args[i - 1]) { - k -= 1; - } - at = k < 0 ? at : k; - const b = k < 0 ? io_bytes(cli_args[i - 1]) : raw[k]; + for (let i = cli_args.length; i > 0; i -= 1) { + const b = io_bytes(cli_args[i - 1]); xs = { $: CID(Con), head: io_list(b, b.length), tail: xs }; } return xs; } -// The process's arguments as the OS holds them: Linux's /proc/self/cmdline, -// or macOS's _NSGetArgv through bun:ffi; none under node on macOS. -function io_args_raw() { - try { - if (process.platform === "linux") { - const b = require("fs").readFileSync("/proc/self/cmdline"); - const xs = []; - for (let at = 0, k; (k = b.indexOf(0, at)) >= 0; at = k + 1) { - xs.push(b.subarray(at, k)); - } - return xs; - } - const ffi = require("bun:ffi"); - const get = { args: [], returns: "ptr" }; - const sys = ffi.dlopen("libSystem.dylib", - { _NSGetArgc: get, _NSGetArgv: get }).symbols; - const argv = ffi.read.ptr(sys._NSGetArgv(), 0); - return Array.from({ length: ffi.read.i32(sys._NSGetArgc(), 0) }, (_, i) => { - const p = ffi.read.ptr(argv, 8 * i); - let n = 0; - while (ffi.read.u8(p, n) !== 0) { - n += 1; - } - return Uint8Array.from({ length: n }, (_, j) => ffi.read.u8(p, j)); - }); - } catch (_) { - return []; - } -} - io_eff(CID(IO.args), io_args); io_eff(CID(IO.args_bytes), io_args_bytes); From 529c7e487bd6d2468cc49870783011e9939c1013 Mon Sep 17 00:00:00 2001 From: Lorenzobattistela Date: Wed, 21 Oct 2026 07:00:09 -0300 Subject: [PATCH 3/4] IO.args_bytes' doc names the JS lanes' loss; file_open_bytes drops Latin-1 The doc said only node on macOS loses a byte that isn't UTF-8; with no raw argv in JS, every JS lane does. The test's Latin-1 case asserted what the file system under /tmp does with a name that isn't UTF-8 (APFS refuses it, ext4 takes it, HFS+ and utf8only ZFS differ), not what File.open_bytes does. Co-Authored-By: Claude Opus 5.5 (1M context) --- bend2/base.bend | 6 +++--- tests/io/file_open_bytes.bend | 26 +------------------------- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/bend2/base.bend b/bend2/base.bend index 48f117257a..88f0d781a9 100644 --- a/bend2/base.bend +++ b/bend2/base.bend @@ -194,9 +194,9 @@ def IO.args() -> IO(List): # IO.args as the OS hands it over: each entry's bytes (0..255), one List # cell each, with no UTF-8 decoding, so an argument that isn't UTF-8 (a -# Latin-1 file name) keeps every byte, and File.open_bytes opens it. Only -# node on macOS hides the bytes: there, a byte that isn't UTF-8 arrives as -# U+FFFD's UTF-8, EF BF BD. +# Latin-1 file name) keeps every byte, and File.open_bytes opens it. The +# JS lanes get the arguments already decoded by the host: there, a byte +# that isn't UTF-8 arrives as U+FFFD's UTF-8, EF BF BD. def IO.args_bytes() -> IO(List>): import "./effs/args.c" import "./effs/args.js" diff --git a/tests/io/file_open_bytes.bend b/tests/io/file_open_bytes.bend index bdee8d724c..9aa9e2c48f 100644 --- a/tests/io/file_open_bytes.bend +++ b/tests/io/file_open_bytes.bend @@ -1,9 +1,6 @@ # io (io_file_open_bytes): File.open_bytes opens a path given as bytes, # with no UTF-8 encoding (#1798). UTF-8 bytes name the file File.open's -# String names; a value past 255 fails with EINVAL and a 0 with EILSEQ. A -# Latin-1 E9 reaches the OS as itself: Linux makes that name, and "caf" -# U+FFFD, all IO.args could give for it, names no file; macOS file systems -# take UTF-8 names only, and refuse it with EILSEQ. +# String names; a value past 255 fails with EINVAL and a 0 with EILSEQ. import Base def ascii(s: String, tail: List<&2, U32>) -> List<&2, U32>: @@ -23,9 +20,6 @@ def pick(ok: Bool, yes: String, no: String) -> String: case False{}: no -def kept(+got: String, want: String) -> String: - pick(String.eq(got, want), "E9 kept", got) - # EILSEQ is 92 on macOS and 84 on Linux. def code(+c: U32) -> String: pick(U32.is_eq(c, 92) || U32.is_eq(c, 84), "EILSEQ", U32.show(c)) @@ -52,20 +46,6 @@ def read(pair: File & Result<&1, &1, U32 & String, String>) -> IO(String): File.close(file) return text -def latin1(r: Result<&1, &1, U32 & String, File>) -> IO(String): - match r: - case Done{file}: - do IO: - File.close(file) - a : Result<&1, &1, U32 & String, File> <- File.open_bytes(path([233]), "r") - a : String <- shown(a) - b : Result<&1, &1, U32 & String, File> <- - File.open("/tmp/bend_io_open_bytes_\u{fffd}", "r") - b : String <- shown(b) - return kept(a ++ " " ++ b, "opened 2") - case Fail{(c, message)}: - IO.pure(String, kept(code(c), "EILSEQ")) - def main() -> IO(Unit): do IO: f : File <- IO.try(File, File.open_bytes(path([195, 169]), "w")) @@ -81,11 +61,7 @@ def main() -> IO(Unit): r : Result<&1, &1, U32 & String, File> <- File.open_bytes(path([0, 120]), "w") t : String <- shown(r) IO.print("nul: " ++ t) - r : Result<&1, &1, U32 & String, File> <- File.open_bytes(path([233]), "w") - t : String <- latin1(r) - IO.print("latin-1: " ++ t) #|utf-8 bytes: hi #|past 255: 22 #|nul: EILSEQ -#|latin-1: E9 kept From fc2ed57502d008a02ce33962810e3a31b0fc014c Mon Sep 17 00:00:00 2001 From: Lorenzobattistela Date: Wed, 21 Oct 2026 07:02:29 -0300 Subject: [PATCH 4/4] args_bytes' header says the Latin-1 bytes are kept in C, not in JS Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/io/args_bytes.bend | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/io/args_bytes.bend b/tests/io/args_bytes.bend index 1d9ff512af..6135ae6951 100644 --- a/tests/io/args_bytes.bend +++ b/tests/io/args_bytes.bend @@ -1,8 +1,9 @@ # io (io_args_bytes): IO.args_bytes is IO.args as the OS hands it over, -# with no UTF-8 decoding (#1798): a Latin-1 argument "caf" E9 arrives as -# 99 97 102 233, where IO.args gives "caf" U+FFFD, the same String as for -# "caf" E8. The harness passes no argument, so every lane sees its program, -# named after this test, and no argument. +# with no UTF-8 decoding (#1798): in C a Latin-1 argument "caf" E9 arrives +# as 99 97 102 233, where IO.args gives "caf" U+FFFD, the same String as +# for "caf" E8; the JS lanes get it decoded, as EF BF BD. The harness +# passes no argument, so every lane sees its program, named after this +# test, and no argument. import Base def latin1(xs: List<&2, U32>) -> String: