A non-UTF-8 argument reaches the program as bytes and opens its file (#1798)

bend2/base.bend

@@ -192,6 +192,15 @@ def IO.args() -> IO(List<String>):
   import "./effs/args.c"
   import "./effs/args.js"
 
+# IO.args as the OS hands it over: each entry's bytes (0..255), one List
+# cell each, with no UTF-8 decoding, so an argument that isn't UTF-8 (a
+# Latin-1 file name) keeps every byte, and File.open_bytes opens it. The
+# JS lanes get the arguments already decoded by the host: there, a byte
+# that isn't UTF-8 arrives as U+FFFD's UTF-8, EF BF BD.
+def IO.args_bytes() -> IO(List<List<&2, U32>>):
+  import "./effs/args.c"
+  import "./effs/args.js"
+
 def Process.run(program: String, args: List<String>, input: String,
   max_output: U32, timeout_ms: U32) ->
   IO(Result<&1, &1, U32 & String, U32 & (String & String)>):
@@ -295,6 +304,14 @@ def File.open(path: String, mode: String) ->
   import "./effs/file_open.c"
   import "./effs/file_open.js"
 
+# The path as its bytes (0..255), with no UTF-8 encoding, so a name that
+# isn't UTF-8, as IO.args_bytes gives it, opens. A value past 255 fails
+# with EINVAL, and a 0 with EILSEQ, as a NUL in File.open's path does.
+def File.open_bytes(path: List<&2, U32>, mode: String) ->
+  IO(Result<&1, &1, U32 & String, File>):
+  import "./effs/file_open.c"
+  import "./effs/file_open.js"
+
 def File.read(file: File, max: U32) ->
   IO(File & Result<&1, &1, U32 & String, String>):
   import "./effs/file_read.c"

bend2/effs/args.c

@@ -1,15 +1,37 @@
 // IO
 // ==
 
-Term io_args_run(Env e, Term* f, IoWork* w) {
+// str makes an argument's value from its bytes: io_str's String, or
+// io_list's bytes for IO.args_bytes.
+static Term io_args_with(Env e, Term (*str)(Env e, const char* p, u64 n)) {
   Term xs = term_pak(CID(Nil), 0);
   for (int i = io_argc; i > 0; i -= 1) {
     const char* a = io_argv[i - 1];
-    xs = io_node(e, CID(Con), io_str(e, a, strlen(a)), xs);
+    xs = io_node(e, CID(Con), str(e, a, strlen(a)), xs);
   }
   return xs;
 }
 
+#ifdef CID(IO.args)
+
+Term io_args_run(Env e, Term* f, IoWork* w) {
+  return io_args_with(e, io_str);
+}
+
 static void __attribute__((constructor)) io_args_use(void) {
   io_eff(CID(IO.args), io_args_run);
 }
+
+#endif
+
+#ifdef CID(IO.args_bytes)
+
+Term io_args_bytes_run(Env e, Term* f, IoWork* w) {
+  return io_args_with(e, io_list);
+}
+
+static void __attribute__((constructor)) io_args_bytes_use(void) {
+  io_eff(CID(IO.args_bytes), io_args_bytes_run);
+}
+
+#endif

bend2/effs/args.js

@@ -9,4 +9,16 @@ function io_args() {
   return xs;
 }
 
+// Node and Bun decode the arguments before a program sees them, so here a
+// byte that isn't UTF-8 arrives as U+FFFD's UTF-8, EF BF BD.
+function io_args_bytes() {
+  let xs = { $: CID(Nil) };
+  for (let i = cli_args.length; i > 0; i -= 1) {
+    const b = io_bytes(cli_args[i - 1]);
+    xs = { $: CID(Con), head: io_list(b, b.length), tail: xs };
+  }
+  return xs;
+}
+
 io_eff(CID(IO.args), io_args);
+io_eff(CID(IO.args_bytes), io_args_bytes);

bend2/effs/file_open.c

@@ -23,20 +23,43 @@ static Term file_open_pack(Env e, IoWork* w) {
   return io_res(e, w, io_hand(w->made));
 }
 
-Term file_open_run(Env e, Term* f, IoWork* w) {
+// cons is CID(SCon) for a String path, opened as its UTF-8, or CID(Con)
+// for a List, opened as its bytes (NULL past 255).
+static Term file_open_start(Env e, Term* f, IoWork* w, u64 cons) {
   uint64_t mn = 0;
-  w->data = io_cstr(e, f[0], &w->size);
+  w->data = io_cbuf(e, f[0], &w->size, cons);
   char* mode = io_cstr(e, f[1], &mn);
   int flags = io_nul(mode, mn) ? -1 : file_open_mode(mode);
   free(mode);
   w->word = (uint32_t)flags;
-  if (io_nul(w->data, w->size) || flags < 0) {
-    w->code = io_nul(w->data, w->size) ? EILSEQ : EINVAL;
+  w->code = w->data == NULL ? EINVAL : io_nul(w->data, w->size) ? EILSEQ
+    : flags < 0 ? EINVAL : 0;
+  if (w->code != 0) {
     return file_open_pack(e, w);
   }
   return io_work(w, file_open_call, file_open_pack);
 }
 
+#ifdef CID(File.open)
+
+Term file_open_run(Env e, Term* f, IoWork* w) {
+  return file_open_start(e, f, w, CID(SCon));
+}
+
 static void __attribute__((constructor)) file_open_use(void) {
   io_eff(CID(File.open), file_open_run);
 }
+
+#endif
+
+#ifdef CID(File.open_bytes)
+
+Term file_open_bytes_run(Env e, Term* f, IoWork* w) {
+  return file_open_start(e, f, w, CID(Con));
+}
+
+static void __attribute__((constructor)) file_open_bytes_use(void) {
+  io_eff(CID(File.open_bytes), file_open_bytes_run);
+}
+
+#endif

bend2/effs/file_open.js

@@ -1,8 +1,11 @@
 // File
 // ====
 
-function file_open(path, mode) {
-  const name = io_bytes(path);
+// name is the path's bytes, or null if a value is past 255.
+function file_open_with(name, mode) {
+  if (name === null) {
+    return io_fail(22);
+  }
   if (name.includes(0)) {
     return io_fail(process.platform === "darwin" ? 92 : 84);
   }
@@ -18,4 +21,13 @@ function file_open(path, mode) {
   }
 }
 
+function file_open(path, mode) {
+  return file_open_with(io_bytes(path), mode);
+}
+
+function file_open_bytes(path, mode) {
+  return file_open_with(io_unlist(path), mode);
+}
+
 io_eff(CID(File.open), file_open);
+io_eff(CID(File.open_bytes), file_open_bytes);

tests/io/args_bytes.bend

@@ -0,0 +1,29 @@
+# io (io_args_bytes): IO.args_bytes is IO.args as the OS hands it over,
+# with no UTF-8 decoding (#1798): in C a Latin-1 argument "caf" E9 arrives
+# as 99 97 102 233, where IO.args gives "caf" U+FFFD, the same String as
+# for "caf" E8; the JS lanes get it decoded, as EF BF BD. The harness
+# passes no argument, so every lane sees its program, named after this
+# test, and no argument.
+import Base
+
+def latin1(xs: List<&2, U32>) -> String:
+  match xs:
+    case Nil{}:
+      SNil{}
+    case x <> t:
+      SCon{Char.from_u32(x), latin1(t)}
+
+def program(xs: List<List<&2, U32>>) -> String:
+  match xs:
+    case Nil{}:
+      "no program"
+    case p <> t:
+      Bool.show(String.contains(latin1(p), "args_bytes")) ++ " "
+        ++ Nat.show(List.length(&1, List<&2, U32>, t))
+
+def main() -> IO(Unit):
+  do IO<Unit>:
+    xs : List<List<&2, U32>> <- IO.args_bytes()
+    IO.print(program(xs))
+
+#|True 0

tests/io/file_open_bytes.bend

@@ -0,0 +1,67 @@
+# io (io_file_open_bytes): File.open_bytes opens a path given as bytes,
+# with no UTF-8 encoding (#1798). UTF-8 bytes name the file File.open's
+# String names; a value past 255 fails with EINVAL and a 0 with EILSEQ.
+import Base
+
+def ascii(s: String, tail: List<&2, U32>) -> List<&2, U32>:
+  match s:
+    case SNil{}:
+      tail
+    case SCon{c, t}:
+      Char.to_u32(c) <> ascii(t, tail)
+
+def path(tail: List<&2, U32>) -> List<&2, U32>:
+  ascii("/tmp/bend_io_open_bytes_", tail)
+
+def pick(ok: Bool, yes: String, no: String) -> String:
+  match ok:
+    case True{}:
+      yes
+    case False{}:
+      no
+
+# EILSEQ is 92 on macOS and 84 on Linux.
+def code(+c: U32) -> String:
+  pick(U32.is_eq(c, 92) || U32.is_eq(c, 84), "EILSEQ", U32.show(c))
+
+def shown(r: Result<&1, &1, U32 & String, File>) -> IO(String):
+  match r:
+    case Done{file}:
+      do IO<String>:
+        File.close(file)
+        return "opened"
+    case Fail{(c, message)}:
+      IO.pure(String, code(c))
+
+def written(pair: File & Result<&1, &1, U32 & String, Unit>) -> IO(Unit):
+  (file, result) = pair
+  do IO<Unit>:
+    IO.pass(Unit, result)
+    File.close(file)
+
+def read(pair: File & Result<&1, &1, U32 & String, String>) -> IO(String):
+  (file, result) = pair
+  do IO<String>:
+    text : String <- IO.pass(String, result)
+    File.close(file)
+    return text
+
+def main() -> IO(Unit):
+  do IO<Unit>:
+    f : File <- IO.try(File, File.open_bytes(path([195, 169]), "w"))
+    p : File & Result<&1, &1, U32 & String, Unit> <- File.write(f, "hi")
+    written(p)
+    f : File <- IO.try(File, File.open("/tmp/bend_io_open_bytes_é", "r"))
+    p : File & Result<&1, &1, U32 & String, String> <- File.read(f, 16)
+    t : String <- read(p)
+    IO.print("utf-8 bytes: " ++ t)
+    r : Result<&1, &1, U32 & String, File> <- File.open_bytes(path([256]), "r")
+    t : String <- shown(r)
+    IO.print("past 255: " ++ t)
+    r : Result<&1, &1, U32 & String, File> <- File.open_bytes(path([0, 120]), "w")
+    t : String <- shown(r)
+    IO.print("nul: " ++ t)
+
+#|utf-8 bytes: hi
+#|past 255: 22
+#|nul: EILSEQ