diff --git a/src/domain/engine.rs b/src/domain/engine.rs index 56971f88..3010b754 100644 --- a/src/domain/engine.rs +++ b/src/domain/engine.rs @@ -253,16 +253,28 @@ impl<'i> Response<'i> { /// Render an Expression as human-readable text. /// Returns (expression_text, body_lines) where body_lines captures multiline /// content separately for distinct styling. +/// The text of a quoted literal, with any interpolation rendered as the +/// expression it came from. Shared by string values and tablet labels. +pub(crate) fn render_pieces(pieces: &[Piece]) -> String { + let mut result = String::new(); + + for piece in pieces { + match piece { + Piece::Text(t) => result.push_str(t), + Piece::Escaped(c) => result.push(*c), + Piece::Interpolation(e) => result.push_str(&render_expression(e)), + } + } + + result +} + fn render_expression_parts(expr: &Expression) -> (String, Vec) { if let Expression::Execution(func, _) = expr { let mut body = Vec::new(); for param in &func.parameters { - if let Expression::Multiline(_, lines, _) = param { - body.extend( - lines - .iter() - .map(|s| s.to_string()), - ); + if let Expression::Multiline(multiline, _) = param { + body.push(multiline.content()); } } if !body.is_empty() { @@ -334,26 +346,21 @@ fn render_expression(expr: &Expression) -> String { args.join(", ") ) } - Expression::Multiline(_, lines, _) => lines.join("\n"), + Expression::Multiline(multiline, _) => multiline.content(), Expression::Variable(id, _) => id .value .to_string(), Expression::Binding(inner, _, _) => render_expression(inner), - Expression::String(pieces, _) => { - let mut result = String::new(); - for piece in pieces { - match piece { - Piece::Text(t) => result.push_str(t), - Piece::Interpolation(e) => result.push_str(&render_expression(e)), - } - } - result - } + Expression::String(pieces, _) => render_pieces(pieces), Expression::Response(value, _) => format!("'{}'", value), Expression::Number(Numeric::Scientific(q), _) => q.to_string(), Expression::Number(Numeric::Integral(n), _) => n.to_string(), Expression::Pair(pair, _) => { - format!("\"{}\" = {}", pair.label, render_expression(&pair.value)) + format!( + "\"{}\" = {}", + render_pieces(&pair.label), + render_expression(&pair.value) + ) } Expression::List(elements, _) => { let items: Vec<_> = elements @@ -375,7 +382,13 @@ fn render_expression(expr: &Expression) -> String { } let entries: Vec<_> = pairs .iter() - .map(|pair| format!("\"{}\" = {}", pair.label, render_expression(&pair.value))) + .map(|pair| { + format!( + "\"{}\" = {}", + render_pieces(&pair.label), + render_expression(&pair.value) + ) + }) .collect(); format!("[{}]", entries.join(", ")) } diff --git a/src/domain/recipe/adapter.rs b/src/domain/recipe/adapter.rs index c23dcb07..632e4367 100644 --- a/src/domain/recipe/adapter.rs +++ b/src/domain/recipe/adapter.rs @@ -7,6 +7,7 @@ use std::collections::{HashMap, HashSet}; use crate::domain::Adapter; +use crate::domain::engine::render_pieces; use crate::language; use super::types::{Document, Ingredient, Ingredients, Prose, Step}; @@ -128,9 +129,7 @@ fn collect_ingredients(items: &mut Vec, scope: &language::Scope, pla if let Some(pairs) = scope.tablet() { for pair in pairs { items.push(Ingredient { - label: pair - .label - .to_string(), + label: render_pieces(&pair.label), quantity: format_value(&pair.value), source: place.map(String::from), }); @@ -143,9 +142,7 @@ fn collect_ingredients(items: &mut Vec, scope: &language::Scope, pla if let Some(pairs) = scope.inline_tablet() { for pair in pairs { items.push(Ingredient { - label: pair - .label - .to_string(), + label: render_pieces(&pair.label), quantity: format_value(&pair.value), source: place.map(String::from), }); diff --git a/src/editor/server.rs b/src/editor/server.rs index adf6dbc1..2454ee0c 100644 --- a/src/editor/server.rs +++ b/src/editor/server.rs @@ -843,6 +843,10 @@ impl TechniqueLanguageServer { "Invalid quantity symbol".to_string(), DiagnosticSeverity::ERROR, ), + ParsingError::InvalidEscape(_) => ( + "Invalid escape sequence".to_string(), + DiagnosticSeverity::ERROR, + ), ParsingError::UnclosedInterpolation(_) => ( "Unclosed interpolation".to_string(), DiagnosticSeverity::ERROR, diff --git a/src/formatting/formatter.rs b/src/formatting/formatter.rs index eb48819c..772830da 100644 --- a/src/formatting/formatter.rs +++ b/src/formatting/formatter.rs @@ -34,6 +34,15 @@ impl Substitutions { } } +fn escaped_as(c: char) -> char { + match c { + '\n' => 'n', + '\r' => 'r', + '\t' => 't', + other => other, + } +} + // Helper function to convert numbers to superscript fn to_superscript(num: i8) -> String { num.to_string() @@ -471,7 +480,7 @@ impl<'i> Formatter<'i> { return Vec::new(); } match expr { - Expression::Multiline(_, _, _) => { + Expression::Multiline(_, _) => { // These are not inline, caller should handle specially Vec::new() } @@ -904,7 +913,7 @@ impl<'i> Formatter<'i> { line = self.builder(); line.add_word(Syntax::Structure, "}"); } - Expression::Multiline(_, _, _) => { + Expression::Multiline(_, _) => { line.flush(); self.add_fragment_reference(Syntax::Structure, "{"); self.increase(4); @@ -921,7 +930,7 @@ impl<'i> Formatter<'i> { .parameters .iter() .any(|p| { - if let Expression::Multiline(_, _, _) = p { + if let Expression::Multiline(_, _) = p { true } else { false @@ -1267,22 +1276,7 @@ impl<'i> Formatter<'i> { self.add_fragment_reference(Syntax::Variable, identifier.value); } Expression::String(pieces, _) => { - self.add_fragment_reference(Syntax::Quote, "\""); - for piece in pieces { - match piece { - Piece::Text(text) => { - // Preserve user string content exactly as written - self.add_fragment_reference(Syntax::String, text); - } - Piece::Interpolation(expr) => { - let fragments = self.render_string_interpolation(expr); - for (syntax, content) in fragments { - self.add_fragment(syntax, content); - } - } - } - } - self.add_fragment_reference(Syntax::Quote, "\""); + self.append_quoted(Syntax::String, pieces); } Expression::Response(value, _) => { self.add_fragment_reference(Syntax::Quote, "'"); @@ -1290,28 +1284,21 @@ impl<'i> Formatter<'i> { self.add_fragment_reference(Syntax::Quote, "'"); } Expression::Number(numeric, _) => self.append_numeric(numeric), - Expression::Multiline(lang, lines, _) => { + Expression::Multiline(multiline, _) => { self.append_char('\n'); self.indent(); self.add_fragment_reference(Syntax::Quote, "```"); - if let Some(which) = lang { + if let Some(which) = multiline.language { self.add_fragment_reference(Syntax::Language, which); } self.append_char('\n'); self.increase(4); - for line in lines { - self.indent(); - // Break multiline content into words for wrapping - for (i, word) in line - .split_ascii_whitespace() - .enumerate() - { - if i > 0 { - self.add_fragment_reference(Syntax::Multiline, " "); - } - self.add_fragment_reference(Syntax::Multiline, word); + for line in multiline.lines() { + if !line.is_empty() { + self.indent(); + self.add_fragment(Syntax::Multiline, line); } self.append_char('\n'); } @@ -1488,7 +1475,7 @@ impl<'i> Formatter<'i> { let mut has_multiline = false; for parameter in &function.parameters { - if let Expression::Multiline(_, _, _) = parameter { + if let Expression::Multiline(_, _) = parameter { has_multiline = true; break; } @@ -1512,10 +1499,29 @@ impl<'i> Formatter<'i> { self.add_fragment_reference(Syntax::Structure, ")"); } - fn append_pair(&mut self, pair: &'i Pair) { + /// Write a quoted literal back out: text verbatim as it was borrowed, + /// escapes in the form they were written, interpolations rendered. + fn append_quoted(&mut self, syntax: Syntax, pieces: &'i [Piece]) { self.add_fragment_reference(Syntax::Quote, "\""); - self.add_fragment_reference(Syntax::Label, pair.label); + for piece in pieces { + match piece { + Piece::Text(text) => self.add_fragment_reference(syntax, text), + Piece::Escaped(c) => { + self.add_fragment(syntax, Cow::Owned(format!("\\{}", escaped_as(*c)))) + } + Piece::Interpolation(expr) => { + let fragments = self.render_string_interpolation(expr); + for (syntax, content) in fragments { + self.add_fragment(syntax, content); + } + } + } + } self.add_fragment_reference(Syntax::Quote, "\""); + } + + fn append_pair(&mut self, pair: &'i Pair) { + self.append_quoted(Syntax::Label, &pair.label); self.add_fragment_reference(Syntax::Neutral, " "); self.add_fragment_reference(Syntax::Structure, "="); self.add_fragment_reference(Syntax::Neutral, " "); diff --git a/src/language/mod.rs b/src/language/mod.rs index d5788b30..15aa0751 100644 --- a/src/language/mod.rs +++ b/src/language/mod.rs @@ -1,10 +1,12 @@ // Types representing the Technique language surface syntax mod error; +mod multiline; mod quantity; mod types; // Re-export all public symbols pub use error::*; +pub use multiline::*; pub use quantity::*; pub use types::*; diff --git a/src/language/multiline.rs b/src/language/multiline.rs new file mode 100644 index 00000000..f2b9054f --- /dev/null +++ b/src/language/multiline.rs @@ -0,0 +1,87 @@ +//! The content of a ``` fence + +use std::borrow::Cow; + +#[derive(Debug, Eq, PartialEq)] +pub struct Multiline<'i> { + pub language: Option<&'i str>, + pub lines: Vec<&'i str>, +} + +impl<'i> Multiline<'i> { + /// The lines as they were present in the input source but with the leading block + /// indentation removed. + pub fn lines(&self) -> impl Iterator> + '_ { + let common = self + .lines + .iter() + .filter(|line| { + !line + .trim_ascii() + .is_empty() + }) + .map(|line| indent(line)) + .min() + .unwrap_or(0); + + self.lines + .iter() + .map(move |line| strip(expand(line), common)) + } + + /// The lines, joined, suitable for use by builtin functions. + pub fn content(&self) -> String { + self.lines() + .collect::>>() + .join("\n") + } +} + +fn indent(line: &str) -> usize { + let mut column = 0; + + for c in line.chars() { + match c { + ' ' => column += 1, + '\t' => column = (column / 4 + 1) * 4, + _ => break, + } + } + + column +} + +/// Expand any tabs present into 4 spaces. If the author needs an actual tab +/// character in a sting literal they can use the `\t` escape. +fn expand(line: &str) -> Cow<'_, str> { + if !line.contains('\t') { + return Cow::Borrowed(line); + } + + let mut result = String::with_capacity(line.len() + 8); + let mut column = 0; + + for c in line.chars() { + if c == '\t' { + let stop = (column / 4 + 1) * 4; + result.push_str(&" ".repeat(stop - column)); + column = stop; + } else { + result.push(c); + column += 1; + } + } + + result.into() +} + +/// Strip leading block indendation from a line. +fn strip(line: Cow<'_, str>, common: usize) -> Cow<'_, str> { + match line { + Cow::Borrowed(text) => Cow::Borrowed(&text[common.min(text.len())..]), + Cow::Owned(mut text) => { + text.drain(..common.min(text.len())); + Cow::Owned(text) + } + } +} diff --git a/src/language/types.rs b/src/language/types.rs index 10258a78..c75ac766 100644 --- a/src/language/types.rs +++ b/src/language/types.rs @@ -1,5 +1,7 @@ //! Types representing an Abstract Syntax Tree for the Technique language +use crate::language::multiline::Multiline; +use crate::language::quantity::Quantity; use crate::regex::*; /// Byte range within the original source. `length` excludes trailing whitespace. @@ -505,13 +507,15 @@ pub struct Function<'i> { #[derive(Debug, PartialEq, Eq)] pub struct Pair<'i> { - pub label: &'i str, + pub label: Vec>, pub value: Expression<'i>, } #[derive(Debug, PartialEq, Eq)] pub enum Piece<'i> { Text(&'i str), + /// The character a backslash escape stood for + Escaped(char), Interpolation(Expression<'i>), } @@ -521,7 +525,7 @@ pub enum Expression<'i> { String(Vec>, Span), Response(&'i str, Span), Number(Numeric<'i>, Span), - Multiline(Option<&'i str>, Vec<&'i str>, Span), + Multiline(Multiline<'i>, Span), Repeat(Box>, Span), Foreach(Vec>, Box>, Span), Within(Box>, Span), @@ -545,9 +549,7 @@ impl PartialEq for Expression<'_> { (Expression::String(a, _), Expression::String(b, _)) => a == b, (Expression::Response(a, _), Expression::Response(b, _)) => a == b, (Expression::Number(a, _), Expression::Number(b, _)) => a == b, - (Expression::Multiline(a1, a2, _), Expression::Multiline(b1, b2, _)) => { - a1 == b1 && a2 == b2 - } + (Expression::Multiline(a, _), Expression::Multiline(b, _)) => a == b, (Expression::Repeat(a, _), Expression::Repeat(b, _)) => a == b, (Expression::Foreach(a1, a2, _), Expression::Foreach(b1, b2, _)) => { a1 == b1 && a2 == b2 @@ -577,8 +579,6 @@ pub enum Numeric<'i> { Scientific(Quantity<'i>), } -pub use crate::language::quantity::Quantity; - // the validate functions all need to have start and end anchors, which seems // like it should be abstracted away. diff --git a/src/linking/linker.rs b/src/linking/linker.rs index 386c7287..efdcf104 100644 --- a/src/linking/linker.rs +++ b/src/linking/linker.rs @@ -108,6 +108,12 @@ fn link_operation<'i>( } Operation::Tablet(entries, _) => { for entry in entries { + // a label is a quoted literal, so it can interpolate too + for fragment in &mut entry.label { + if let Fragment::Interpolation(op) = fragment { + link_operation(op, library, problems); + } + } link_operation(&mut entry.value, library, problems); } } @@ -119,7 +125,7 @@ fn link_operation<'i>( Operation::Variable(_, _) | Operation::Number(_, _) | Operation::Response(_, _) - | Operation::Multiline(_, _, _) + | Operation::Verbatim(_, _) | Operation::Prose(_, _) | Operation::Hole(_) | Operation::Unit(_) => {} diff --git a/src/parsing/checks/parser.rs b/src/parsing/checks/parser.rs index eb3ad040..9d4e0feb 100644 --- a/src/parsing/checks/parser.rs +++ b/src/parsing/checks/parser.rs @@ -637,6 +637,174 @@ fn string_delimited_blocks() { assert_eq!(input.offset, 21); } +#[test] +fn string_escapes() { + let mut input = Parser::new(); + + // an escaped quote is content, and does not end the literal + input.initialize(r#"{ "say \"hello\" now" }"#); + assert_eq!( + input.read_code_block(), + Ok(vec![Expression::String( + vec![ + Piece::Text("say "), + Piece::Escaped('"'), + Piece::Text("hello"), + Piece::Escaped('"'), + Piece::Text(" now"), + ], + Span::default() + )]) + ); + + // an escaped brace is content, and does not open an interpolation + input.initialize(r#"{ "awk '\{print $1\}'" }"#); + assert_eq!( + input.read_code_block(), + Ok(vec![Expression::String( + vec![ + Piece::Text("awk '"), + Piece::Escaped('{'), + Piece::Text("print $1"), + Piece::Escaped('}'), + Piece::Text("'"), + ], + Span::default() + )]) + ); + + // an unescaped brace still interpolates, alongside escapes + input.initialize(r#"{ "deploy { customer } \"now\"" }"#); + assert_eq!( + input.read_code_block(), + Ok(vec![Expression::String( + vec![ + Piece::Text("deploy "), + Piece::Interpolation(Expression::Variable( + Identifier::new("customer"), + Span::default() + )), + Piece::Text(" "), + Piece::Escaped('"'), + Piece::Text("now"), + Piece::Escaped('"'), + ], + Span::default() + )]) + ); + + // the remaining escapes + input.initialize(r#"{ "a\nb\rc\td\\e" }"#); + assert_eq!( + input.read_code_block(), + Ok(vec![Expression::String( + vec![ + Piece::Text("a"), + Piece::Escaped('\n'), + Piece::Text("b"), + Piece::Escaped('\r'), + Piece::Text("c"), + Piece::Escaped('\t'), + Piece::Text("d"), + Piece::Escaped('\\'), + Piece::Text("e"), + ], + Span::default() + )]) + ); + + // a lone closing brace is literal, needing no escape + input.initialize(r#"{ "a } b" }"#); + assert_eq!( + input.read_code_block(), + Ok(vec![Expression::String( + vec![Piece::Text("a } b")], + Span::default() + )]) + ); +} + +#[test] +fn tablet_label_escapes() { + let mut input = Parser::new(); + + // a label is a quoted literal like any other, so it escapes the same way + input.initialize(r#"{ ["say \"hi\"" = 1] }"#); + assert_eq!( + input.read_code_block(), + Ok(vec![Expression::Tablet( + vec![Pair { + label: vec![ + Piece::Text("say "), + Piece::Escaped('"'), + Piece::Text("hi"), + Piece::Escaped('"'), + ], + value: Expression::Number(Numeric::Integral(1), Span::default()) + }], + Span::default() + )]) + ); + + // and interpolates the same way + input.initialize(r#"{ ["order { n }" = 2] }"#); + assert_eq!( + input.read_code_block(), + Ok(vec![Expression::Tablet( + vec![Pair { + label: vec![ + Piece::Text("order "), + Piece::Interpolation(Expression::Variable( + Identifier::new("n"), + Span::default() + )), + ], + value: Expression::Number(Numeric::Integral(2), Span::default()) + }], + Span::default() + )]) + ); +} + +#[test] +fn string_escapes_rejected() { + let mut input = Parser::new(); + + // escaping is strict, so content needing backslashes of its own has to + // be written as a multiline instead + input.initialize(r#"{ "grep '^\s*\d+'" }"#); + match input.read_code_block() { + Err(ParsingError::InvalidEscape(_)) => {} + other => panic!("Expected InvalidEscape, got: {:?}", other), + } + + // a backslash with nothing following it inside the literal + input.initialize(r#"{ "oops \" }"#); + assert!( + input + .read_code_block() + .is_err() + ); +} + +#[test] +fn string_escapes_do_not_expose_structure() { + let mut input = Parser::new(); + + // the escaped quotes must not end the literal as far as the line + // scanner is concerned, otherwise the ordinal inside it reads as the + // start of a step + input.initialize("probe :\n 1. { exec(\"say \\\"2. not a step\\\" now\") }\n"); + let result = input.read_procedure(); + assert!(result.is_ok(), "expected a procedure, got {:?}", result); + + let procedure = result.unwrap(); + let steps = procedure + .elements + .len(); + assert_eq!(steps, 1, "the ordinal inside the string became a step"); +} + #[test] fn taking_until() { let mut input = Parser::new(); @@ -1494,8 +1662,10 @@ echo "Done"```) }"#, Function { target: Identifier::new("exec"), parameters: vec![Expression::Multiline( - Some("bash"), - vec!["ls -l", "echo \"Done\""], + Multiline { + language: Some("bash"), + lines: vec!["ls -l", "echo \"Done\""], + }, Span::default() )] }, @@ -1690,8 +1860,10 @@ ls -la, please Function { target: Identifier::new("exec"), parameters: vec![Expression::Multiline( - Some("bash"), - vec!["ls -la, please"], + Multiline { + language: Some("bash"), + lines: vec!["ls -la, please"], + }, Span::default() )] }, @@ -1714,8 +1886,10 @@ echo "hello, world" target: Identifier::new("combine"), parameters: vec![ Expression::Multiline( - Some("bash"), - vec!["echo \"hello, world\""], + Multiline { + language: Some("bash"), + lines: vec!["echo \"hello, world\""], + }, Span::default() ), Expression::String(vec![Piece::Text("second, arg")], Span::default()) @@ -1726,6 +1900,83 @@ echo "hello, world" ); } +#[test] +fn multiline_indent_tabs_expand_to_spaces() { + let mut input = Parser::new(); + + // The parser keeps each line exactly as written, tabs and all, so the + // lines stay borrowed from the source. + input.initialize( + "{ exec(```json\n {\n \t\t\t\"a\": 1,\n \t\t},\n ```) }", + ); + assert_eq!( + input.read_code_block(), + Ok(vec![Expression::Execution( + Function { + target: Identifier::new("exec"), + parameters: vec![Expression::Multiline( + Multiline { + language: Some("json"), + lines: vec![" {", " \t\t\t\"a\": 1,", " \t\t},"], + }, + Span::default() + )] + }, + Span::default() + )]) + ); + + // Reading them back is what interprets the indentation: a tab advances + // to the next four column stop, so the brace lines sit a level outside + // the key rather than, as counting bytes would have it, a level inside. + let multiline = Multiline { + language: Some("json"), + lines: vec![" {", " \t\t\t\"a\": 1,", " \t\t},"], + }; + assert_eq!( + multiline + .lines() + .collect::>(), + vec!["{", " \"a\": 1,", "},"] + ); +} + +#[test] +fn multiline_dedent_governed_by_least_indented_line() { + let mut input = Parser::new(); + + // A line further left than the one above it is ordinary in real content + // — the closing brace of a JSON object sits outside its keys. The block + // is dedented by the least indented line so that nothing is cut off. + input.initialize( + r#"{ exec(```json + { + "src": ["a"], + } + ```) }"#, + ); + assert_eq!( + input.read_code_block(), + Ok(vec![Expression::Execution( + Function { + target: Identifier::new("exec"), + parameters: vec![Expression::Multiline( + Multiline { + language: Some("json"), + lines: vec![ + " {", + " \"src\": [\"a\"],", + " }" + ], + }, + Span::default() + )] + }, + Span::default() + )]) + ); +} + #[test] fn multiline() { let mut input = Parser::new(); @@ -1747,15 +1998,17 @@ fn multiline() { Function { target: Identifier::new("exec"), parameters: vec![Expression::Multiline( - Some("bash"), - vec![ - "./stuff", - "", - "if [ true ]", - "then", - " ./other args", - "fi" - ], + Multiline { + language: Some("bash"), + lines: vec![ + " ./stuff", + "", + " if [ true ]", + " then", + " ./other args", + " fi" + ], + }, Span::default() )] }, @@ -1776,8 +2029,10 @@ echo "Done"```) }"#, Function { target: Identifier::new("exec"), parameters: vec![Expression::Multiline( - None, - vec!["ls -l", "echo \"Done\""], + Multiline { + language: None, + lines: vec!["ls -l", "echo \"Done\""], + }, Span::default() )] }, @@ -1802,15 +2057,17 @@ echo "Ending"```) }"#, Function { target: Identifier::new("exec"), parameters: vec![Expression::Multiline( - Some("shell"), - vec![ - "echo \"Starting\"", - "", - "echo \"Middle section\"", - "", - "", - "echo \"Ending\"" - ], + Multiline { + language: Some("shell"), + lines: vec![ + "echo \"Starting\"", + "", + "echo \"Middle section\"", + "", + "", + "echo \"Ending\"" + ], + }, Span::default() )] }, @@ -1837,15 +2094,17 @@ echo "Ending"```) }"#, Function { target: Identifier::new("exec"), parameters: vec![Expression::Multiline( - Some("python"), - vec![ - "def hello():", - " print(\"Hello\")", - " if True:", - " print(\"World\")", - "", - "hello()" - ], + Multiline { + language: Some("python"), + lines: vec![ + " def hello():", + " print(\"Hello\")", + " if True:", + " print(\"World\")", + "", + " hello()" + ], + }, Span::default() )] }, @@ -1866,8 +2125,10 @@ echo test Function { target: Identifier::new("exec"), parameters: vec![Expression::Multiline( - None, - vec!["echo test"], + Multiline { + language: None, + lines: vec!["echo test"], + }, Span::default() )] }, @@ -1892,15 +2153,17 @@ echo test Function { target: Identifier::new("exec"), parameters: vec![Expression::Multiline( - Some("yaml"), - vec![ - "name: test", - "items:", - " - item1", - " - item2", - "config:", - " enabled: true" - ], + Multiline { + language: Some("yaml"), + lines: vec![ + " name: test", + " items:", + " - item1", + " - item2", + " config:", + " enabled: true" + ], + }, Span::default() )] }, @@ -1920,7 +2183,7 @@ fn tablets() { result, Ok(vec![Expression::Tablet( vec![Pair { - label: "name", + label: vec![Piece::Text("name")], value: Expression::String(vec![Piece::Text("Johannes Grammerly")], Span::default()) }], Span::default() @@ -1940,14 +2203,14 @@ fn tablets() { Ok(vec![Expression::Tablet( vec![ Pair { - label: "name", + label: vec![Piece::Text("name")], value: Expression::String( vec![Piece::Text("Alice of Chains")], Span::default() ) }, Pair { - label: "age", + label: vec![Piece::Text("age")], value: Expression::String(vec![Piece::Text("29")], Span::default()) } ], @@ -1969,15 +2232,15 @@ fn tablets() { Ok(vec![Expression::Tablet( vec![ Pair { - label: "answer", + label: vec![Piece::Text("answer")], value: Expression::Number(Numeric::Integral(42), Span::default()) }, Pair { - label: "message", + label: vec![Piece::Text("message")], value: Expression::Variable(Identifier::new("msg"), Span::default()) }, Pair { - label: "timestamp", + label: vec![Piece::Text("timestamp")], value: Expression::Execution( Function { target: Identifier::new("now"), @@ -2022,14 +2285,14 @@ fn tablets() { Ok(vec![Expression::Tablet( vec![ Pair { - label: "context", + label: vec![Piece::Text("context")], value: Expression::String( vec![Piece::Text("Details about the thing")], Span::default() ) }, Pair { - label: "status", + label: vec![Piece::Text("status")], value: Expression::Variable(Identifier::new("active"), Span::default()) } ], @@ -2129,8 +2392,10 @@ ls -la, please result, Ok(vec![Expression::List( vec![Expression::Multiline( - Some("bash"), - vec!["ls -la, please"], + Multiline { + language: Some("bash"), + lines: vec!["ls -la, please"], + }, Span::default() )], Span::default() @@ -2217,11 +2482,11 @@ fn tablet_inline_commas() { Ok(vec![Expression::Tablet( vec![ Pair { - label: "answer", + label: vec![Piece::Text("answer")], value: Expression::Number(Numeric::Integral(42), Span::default()) }, Pair { - label: "truth", + label: vec![Piece::Text("truth")], value: Expression::String(vec![Piece::Text("yes")], Span::default()) } ], diff --git a/src/parsing/parser.rs b/src/parsing/parser.rs index 7f73d8f5..a404ef26 100644 --- a/src/parsing/parser.rs +++ b/src/parsing/parser.rs @@ -65,6 +65,7 @@ pub enum ParsingError { InvalidQuantityUncertainty(Span), InvalidQuantityMagnitude(Span), InvalidQuantitySymbol(Span), + InvalidEscape(Span), // highest priority UnclosedInterpolation(Span), } @@ -105,6 +106,7 @@ impl ParsingError { | ParsingError::InvalidQuantityUncertainty(span) | ParsingError::InvalidQuantityMagnitude(span) | ParsingError::InvalidQuantitySymbol(span) + | ParsingError::InvalidEscape(span) | ParsingError::UnclosedInterpolation(span) => *span, ParsingError::Expected(span, _) | ParsingError::ExpectedMatchingChar(span, _, _, _) @@ -460,7 +462,34 @@ impl<'i> Parser<'i> { let mut begun = false; let mut unterminated = false; - if start_char == end_char { + if start_char == '"' { + // A double quoted literal is what Literals already knows how to + // scan: an escaped quote does not close it, and it cannot carry + // across a line ending. + let mut literals = Literals::new(); + + for (i, c) in self + .source + .char_indices() + { + if literals.opaque(&self.source[i..]) { + continue; + } + + if !begun { + if c == start_char { + begun = true; + } + } else { + // the literal is over, either at the closing quote or + // because the line ended without one + if c == end_char { + l = i + 1; // add end character + } + break; + } + } + } else if start_char == end_char { // Simple case: same character for start and end (like X...X) for (i, c) in self .source @@ -715,7 +744,23 @@ impl<'i> Parser<'i> { } let indent = trimmed.as_ptr() as usize - base; let mut parser = outer.subparser(indent, trimmed); - results.push(function(&mut parser)?); + let result = function(&mut parser)?; + + // whatever the element did not consume is not part of it, and + // silently dropping it would hide the malformed input that left + // it behind + let leftover = parser + .source + .trim_ascii(); + if !leftover.is_empty() { + let offset = leftover.as_ptr() as usize - base; + return Err(ParsingError::Unrecognized(Span::new( + outer.offset + offset, + leftover.len(), + ))); + } + + results.push(result); outer .problems .extend(parser.problems); @@ -1654,7 +1699,7 @@ impl<'i> Parser<'i> { } else if content.starts_with('(') { self.read_tuple_literal() } else if content.starts_with("```") { - let (lang, lines) = self + let (language, lines) = self .take_block_delimited("```", |inner| inner.parse_multiline_content()) .map_err(|err| match err { ParsingError::Expected(span, "the corresponding end delimiter") => { @@ -1663,7 +1708,13 @@ impl<'i> Parser<'i> { _ => err, })?; let span = self.span_since(start); - Ok(Expression::Multiline(lang, lines, span)) + Ok(Expression::Multiline( + Multiline { + language, + lines, + }, + span, + )) } else if is_numeric(content) { let numeric = self.read_numeric()?; let span = self.span_since(start); @@ -1933,8 +1984,9 @@ impl<'i> Parser<'i> { outer.take_elements(true, |inner| { if is_pair(inner.source) { let pair_start = inner.offset; - let label = - inner.take_block_chars("a label", '"', '"', |label| Ok(label.source))?; + let label = inner.take_block_chars("a label", '"', '"', |label| { + label.parse_string_pieces(label.source) + })?; inner.trim_whitespace(); inner.advance(1); // consume '=' (is_pair guarantees it) inner.trim_whitespace(); @@ -1998,71 +2050,111 @@ impl<'i> Parser<'i> { Ok(Expression::Tuple(elements, span)) } + /// The text a backslash escape at `i` stands for. Escaping is strict so + /// that a sequence we don't recognize is a mistake rather than content; + /// anything needing backslashes of its own goes in a multiline instead. + fn escaped(&self, bytes: &[u8], i: usize) -> Result { + match bytes.get(i + 1) { + Some(b'\\') => Ok('\\'), + Some(b'"') => Ok('"'), + Some(b'n') => Ok('\n'), + Some(b'r') => Ok('\r'), + Some(b't') => Ok('\t'), + Some(b'{') => Ok('{'), + Some(b'}') => Ok('}'), + _ => { + let width = if i + 1 < bytes.len() { 2 } else { 1 }; + Err(ParsingError::InvalidEscape(Span::new( + self.offset + i, + width, + ))) + } + } + } + + /// Split a string literal's raw content into runs of text and the + /// interpolations between them, decoding escapes as we go. Decoding and + /// splitting are one pass because an escaped brace must not be mistaken + /// for the start of an interpolation. Each escape yields a one character + /// piece, so the surrounding text stays borrowed from the source. fn parse_string_pieces(&mut self, raw: &'i str) -> Result>, ParsingError> { - // Quick check: if no braces, just return a single text piece - if !raw.contains('{') { + // Quick check: nothing to decode and nothing to interpolate + if !raw.contains(['{', '\\']) { return Ok(vec![Piece::Text(raw)]); } + let bytes = raw.as_bytes(); let mut pieces = Vec::new(); - let mut current_pos = 0; + let mut run = 0; // start of the text being borrowed + let mut i = 0; - while current_pos < raw.len() { - // Look for the start of an interpolation - if let Some(brace_start) = raw[current_pos..].find('{') { - let absolute_brace_start = current_pos + brace_start; + while i < bytes.len() { + match bytes[i] { + b'\\' => { + if i > run { + pieces.push(Piece::Text(&raw[run..i])); + } - // Add text before the brace if any - if brace_start > 0 { - pieces.push(Piece::Text(&raw[current_pos..absolute_brace_start])); + pieces.push(Piece::Escaped(self.escaped(bytes, i)?)); + i += 2; + run = i; } - - // Find the matching closing brace - let mut brace_depth = 0; - let mut brace_end = None; - - for (i, c) in raw[absolute_brace_start..].char_indices() { - if c == '{' { - brace_depth += 1; - } else if c == '}' { - brace_depth -= 1; - if brace_depth == 0 { - brace_end = Some(absolute_brace_start + i); - break; - } + b'{' => { + if i > run { + pieces.push(Piece::Text(&raw[run..i])); } + + let end = self.find_interpolation_end(raw, i)?; + let content = &raw[i + 1..end]; + + let mut parser = self.subparser(i + 1, content); + let expression = parser.read_expression()?; + pieces.push(Piece::Interpolation(expression)); + + i = end + 1; + run = i; } + _ => i += 1, + } + } - match brace_end { - Some(end_pos) => { - // Extract the content between braces - let expr_content = &raw[absolute_brace_start + 1..end_pos]; + if run < raw.len() { + pieces.push(Piece::Text(&raw[run..])); + } - // Parse the expression using existing machinery - let mut parser = self.subparser(absolute_brace_start + 1, expr_content); - let expression = parser.read_expression()?; - pieces.push(Piece::Interpolation(expression)); + Ok(pieces) + } - current_pos = end_pos + 1; - } - None => { - // Unmatched brace - point to the opening brace position - return Err(ParsingError::UnclosedInterpolation(Span::new( - self.offset + absolute_brace_start, - 0, - ))); - } + /// Locate the '}' matching the '{' at `start`, counting nesting and + /// stepping over escapes so that an escaped brace neither opens nor + /// closes an interpolation. + fn find_interpolation_end(&self, raw: &str, start: usize) -> Result { + let bytes = raw.as_bytes(); + let mut depth = 0; + let mut i = start; + + while i < bytes.len() { + match bytes[i] { + b'\\' => { + i += 2; + continue; } - } else { - // No more braces - add the rest as text - if current_pos < raw.len() { - pieces.push(Piece::Text(&raw[current_pos..])); + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + return Ok(i); + } } - break; + _ => {} } + i += 1; } - Ok(pieces) + Err(ParsingError::UnclosedInterpolation(Span::new( + self.offset + start, + 0, + ))) } /// Consume an identifier. As with the other smaller read methods, we do a @@ -2770,7 +2862,7 @@ impl<'i> Parser<'i> { } fn parse_multiline_content(&mut self) -> Result<(Option<&'i str>, Vec<&'i str>), ParsingError> { - let mut lines: Vec<&str> = self + let mut lines: Vec<&'i str> = self .source .lines() .collect(); @@ -2784,42 +2876,19 @@ impl<'i> Parser<'i> { let lang = if !first.is_empty() { Some(first) } else { None }; lines.remove(0); - let second = lines[0]; - - // We let the indentation of the first line govern the rest of the block - let indent = second.len() - - second - .trim_ascii_start() - .len(); - - // Trim consistent leading whitespace while preserving internal indentation - let mut result = Vec::with_capacity(lines.len()); - - for line in lines { - // the final line with ``` will be likely shorter, irrespective of - // anything else going on. - let i = indent.min(line.len()); - - // now grab the text after the designated indent point. We check - // to make sure there's nothing before that point, otherwise we - // would have truncated the user's text. That's not allowed! - let (before, after) = line.split_at(i); - if !before - .trim_ascii() - .is_empty() - { - return Err(ParsingError::InvalidMultiline(Span::new(self.offset, 0))); - } - - result.push(after) - } - - // Remove trailing empty line if it's just from the closing ``` delimiter - if !result.is_empty() && result[result.len() - 1].is_empty() { - result.pop(); + // Drop the trailing line if it was just the indentation the closing + // ``` delimiter was sitting on + if lines + .last() + .is_some_and(|line| { + line.trim_ascii() + .is_empty() + }) + { + lines.pop(); } - Ok((lang, result)) + Ok((lang, lines)) } /// Consume parameters to an invocation or function: a parenthesised, @@ -3550,6 +3619,7 @@ enum Within { struct Literals { within: Within, delimiter: u8, + escaped: bool, } impl Literals { @@ -3557,6 +3627,7 @@ impl Literals { Literals { within: Within::Text, delimiter: 0, + escaped: false, } } @@ -3593,15 +3664,28 @@ impl Literals { true } else if rest.starts_with('"') { self.within = Within::String; + self.escaped = false; false } else { false } } // a string is closed by the next quote, or by the line ending; a - // fence is how text is carried across lines + // fence is how text is carried across lines. The line ending is + // tested first so that a trailing backslash cannot carry the + // string into the next line. Within::String => { - if rest.starts_with('"') || rest.starts_with('\n') { + if rest.starts_with('\n') { + self.within = Within::Text; + self.escaped = false; + false + } else if self.escaped { + self.escaped = false; + true + } else if rest.starts_with('\\') { + self.escaped = true; + true + } else if rest.starts_with('"') { self.within = Within::Text; false } else { @@ -3773,15 +3857,31 @@ fn is_string_literal(content: &str) -> bool { /// then be followed by an expression). fn is_pair(content: &str) -> bool { let content = content.trim_ascii_start(); - let Some(rest) = content.strip_prefix('"') else { + if !content.starts_with('"') { return false; - }; - match rest.split_once('"') { - Some((_label, after)) => after - .trim_ascii_start() - .starts_with('='), - None => false, } + + // the label is a quoted literal, so an escaped quote is content rather + // than the end of it + let mut literals = Literals::new(); + let mut begun = false; + + for (i, c) in content.char_indices() { + if literals.opaque(&content[i..]) { + continue; + } + + if !begun { + begun = true; + } else { + return c == '"' + && content[i + 1..] + .trim_ascii_start() + .starts_with('='); + } + } + + false } /// Detect the empty tablet literal `[=]`. A bare `[]` is the empty list; the diff --git a/src/problem/messages.rs b/src/problem/messages.rs index c2e08de8..a9f56bdd 100644 --- a/src/problem/messages.rs +++ b/src/problem/messages.rs @@ -63,6 +63,20 @@ enclose those names in parenthesis. For example: .to_string(), ) } + ParsingError::InvalidEscape(_) => ( + "Invalid escape sequence".to_string(), + r#" +Within a string literal a backslash begins an escape sequence. The known +escape sequences are '\"', '\n', '\r', '\t', '\{', and '\}'. Write '\\' to get +a single actual backslash character. + +If you need to write something with extensive use of backslashes such as a +regular expression you might prefer a multiline string delimited by ```, where +no special interpretation escape or interpolation characters takes place. + "# + .trim_ascii() + .to_string(), + ), ParsingError::UnclosedInterpolation(_) => ( "Unclosed string interpolation".to_string(), r#" diff --git a/src/program/types.rs b/src/program/types.rs index 9d0125f9..07db1c94 100644 --- a/src/program/types.rs +++ b/src/program/types.rs @@ -124,7 +124,7 @@ pub enum Operation<'i> { Number(language::Numeric<'i>, Span), String(Vec>, Span), Response(&'i str, Span), - Multiline(Option<&'i str>, Vec<&'i str>, Span), + Verbatim(&'i language::Multiline<'i>, Span), Tablet(Vec>, Span), List(Vec>, Span), Tuple(Vec>, Span), @@ -189,7 +189,7 @@ impl<'i> Operation<'i> { | Operation::Number(_, span) | Operation::String(_, span) | Operation::Response(_, span) - | Operation::Multiline(_, _, span) + | Operation::Verbatim(_, span) | Operation::Tablet(_, span) | Operation::List(_, span) | Operation::Tuple(_, span) @@ -277,12 +277,13 @@ pub enum ExecutableRef<'i> { #[derive(Debug, Eq, PartialEq)] pub enum Fragment<'i> { Text(&'i str), + Escaped(char), Interpolation(Operation<'i>), } /// An entry in a tablet: a label paired with a value-producing operation. #[derive(Debug, Eq, PartialEq)] pub struct Entry<'i> { - pub label: &'i str, + pub label: Vec>, pub value: Operation<'i>, } diff --git a/src/resolution/resolver.rs b/src/resolution/resolver.rs index ea579706..f09a1198 100644 --- a/src/resolution/resolver.rs +++ b/src/resolution/resolver.rs @@ -153,13 +153,19 @@ fn resolve_operation<'i>( } Operation::Tablet(entries, _) => { for entry in entries { + // a label is a quoted literal, so it can interpolate too + for fragment in &mut entry.label { + if let Fragment::Interpolation(op) = fragment { + resolve_operation(op, known, arities, problems); + } + } resolve_operation(&mut entry.value, known, arities, problems); } } Operation::Variable(_, _) | Operation::Number(_, _) | Operation::Response(_, _) - | Operation::Multiline(_, _, _) + | Operation::Verbatim(_, _) | Operation::Prose(_, _) | Operation::Hole(_) | Operation::Unit(_) => {} @@ -229,13 +235,19 @@ fn gather_iterated<'i>(op: &Operation<'i>, iterated: &mut HashSet<&'i str>) { } Operation::Tablet(entries, _) => { for entry in entries { + // a label is a quoted literal, so it can interpolate too + for fragment in &entry.label { + if let Fragment::Interpolation(op) = fragment { + gather_iterated(op, iterated); + } + } gather_iterated(&entry.value, iterated); } } Operation::Variable(_, _) | Operation::Number(_, _) | Operation::Response(_, _) - | Operation::Multiline(_, _, _) + | Operation::Verbatim(_, _) | Operation::Prose(_, _) | Operation::Hole(_) | Operation::Unit(_) => {} @@ -300,13 +312,19 @@ fn mark_iterated<'i>(op: &mut Operation<'i>, iterated: &HashSet<&str>) { } Operation::Tablet(entries, _) => { for entry in entries { + // a label is a quoted literal, so it can interpolate too + for fragment in &mut entry.label { + if let Fragment::Interpolation(op) = fragment { + mark_iterated(op, iterated); + } + } mark_iterated(&mut entry.value, iterated); } } Operation::Variable(_, _) | Operation::Number(_, _) | Operation::Response(_, _) - | Operation::Multiline(_, _, _) + | Operation::Verbatim(_, _) | Operation::Prose(_, _) | Operation::Hole(_) | Operation::Unit(_) => {} @@ -402,12 +420,18 @@ fn check_scope<'i>( } Operation::Tablet(entries, _) => { for entry in entries { + // a label is a quoted literal, so it can interpolate too + for fragment in &entry.label { + if let Fragment::Interpolation(op) = fragment { + check_scope(op, scope, problems); + } + } check_scope(&entry.value, scope, problems); } } Operation::Number(_, _) | Operation::Response(_, _) - | Operation::Multiline(_, _, _) + | Operation::Verbatim(_, _) | Operation::Prose(_, _) | Operation::Hole(_) | Operation::Unit(_) => {} @@ -471,13 +495,19 @@ fn check_costs<'i>(op: &Operation<'i>, problems: &mut Vec>) } Operation::Tablet(entries, _) => { for entry in entries { + // a label is a quoted literal, so it can interpolate too + for fragment in &entry.label { + if let Fragment::Interpolation(op) = fragment { + check_costs(op, problems); + } + } check_costs(&entry.value, problems); } } Operation::Variable(_, _) | Operation::Number(_, _) | Operation::Response(_, _) - | Operation::Multiline(_, _, _) + | Operation::Verbatim(_, _) | Operation::Prose(_, _) | Operation::Hole(_) | Operation::Unit(_) => {} @@ -490,7 +520,7 @@ fn literal_not_a_quantity(op: &Operation) -> bool { match op { Operation::String(_, _) | Operation::Response(_, _) - | Operation::Multiline(_, _, _) + | Operation::Verbatim(_, _) | Operation::Tablet(_, _) | Operation::List(_, _) | Operation::Tuple(_, _) diff --git a/src/runner/checks/evaluator.rs b/src/runner/checks/evaluator.rs index 86d9efb8..8c2f2233 100644 --- a/src/runner/checks/evaluator.rs +++ b/src/runner/checks/evaluator.rs @@ -1,4 +1,4 @@ -use crate::language::{Identifier, Numeric as LangNumeric, Span}; +use crate::language::{Identifier, Multiline, Numeric as LangNumeric, Span}; use crate::program::{Entry, Executable, ExecutableRef, Fragment, Operation}; use crate::runner::context::Context; use crate::runner::evaluator::{ @@ -83,7 +83,11 @@ fn string_interpolation() { fn multiline_joins_with_newlines() { let library = Library::core(); let context = Context::native(false); - let op = Operation::Multiline(None, vec!["foo", "bar", "baz"], Span::default()); + let multiline = Multiline { + language: None, + lines: vec!["foo", "bar", "baz"], + }; + let op = Operation::Verbatim(&multiline, Span::default()); let mut env = Environment::new(); let v = evaluate(&library, &context, &mut env, &op).expect("evaluated"); assert_eq!(v, value::Value::Literali("foo\nbar\nbaz".to_string())); @@ -96,11 +100,11 @@ fn tablet_entries_evaluate() { let op = Operation::Tablet( vec![ Entry { - label: "name", + label: vec![Fragment::Text("name")], value: Operation::String(vec![Fragment::Text("Kowalski")], Span::default()), }, Entry { - label: "count", + label: vec![Fragment::Text("count")], value: Operation::Number(LangNumeric::Integral(7), Span::default()), }, ], diff --git a/src/runner/evaluator.rs b/src/runner/evaluator.rs index f7dd3ed6..62108e43 100644 --- a/src/runner/evaluator.rs +++ b/src/runner/evaluator.rs @@ -127,6 +127,31 @@ fn kind(value: &Value) -> &'static str { /// /// Fails with `UnboundVariable` etc if the operation cannot be resolved; /// specifically at this point values of variables need to be known from the +/// Concatenate the fragments of a quoted literal, evaluating any +/// interpolation as it is reached. Shared by string values and the labels of +/// tablet entries, which parse the same way. +fn evaluate_fragments<'i>( + library: &Library, + context: &Context, + env: &mut Environment, + fragments: &[Fragment<'i>], +) -> Result { + let mut text = String::new(); + + for fragment in fragments { + match fragment { + Fragment::Text(t) => text.push_str(t), + Fragment::Escaped(c) => text.push(*c), + Fragment::Interpolation(inner) => match evaluate(library, context, env, inner)? { + Value::Literali(s) => text.push_str(&s), + other => text.push_str(&other.to_string()), + }, + } + } + + Ok(text) +} + /// `Environment` otherwise the `Operation` can't be evaluated. /// /// A resolved `Execute` dispatches through the passed in `Library` to its @@ -150,32 +175,15 @@ pub fn evaluate<'i>( }), Operation::Number(n, _) => Ok(Value::Quanticle(Numeric::from(n))), Operation::Response(value, _) => Ok(Value::Enumerati(value.to_string())), - Operation::String(fragments, _) => { - let mut text = String::new(); - for fragment in fragments { - match fragment { - Fragment::Text(t) => text.push_str(t), - Fragment::Interpolation(inner) => { - match evaluate(library, context, env, inner)? { - Value::Literali(s) => text.push_str(&s), - other => text.push_str(&other.to_string()), - } - } - } - } - Ok(Value::Literali(text)) - } - Operation::Multiline(_, lines, _) => Ok(Value::Literali(lines.join("\n"))), + Operation::String(fragments, _) => Ok(Value::Literali(evaluate_fragments( + library, context, env, fragments, + )?)), + Operation::Verbatim(multiline, _) => Ok(Value::Literali(multiline.content())), Operation::Tablet(entries, _) => { let mut pairs = Vec::with_capacity(entries.len()); for entry in entries { let v = evaluate(library, context, env, &entry.value)?; - pairs.push(( - entry - .label - .to_string(), - v, - )); + pairs.push((evaluate_fragments(library, context, env, &entry.label)?, v)); } Ok(Value::Tabularum(pairs)) } diff --git a/src/runner/runner.rs b/src/runner/runner.rs index f599849d..0587620c 100644 --- a/src/runner/runner.rs +++ b/src/runner/runner.rs @@ -517,7 +517,7 @@ impl<'i, D: Driver> Runner<'i, D> { | Operation::Number(_, _) | Operation::Response(_, _) | Operation::String(_, _) - | Operation::Multiline(_, _, _) + | Operation::Verbatim(_, _) | Operation::Tablet(_, _) | Operation::List(_, _) | Operation::Tuple(_, _) diff --git a/src/translation/checks/translate.rs b/src/translation/checks/translate.rs index b3c2debd..f1ee4d75 100644 --- a/src/translation/checks/translate.rs +++ b/src/translation/checks/translate.rs @@ -908,8 +908,8 @@ run : panic!("expected Tablet, got {:?}", ops[0]); }; assert_eq!(entries.len(), 2); - assert_eq!(entries[0].label, "speed"); - assert_eq!(entries[1].label, "weight"); + assert_eq!(entries[0].label, vec![Fragment::Text("speed")]); + assert_eq!(entries[1].label, vec![Fragment::Text("weight")]); } #[test] @@ -1475,12 +1475,11 @@ run : let Operation::Execute(executable, _) = &ops[0] else { panic!("expected Execute"); }; - let Operation::Multiline(lang, lines, _) = &executable.arguments[0] else { + let Operation::Verbatim(multiline, _) = &executable.arguments[0] else { panic!("expected Multiline, got {:?}", executable.arguments[0]); }; - assert_eq!(*lang, Some("bash")); - assert_eq!(lines.len(), 1); - assert_eq!(lines[0], "ip addr"); + assert_eq!(multiline.language, Some("bash")); + assert_eq!(multiline.lines, vec![" ip addr"]); } #[test] diff --git a/src/translation/translator.rs b/src/translation/translator.rs index e5adc212..85d6e1d2 100644 --- a/src/translation/translator.rs +++ b/src/translation/translator.rs @@ -744,25 +744,31 @@ impl<'i> Translator<'i> { } } + /// The pieces of a quoted literal, shared by string values and the + /// labels of tablet entries since the two parse identically. + fn translate_pieces(&mut self, pieces: &'i [language::Piece<'i>]) -> Vec> { + pieces + .iter() + .map(|piece| match piece { + language::Piece::Text(text) => Fragment::Text(text), + language::Piece::Escaped(c) => Fragment::Escaped(*c), + language::Piece::Interpolation(expr) => { + Fragment::Interpolation(self.translate_expression(expr)) + } + }) + .collect() + } + fn translate_expression(&mut self, expression: &'i language::Expression<'i>) -> Operation<'i> { match expression { language::Expression::Variable(id, span) => Operation::Variable(*id, *span), language::Expression::Number(numeric, span) => Operation::Number(*numeric, *span), language::Expression::String(pieces, span) => { - let fragments = pieces - .iter() - .map(|piece| match piece { - language::Piece::Text(text) => Fragment::Text(text), - language::Piece::Interpolation(expr) => { - Fragment::Interpolation(self.translate_expression(expr)) - } - }) - .collect(); - Operation::String(fragments, *span) + Operation::String(self.translate_pieces(pieces), *span) } language::Expression::Response(value, span) => Operation::Response(value, *span), - language::Expression::Multiline(lang, lines, span) => { - Operation::Multiline(*lang, lines.clone(), *span) + language::Expression::Multiline(multiline, span) => { + Operation::Verbatim(multiline, *span) } language::Expression::Pair(pair, span) => { // A standalone labelled value widens to a single-entry @@ -770,7 +776,7 @@ impl<'i> Translator<'i> { // single-element list. Operation::Tablet( vec![Entry { - label: pair.label, + label: self.translate_pieces(&pair.label), value: self.translate_expression(&pair.value), }], *span, @@ -780,7 +786,7 @@ impl<'i> Translator<'i> { let entries = pairs .iter() .map(|pair| Entry { - label: pair.label, + label: self.translate_pieces(&pair.label), value: self.translate_expression(&pair.value), }) .collect(); diff --git a/tests/formatting/formatter.rs b/tests/formatting/formatter.rs index 287a2387..7b1e564b 100644 --- a/tests/formatting/formatter.rs +++ b/tests/formatting/formatter.rs @@ -214,8 +214,10 @@ win_le_tour : Bicycle -> YellowJersey Function { target: Identifier::new("exec"), parameters: vec![Expression::Multiline( - Some("bash"), - vec!["rm -rf /"], + Multiline { + language: Some("bash"), + lines: vec!["rm -rf /"], + }, Span::default(), )], }, @@ -271,8 +273,10 @@ vibe_coding : Function { target: Identifier::new("exec"), parameters: vec![Expression::Multiline( - Some("bash"), - vec!["rm -rf /"], + Multiline { + language: Some("bash"), + lines: vec!["rm -rf /"], + }, Span::default(), )], }, @@ -339,7 +343,7 @@ We must take action! expressions: vec![Expression::Tablet( vec![ Pair { - label: "timestamp", + label: vec![Piece::Text("timestamp")], value: Expression::Execution( Function { target: Identifier::new("now"), @@ -349,7 +353,7 @@ We must take action! ), }, Pair { - label: "message", + label: vec![Piece::Text("message")], value: Expression::Variable( Identifier::new("msg"), Span::default(), @@ -394,6 +398,60 @@ Record everything, with timestamps. ); } + #[test] + fn multiline_content_written_verbatim() { + // A multiline is raw content. Its internal indentation and any run + // of whitespace inside a line have to survive being written back, + // otherwise formatting a document quietly rewrites the JSON or the + // shell script the author put there. + let document = Document { + source: None, + header: None, + body: Some(Technique::Procedures(vec![Procedure { + name: Identifier::new("deploy"), + parameters: None, + signature: None, + elements: vec![Element::CodeBlock( + vec![Expression::Execution( + Function { + target: Identifier::new("exec"), + parameters: vec![Expression::Multiline( + Multiline { + language: Some("json"), + lines: vec![" {", "\"ip\": [\"tcp:22\"],", " }"], + }, + Span::default(), + )], + }, + Span::default(), + )], + vec![], + Span::default(), + )], + span: Span::default(), + }])), + }; + + let result = format_with_renderer(&document, 78); + assert_eq!( + combine(result), + trim( + r#" +deploy : +{ + exec( + ```json + { + "ip": ["tcp:22"], + } + ``` + ) +} + "# + ) + ); + } + #[test] fn nested_scopes() { let document = Document {