diff --git a/.gitignore b/.gitignore index 60416024b..d7ea26e83 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ target/ .direnv/ result +# mq-test snapshot store (assert_snapshot); __snapshots__/ itself IS checked in +.mq-test-store/ + **/*.rs.bk *.pdb diff --git a/Cargo.lock b/Cargo.lock index 00dc46ad9..49dd92be0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2849,6 +2849,7 @@ dependencies = [ "rstest", "rustc-hash", "serde_json", + "similar 3.1.2", ] [[package]] diff --git a/crates/mq-lang/modules/__snapshots__/json_test/json_stringify.snap b/crates/mq-lang/modules/__snapshots__/json_test/json_stringify.snap new file mode 100644 index 000000000..fc6b6cb1c --- /dev/null +++ b/crates/mq-lang/modules/__snapshots__/json_test/json_stringify.snap @@ -0,0 +1 @@ +{"users": [{"name": "Alice", "id": 1, "email": "alice@example.com", "roles": ["admin", "user"]}, {"name": "Bob", "id": 2, "email": "bob@example.com", "roles": ["user"]}, {"name": "Charlie", "id": 3, "email": "charlie@example.com", "roles": ["editor", "user"]}], "meta": {"count": 3, "generated_at": "2024-06-01T12:00:00Z"}} \ No newline at end of file diff --git a/crates/mq-lang/modules/__snapshots__/json_test/json_to_markdown_table.snap b/crates/mq-lang/modules/__snapshots__/json_test/json_to_markdown_table.snap new file mode 100644 index 000000000..ab975f6fe --- /dev/null +++ b/crates/mq-lang/modules/__snapshots__/json_test/json_to_markdown_table.snap @@ -0,0 +1,5 @@ +| name | id | email | roles | +| --- | --- | --- | --- | +| Alice | 1 | alice@example.com | ["admin", "user"] | +| Bob | 2 | bob@example.com | ["user"] | +| Charlie | 3 | charlie@example.com | ["editor", "user"] | \ No newline at end of file diff --git a/crates/mq-lang/modules/__snapshots__/section_test/section_content.snap b/crates/mq-lang/modules/__snapshots__/section_test/section_content.snap new file mode 100644 index 000000000..bae1dce58 --- /dev/null +++ b/crates/mq-lang/modules/__snapshots__/section_test/section_content.snap @@ -0,0 +1,4 @@ +This is the introduction section. + +## Advanced Topics +This section covers advanced topics. diff --git a/crates/mq-lang/modules/__snapshots__/section_test/section_flatten.snap b/crates/mq-lang/modules/__snapshots__/section_test/section_flatten.snap new file mode 100644 index 000000000..871f4340d --- /dev/null +++ b/crates/mq-lang/modules/__snapshots__/section_test/section_flatten.snap @@ -0,0 +1,8 @@ +# Introduction +This is the introduction section. + +## Advanced Topics +This section covers advanced topics. + +# Conclusion +This is the conclusion section. diff --git a/crates/mq-lang/modules/__snapshots__/table_test/table_to_markdown.snap b/crates/mq-lang/modules/__snapshots__/table_test/table_to_markdown.snap new file mode 100644 index 000000000..1af0c7cce --- /dev/null +++ b/crates/mq-lang/modules/__snapshots__/table_test/table_to_markdown.snap @@ -0,0 +1,4 @@ +| Name | Age | City | +| ----- | --- | ----- | +| Alice | 30 | Tokyo | +| Bob | 25 | Osaka | diff --git a/crates/mq-lang/modules/__snapshots__/toon_test/toon_stringify.snap b/crates/mq-lang/modules/__snapshots__/toon_test/toon_stringify.snap new file mode 100644 index 000000000..06e806dc0 --- /dev/null +++ b/crates/mq-lang/modules/__snapshots__/toon_test/toon_stringify.snap @@ -0,0 +1,16 @@ +context: + task: Our favorite hikes together + location: Boulder + season: spring_2025 +friends[3]: ana,luis,sam +hikes[3]{name,id,distanceKm,elevationGain,companion,wasSunny}: + Blue Lake Trail,1,7.5,320,ana,true + Ridge Overlook,2,9.2,540,luis,false + Wildflower Loop,3,5.1,180,sam,true +items[3]: + - 1 + - a: 1 + - text +items2[2]{name,price}: + Laptop,999 + Mouse,29 \ No newline at end of file diff --git a/crates/mq-lang/modules/__snapshots__/xml_test/xml_to_markdown_table.snap b/crates/mq-lang/modules/__snapshots__/xml_test/xml_to_markdown_table.snap new file mode 100644 index 000000000..1b11cc6ba --- /dev/null +++ b/crates/mq-lang/modules/__snapshots__/xml_test/xml_to_markdown_table.snap @@ -0,0 +1,115 @@ +| Index | Tag | Attributes | Text | +| --- | --- | --- | --- | +| 0 | user | id=1 | | +| 1 | user | id=2 | | +| 2 | user | id=3 | | + +## Children of user: + +| Index | Tag | Attributes | Text | +| --- | --- | --- | --- | +| 0 | name | | Alice | +| 1 | email | | alice@example.com | +| 2 | roles | | | + +### Children of name: + +| Tag | Attributes | Text | Children | +| --- | --- | --- | --- | +| name | | Alice | 0 | + +### Children of email: + +| Tag | Attributes | Text | Children | +| --- | --- | --- | --- | +| email | | alice@example.com | 0 | + +### Children of roles: + +| Index | Tag | Attributes | Text | +| --- | --- | --- | --- | +| 0 | role | | admin | +| 1 | role | | user | + +#### Children of role: + +| Tag | Attributes | Text | Children | +| --- | --- | --- | --- | +| role | | admin | 0 | + +#### Children of role: + +| Tag | Attributes | Text | Children | +| --- | --- | --- | --- | +| role | | user | 0 | + +## Children of user: + +| Index | Tag | Attributes | Text | +| --- | --- | --- | --- | +| 0 | name | | Bob | +| 1 | email | | bob@example.com | +| 2 | roles | | | + +### Children of name: + +| Tag | Attributes | Text | Children | +| --- | --- | --- | --- | +| name | | Bob | 0 | + +### Children of email: + +| Tag | Attributes | Text | Children | +| --- | --- | --- | --- | +| email | | bob@example.com | 0 | + +### Children of roles: + +| Index | Tag | Attributes | Text | +| --- | --- | --- | --- | +| 0 | role | | user | + +#### Children of role: + +| Tag | Attributes | Text | Children | +| --- | --- | --- | --- | +| role | | user | 0 | + +## Children of user: + +| Index | Tag | Attributes | Text | +| --- | --- | --- | --- | +| 0 | name | | Charlie | +| 1 | email | | charlie@example.com | +| 2 | roles | | | + +### Children of name: + +| Tag | Attributes | Text | Children | +| --- | --- | --- | --- | +| name | | Charlie | 0 | + +### Children of email: + +| Tag | Attributes | Text | Children | +| --- | --- | --- | --- | +| email | | charlie@example.com | 0 | + +### Children of roles: + +| Index | Tag | Attributes | Text | +| --- | --- | --- | --- | +| 0 | role | | editor | +| 1 | role | | user | + +#### Children of role: + +| Tag | Attributes | Text | Children | +| --- | --- | --- | --- | +| role | | editor | 0 | + +#### Children of role: + +| Tag | Attributes | Text | Children | +| --- | --- | --- | --- | +| role | | user | 0 | \ No newline at end of file diff --git a/crates/mq-lang/modules/__snapshots__/yaml_test/yaml_to_json.snap b/crates/mq-lang/modules/__snapshots__/yaml_test/yaml_to_json.snap new file mode 100644 index 000000000..e7e43f94b --- /dev/null +++ b/crates/mq-lang/modules/__snapshots__/yaml_test/yaml_to_json.snap @@ -0,0 +1 @@ +{"array": ["item1", "item2", "item3"], "number": 42, "float": 3.14, "bool_true": true, "bool_false": false, "null_value": null, "object": {"key1": "value1", "key2": "value2"}, "nested": {"arr": ["a", "b"], "obj": {"subkey": "subval"}}} \ No newline at end of file diff --git a/crates/mq-lang/modules/__snapshots__/yaml_test/yaml_to_markdown_table.snap b/crates/mq-lang/modules/__snapshots__/yaml_test/yaml_to_markdown_table.snap new file mode 100644 index 000000000..3dd01a108 --- /dev/null +++ b/crates/mq-lang/modules/__snapshots__/yaml_test/yaml_to_markdown_table.snap @@ -0,0 +1,10 @@ +| Key | Value | +| --- | --- | +| array | ["item1", "item2", "item3"] | +| number | 42 | +| float | 3.14 | +| bool_true | true | +| bool_false | false | +| null_value | | +| object | {"key1": "value1", "key2": "value2"} | +| nested | {"arr": ["a", "b"], "obj": {"subkey": "subval"}} | \ No newline at end of file diff --git a/crates/mq-lang/modules/json_test.mq b/crates/mq-lang/modules/json_test.mq index f94e2a9f6..f3c43011a 100644 --- a/crates/mq-lang/modules/json_test.mq +++ b/crates/mq-lang/modules/json_test.mq @@ -12,11 +12,11 @@ end def test_json_stringify(): let result = json::json_stringify(json::json_parse(json_input)) - | assert_eq(result, "{\"users\": [{\"name\": \"Alice\", \"id\": 1, \"email\": \"alice@example.com\", \"roles\": [\"admin\", \"user\"]}, {\"name\": \"Bob\", \"id\": 2, \"email\": \"bob@example.com\", \"roles\": [\"user\"]}, {\"name\": \"Charlie\", \"id\": 3, \"email\": \"charlie@example.com\", \"roles\": [\"editor\", \"user\"]}], \"meta\": {\"count\": 3, \"generated_at\": \"2024-06-01T12:00:00Z\"}}") + | assert_snapshot("json_stringify", result) end def test_json_to_markdown_table(): let result = json::json_parse(json_input) | let result = json::json_to_markdown_table(result["users"]) - | assert_eq(result, "| name | id | email | roles |\n| --- | --- | --- | --- |\n| Alice | 1 | alice@example.com | [\"admin\", \"user\"] |\n| Bob | 2 | bob@example.com | [\"user\"] |\n| Charlie | 3 | charlie@example.com | [\"editor\", \"user\"] |") + | assert_snapshot("json_to_markdown_table", result) end diff --git a/crates/mq-lang/modules/section_test.mq b/crates/mq-lang/modules/section_test.mq index 407784fb1..d8989dee9 100644 --- a/crates/mq-lang/modules/section_test.mq +++ b/crates/mq-lang/modules/section_test.mq @@ -43,7 +43,7 @@ def test_section_content(): let sections = do test_sections | section::split(1); | let first_section = first(sections) | let content_nodes = section::content(first_section) - | assert(len(content_nodes) > 0) + | assert_snapshot("section_content", to_markdown_string(content_nodes)) end def test_section_level(): @@ -68,7 +68,7 @@ end def test_section_flatten(): let sections = do test_sections | section::split(1); | let flattened = section::collect(sections) - | assert(is_array(flattened)) + | assert_snapshot("section_flatten", to_markdown_string(flattened)) end def test_section_section_pattern(): @@ -106,7 +106,7 @@ end def test_section_bodies_pipe(): let body_list = do test_sections | section::section("Introduction") | section::bodies(); | assert_eq(len(body_list), 1) - | assert(is_array(first(body_list))) + | assert_eq(to_markdown_string(first(body_list)), "This is the introduction section.\n") end # @parametrize([[1, 2, 1], [2, 1, 2], [1..2, 3, 1]]) diff --git a/crates/mq-lang/modules/table_test.mq b/crates/mq-lang/modules/table_test.mq index a8112fafc..9678db670 100644 --- a/crates/mq-lang/modules/table_test.mq +++ b/crates/mq-lang/modules/table_test.mq @@ -82,8 +82,7 @@ end def test_table_to_markdown(): let t = first(table::tables(table_nodes)) | let md_result = table::to_markdown(t) - | assert(is_array(md_result)) - | assert(len(md_result) > 0) + | assert_snapshot("table_to_markdown", to_markdown_string(md_result)) end def test_table_map_rows(): diff --git a/crates/mq-lang/modules/toon_test.mq b/crates/mq-lang/modules/toon_test.mq index 20a8b0a3b..c0c3cb8c2 100644 --- a/crates/mq-lang/modules/toon_test.mq +++ b/crates/mq-lang/modules/toon_test.mq @@ -12,7 +12,7 @@ end def test_toon_stringify(): let result = toon::toon_stringify(toon::toon_parse(toon_input)) - | assert_eq(result, "context:\n task: Our favorite hikes together\n location: Boulder\n season: spring_2025\nfriends[3]: ana,luis,sam\nhikes[3]{name,id,distanceKm,elevationGain,companion,wasSunny}:\n Blue Lake Trail,1,7.5,320,ana,true\n Ridge Overlook,2,9.2,540,luis,false\n Wildflower Loop,3,5.1,180,sam,true\nitems[3]:\n - 1\n - a: 1\n - text\nitems2[2]{name,price}:\n Laptop,999\n Mouse,29") + | assert_snapshot("toon_stringify", result) end def test_toon_round_trip(): diff --git a/crates/mq-lang/modules/xml_test.mq b/crates/mq-lang/modules/xml_test.mq index c84f7eba0..eff8372d3 100644 --- a/crates/mq-lang/modules/xml_test.mq +++ b/crates/mq-lang/modules/xml_test.mq @@ -17,5 +17,5 @@ end def test_xml_to_markdown_table(): let result = xml::xml_to_markdown_table(xml::xml_parse(xml_input)) - | assert_eq(result, "| Index | Tag | Attributes | Text |\n| --- | --- | --- | --- |\n| 0 | user | id=1 | |\n| 1 | user | id=2 | |\n| 2 | user | id=3 | |\n\n## Children of user:\n\n| Index | Tag | Attributes | Text |\n| --- | --- | --- | --- |\n| 0 | name | | Alice |\n| 1 | email | | alice@example.com |\n| 2 | roles | | |\n\n### Children of name:\n\n| Tag | Attributes | Text | Children |\n| --- | --- | --- | --- |\n| name | | Alice | 0 |\n\n### Children of email:\n\n| Tag | Attributes | Text | Children |\n| --- | --- | --- | --- |\n| email | | alice@example.com | 0 |\n\n### Children of roles:\n\n| Index | Tag | Attributes | Text |\n| --- | --- | --- | --- |\n| 0 | role | | admin |\n| 1 | role | | user |\n\n#### Children of role:\n\n| Tag | Attributes | Text | Children |\n| --- | --- | --- | --- |\n| role | | admin | 0 |\n\n#### Children of role:\n\n| Tag | Attributes | Text | Children |\n| --- | --- | --- | --- |\n| role | | user | 0 |\n\n## Children of user:\n\n| Index | Tag | Attributes | Text |\n| --- | --- | --- | --- |\n| 0 | name | | Bob |\n| 1 | email | | bob@example.com |\n| 2 | roles | | |\n\n### Children of name:\n\n| Tag | Attributes | Text | Children |\n| --- | --- | --- | --- |\n| name | | Bob | 0 |\n\n### Children of email:\n\n| Tag | Attributes | Text | Children |\n| --- | --- | --- | --- |\n| email | | bob@example.com | 0 |\n\n### Children of roles:\n\n| Index | Tag | Attributes | Text |\n| --- | --- | --- | --- |\n| 0 | role | | user |\n\n#### Children of role:\n\n| Tag | Attributes | Text | Children |\n| --- | --- | --- | --- |\n| role | | user | 0 |\n\n## Children of user:\n\n| Index | Tag | Attributes | Text |\n| --- | --- | --- | --- |\n| 0 | name | | Charlie |\n| 1 | email | | charlie@example.com |\n| 2 | roles | | |\n\n### Children of name:\n\n| Tag | Attributes | Text | Children |\n| --- | --- | --- | --- |\n| name | | Charlie | 0 |\n\n### Children of email:\n\n| Tag | Attributes | Text | Children |\n| --- | --- | --- | --- |\n| email | | charlie@example.com | 0 |\n\n### Children of roles:\n\n| Index | Tag | Attributes | Text |\n| --- | --- | --- | --- |\n| 0 | role | | editor |\n| 1 | role | | user |\n\n#### Children of role:\n\n| Tag | Attributes | Text | Children |\n| --- | --- | --- | --- |\n| role | | editor | 0 |\n\n#### Children of role:\n\n| Tag | Attributes | Text | Children |\n| --- | --- | --- | --- |\n| role | | user | 0 |") + | assert_snapshot("xml_to_markdown_table", result) end diff --git a/crates/mq-lang/modules/yaml_test.mq b/crates/mq-lang/modules/yaml_test.mq index fcaacc1fb..15f6c4e65 100644 --- a/crates/mq-lang/modules/yaml_test.mq +++ b/crates/mq-lang/modules/yaml_test.mq @@ -17,12 +17,12 @@ end def test_yaml_to_json(): let result = yaml::yaml_to_json(yaml::yaml_parse(yaml_input)) - | assert_eq(result, "{\"array\": [\"item1\", \"item2\", \"item3\"], \"number\": 42, \"float\": 3.14, \"bool_true\": true, \"bool_false\": false, \"null_value\": null, \"object\": {\"key1\": \"value1\", \"key2\": \"value2\"}, \"nested\": {\"arr\": [\"a\", \"b\"], \"obj\": {\"subkey\": \"subval\"}}}") + | assert_snapshot("yaml_to_json", result) end def test_yaml_to_markdown_table(): let result = yaml::yaml_to_markdown_table(yaml::yaml_parse(yaml_input)) - | assert_eq(result, "| Key | Value |\n| --- | --- |\n| array | [\"item1\", \"item2\", \"item3\"] |\n| number | 42 |\n| float | 3.14 |\n| bool_true | true |\n| bool_false | false |\n| null_value | |\n| object | {\"key1\": \"value1\", \"key2\": \"value2\"} |\n| nested | {\"arr\": [\"a\", \"b\"], \"obj\": {\"subkey\": \"subval\"}} |") + | assert_snapshot("yaml_to_markdown_table", result) end def test_to_front_matter(): diff --git a/crates/mq-test/Cargo.toml b/crates/mq-test/Cargo.toml index d810aae63..48e1a0ec1 100644 --- a/crates/mq-test/Cargo.toml +++ b/crates/mq-test/Cargo.toml @@ -18,6 +18,7 @@ mq-lang = {workspace = true, features = ["cst", "debugger", "file-io", "http", " rayon = {workspace = true} rustc-hash = {workspace = true} serde_json = {workspace = true} +similar = {workspace = true} [dev-dependencies] rstest = {workspace = true} diff --git a/crates/mq-test/README.md b/crates/mq-test/README.md index 050fa39c3..de25d4c38 100644 --- a/crates/mq-test/README.md +++ b/crates/mq-test/README.md @@ -59,6 +59,9 @@ mq-test --tag smoke # Run test files in parallel once more than 4 files are discovered mq-test --parallel-threshold 4 + +# Accept the current output of every assert_snapshot(...) call as the new golden snapshot +mq-test --update-snapshots ``` ## Coverage @@ -201,29 +204,57 @@ function via a separate comment line placed right above it. merged safely across parallel files, and each file's report prints atomically so concurrent files' output never interleaves. -A failing test in one file no longer stops other files from running: every -discovered file always runs to completion, and `mq-test` exits non-zero if -any test in any file failed. +A failing test in one file no longer stops other files from running: every discovered file always runs to completion, and `mq-test` exits non-zero if any test in any file failed. ### Test Helpers Tests use the built-in `assert_eq` and related helpers from the `test` module: -| Function | Description | -| --------------------------- | ------------------------------ | -| `assert_eq(actual, expect)` | Fails if `actual != expect` | -| `assert(cond)` | Fails if `cond` is not `true` | -| `test_case(name, fn)` | Registers a named test case | -| `run_tests(cases)` | Runs all registered test cases | +| Function | Description | +| -------------------------------- | ------------------------------ | +| `assert_eq(actual, expect)` | Fails if `actual != expect` | +| `assert(cond)` | Fails if `cond` is not `true` | +| `assert_snapshot(name, actual)` | Fails if `actual` doesn't match the golden snapshot `name` | +| `test_case(name, fn)` | Registers a named test case | +| `run_tests(cases)` | Runs all registered test cases | The runner automatically generates a `run_tests(flatten([...]))` call from all discovered test functions — test files do not need to maintain a manual list. +### Snapshot Testing + +`assert_snapshot(name, actual)` compares `actual` against a golden file, for outputs too large to usefully inline in an `assert_eq` diff (e.g. a rendered document): + +```mq +include "test" +| + +def test_renders_the_full_report(): + assert_snapshot("report", render_report(data)) +end +``` + +The layout follows the `suite`/`ref`/`store` split from +[typst's test runner](https://github.com/typst/typst/tree/main/tests): the test file +itself plus whatever it feeds into `assert_snapshot` is the "suite" (input), and: + +- **ref** — `__snapshots__//.snap`, next to the test file. This is + the golden, checked-in expected value. + +A snapshot that doesn't exist yet fails the test (it does not get created implicitly) — this is the one exception in `mq-test` where "no such thing" is a real failure, not an error to silently paper over. Run with `--update-snapshots` to create or overwrite golden snapshots from the current output: + +```bash +mq-test --update-snapshots +``` + +`assert_snapshot` is real, unmocked disk I/O — the one deliberate exception to the +hermetic `Io` described below, since golden files must survive across runs and be +checked into version control. + ### Mocking File and Network I/O Each test file runs against an in-memory, hermetic `Io` — no real disk or network access — -so `read_file`/`write_file`/`http` are always allowed, regardless of the `--allow-read` / -`--allow-write` / `--allow-net` flags the CLI itself requires. +so `read_file`/`write_file`/`http` are always allowed, regardless of the `--allow-read` / `--allow-write` / `--allow-net` flags the CLI itself requires. Files can be seeded from within a test simply by writing them first: @@ -250,8 +281,7 @@ def test_reads_a_mocked_api_response(): end ``` -State (files written, mocked responses) does not leak between test files — each gets a -fresh in-memory `Io`. +State (files written, mocked responses) does not leak between test files — each gets a fresh in-memory `Io`. ## Example @@ -295,3 +325,4 @@ cargo build -p mq-test ## License MIT + diff --git a/crates/mq-test/src/coverage.rs b/crates/mq-test/src/coverage.rs index b8f7f08ed..7defb7598 100644 --- a/crates/mq-test/src/coverage.rs +++ b/crates/mq-test/src/coverage.rs @@ -6,6 +6,7 @@ use mq_lang::{CstNode, CstNodeKind, DebugContext, DebuggerAction, DebuggerHandle use rustc_hash::{FxHashMap, FxHashSet}; use crate::highlight; +use crate::html; /// Output format for a coverage report. #[derive(clap::ValueEnum, Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -294,13 +295,6 @@ pub(crate) fn format_lcov_report(coverages: &[FileCoverage]) -> String { out } -pub(crate) fn html_escape(s: &str) -> String { - s.replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) -} - /// Classifies a coverage percentage into a badge color tier. fn badge_class(pct: f64) -> &'static str { if pct >= 80.0 { @@ -448,7 +442,7 @@ fn format_html_source(cov: &FileCoverage, anchor: &str) -> String { \x20 \n\ \x20 \n", anchor = anchor, - file = html_escape(&cov.file.display().to_string()), + file = html::escape(&cov.file.display().to_string()), pct = cov.percent(), covered = cov.covered_lines(), total = cov.total_lines(), @@ -482,7 +476,7 @@ pub(crate) fn format_html_report(coverages: &[FileCoverage]) -> String { \x20 {covered}/{total}\n\ \x20 {uncovered_text}\n\ \x20 \n", - file = html_escape(&cov.file.display().to_string()), + file = html::escape(&cov.file.display().to_string()), badge = badge_class(cov.percent()), pct = cov.percent(), covered = cov.covered_lines(), @@ -498,16 +492,8 @@ pub(crate) fn format_html_report(coverages: &[FileCoverage]) -> String { (total_covered as f64 / total_lines as f64) * 100.0 }; - format!( - "\n\ - \n\ - \n\ - \x20 \n\ - \x20 mq-test coverage report\n\ - \x20 \n\ - \n\ - \n\ - \x20

Coverage report

\n\ + let body = format!( + "\x20

Coverage report

\n\ \x20 \n\ \x20 \n\ \x20 \n\ @@ -519,11 +505,11 @@ pub(crate) fn format_html_report(coverages: &[FileCoverage]) -> String { \x20 \n\ \x20 \n\ \x20
FileCoverageLinesUncovered lines
Total{overall:.1}%{total_covered}/{total_lines}
\n\ - {sources}\ - \n\ - \n", + {sources}", overall_badge = badge_class(overall), - ) + ); + + html::page("mq-test coverage report", HTML_STYLE, &body) } /// Renders a Markdown coverage report: a summary table followed by each diff --git a/crates/mq-test/src/highlight.rs b/crates/mq-test/src/highlight.rs index 75c818704..c3483cfa5 100644 --- a/crates/mq-test/src/highlight.rs +++ b/crates/mq-test/src/highlight.rs @@ -8,7 +8,7 @@ use mq_lang::{CstNode, CstNodeKind, CstTrivia, Shared, TokenKind}; use rustc_hash::FxHashMap; -use crate::coverage::html_escape; +use crate::html::escape as html_escape; /// Coarse lexical category used to pick a `tok-*` CSS class for a token. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/mq-test/src/html.rs b/crates/mq-test/src/html.rs new file mode 100644 index 000000000..363a6c95d --- /dev/null +++ b/crates/mq-test/src/html.rs @@ -0,0 +1,28 @@ +//! Shared building blocks for mq-test's self-contained HTML reports (coverage, snapshot diffs). + +/// Escapes `&`, `<`, `>`, and `"` for safe embedding in HTML. +pub(crate) fn escape(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} + +/// Wraps `body` in a minimal, self-contained HTML page with an inline `\n\ + \n\ + \n\ + {body}\ + \n\ + \n", + title = escape(title), + ) +} diff --git a/crates/mq-test/src/lib.rs b/crates/mq-test/src/lib.rs index fd2f2594f..a9357c4f3 100644 --- a/crates/mq-test/src/lib.rs +++ b/crates/mq-test/src/lib.rs @@ -1,6 +1,8 @@ mod coverage; mod highlight; +mod html; mod runner; +mod snapshot; pub use coverage::{CoverageFormat, FileCoverage}; pub use runner::TestRunner; diff --git a/crates/mq-test/src/main.rs b/crates/mq-test/src/main.rs index 680d552a8..685ccc97d 100644 --- a/crates/mq-test/src/main.rs +++ b/crates/mq-test/src/main.rs @@ -1,6 +1,8 @@ mod coverage; mod highlight; +mod html; mod runner; +mod snapshot; use clap::Parser; use coverage::CoverageFormat; @@ -32,7 +34,9 @@ use std::{path::PathBuf, process::ExitCode}; ## Only run tests tagged \"smoke\" (see `# @tags(...)` in test files):\n\ mq-test --tag smoke\n\n\ ## Run test files in parallel once more than 4 files are discovered:\n\ - mq-test --parallel-threshold 4")] + mq-test --parallel-threshold 4\n\n\ + ## Accept the current output of every assert_snapshot(...) call as the new golden snapshot:\n\ + mq-test --update-snapshots")] struct Cli { /// Path(s) to mq test files. /// Defaults to **/*.mq in the current directory when omitted. @@ -70,6 +74,11 @@ struct Cli { /// Omit to always run files sequentially. #[arg(short = 'P', long)] parallel_threshold: Option, + + /// Write the current output of every `assert_snapshot(name, actual)` call as its new + /// golden snapshot, instead of comparing against the existing one. + #[arg(long)] + update_snapshots: bool, } fn main() -> ExitCode { @@ -83,6 +92,7 @@ fn main() -> ExitCode { .with_filter(cli.filter) .with_tags(cli.tags) .with_parallel_threshold(cli.parallel_threshold.unwrap_or(usize::MAX)) + .with_update_snapshots(cli.update_snapshots) .run() { Ok(true) => ExitCode::SUCCESS, diff --git a/crates/mq-test/src/runner.rs b/crates/mq-test/src/runner.rs index 2c5c9bdb0..038185805 100644 --- a/crates/mq-test/src/runner.rs +++ b/crates/mq-test/src/runner.rs @@ -9,6 +9,7 @@ use std::sync::Mutex; use std::sync::atomic::{AtomicBool, Ordering}; use crate::coverage::{self, CoverageData, CoverageFormat, CoverageHandler, FileCoverage}; +use crate::snapshot; /// Parsed test annotation from a leading comment. #[derive(Debug, PartialEq)] @@ -63,6 +64,7 @@ pub struct TestRunner { filter: Option, tags: Vec, parallel_threshold: usize, + update_snapshots: bool, } impl TestRunner { @@ -78,6 +80,7 @@ impl TestRunner { filter: None, tags: Vec::new(), parallel_threshold: usize::MAX, + update_snapshots: false, } } @@ -126,6 +129,13 @@ impl TestRunner { self } + /// When `true`, `assert_snapshot(name, actual)` writes `actual` as the new golden + /// snapshot instead of comparing against it, for every snapshot exercised by the run. + pub fn with_update_snapshots(mut self, update_snapshots: bool) -> Self { + self.update_snapshots = update_snapshots; + self + } + /// Discovers and executes all test functions. /// /// A file that fails to read, parse, or evaluate is reported in place but does not @@ -183,6 +193,22 @@ impl TestRunner { engine.set_search_paths(vec![parent.to_path_buf()]); } + { + let snapshot_file = file.clone(); + let update_snapshots = self.update_snapshots; + engine.register_fn( + "assert_snapshot", + move |name: String, actual: String| -> mq_lang::HostFnResult { + Ok(snapshot::check_snapshot( + &snapshot_file, + &name, + &actual, + update_snapshots, + )) + }, + ); + } + if self.coverage { engine.set_debugger_handler(Box::new(CoverageHandler(coverage_data.clone()))); let debugger = engine.debugger(); @@ -193,8 +219,6 @@ impl TestRunner { .set_command(mq_lang::DebuggerCommand::StepInto); } - // Snapshot before eval, so only modules newly touched by *this* file's - // engine (the only one that can resolve their paths) get attributed to it. let before_modules: FxHashSet = if self.coverage { coverage_data.snapshot().keys().cloned().collect() } else { @@ -291,9 +315,6 @@ impl TestRunner { Ok(!any_failed.load(Ordering::Relaxed)) } - /// Renders a file-level failure (parse/eval error, not a failing test) with a - /// per-file header. mq-lang's top-level query has no file name of its own, so this - /// re-attaches `file` when the diagnostic's source name is blank. fn render_file_error(file: &Path, mut error: mq_lang::Error) -> String { if error.source_code.name().is_empty() { error.source_code = NamedSource::new(file.display().to_string(), error.source_code.inner().clone()); @@ -305,7 +326,6 @@ impl TestRunner { ) } - /// Returns `true` if `test` should run given `self.filter`/`self.tags`. fn matches(&self, test: &DiscoveredTest) -> bool { let name_matches = match &self.filter { Some(filter) => Self::display_name(test.name()) @@ -379,10 +399,6 @@ impl TestRunner { tests } - /// Parses a comment into a `TestAnnotation`. - /// - /// Supported forms: `@test`, `[test]`, `@parametrize(expr)`, `@tags(a, b)`. - /// Unknown `@name(...)` annotations are silently ignored. fn parse_annotation(comment: &str) -> Option { let s = comment.trim(); @@ -444,9 +460,6 @@ impl TestRunner { } /// Builds the `run_tests(flatten([...]))` call appended to the file content. - /// - /// Simple tests are `[test_case(...)]`; parametrized tests expand via - /// `map(zip(range(...), params), fn(...))`. `flatten` merges both into one list. fn build_test_query(content: &str, tests: &[DiscoveredTest]) -> String { let cases = tests .iter() @@ -481,9 +494,6 @@ impl TestRunner { } } -/// Builds the command that opens `path` in the OS default application for -/// `target_os` (as in `std::env::consts::OS`): `open` on macOS, `start` on -/// Windows, `xdg-open` elsewhere. fn build_open_command(path: &Path, target_os: &str) -> std::process::Command { let mut cmd = if target_os == "macos" { std::process::Command::new("open") @@ -874,6 +884,18 @@ mod tests { dir } + /// Writes a minimal test file exercising `assert_snapshot("greeting", "hello world")`, + /// shared by the `assert_snapshot` integration tests below. + fn write_greeting_snapshot_test_file(dir: &Path) -> PathBuf { + let test_file = dir.join("tests.mq"); + fs::write( + &test_file, + "include \"test\"\n|\ndef test_greeting():\n assert_snapshot(\"greeting\", \"hello world\")\nend\n", + ) + .unwrap(); + test_file + } + #[test] fn test_coverage_reports_only_the_imported_module() { let dir = temp_project_dir("coverage_only_imported"); @@ -1173,4 +1195,50 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + + #[test] + fn test_assert_snapshot_creates_then_matches_across_runs() { + let dir = temp_project_dir("snapshot_create_then_match"); + let test_file = write_greeting_snapshot_test_file(&dir); + + // First run with --update-snapshots creates the golden file and passes. + assert!( + TestRunner::new(vec![test_file.clone()]) + .with_update_snapshots(true) + .run() + .unwrap() + ); + assert!(dir.join("__snapshots__/tests/greeting.snap").exists()); + + // A normal run now compares against it and still passes. + assert!(TestRunner::new(vec![test_file]).run().unwrap()); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn test_assert_snapshot_fails_on_mismatch_and_writes_store() { + let dir = temp_project_dir("snapshot_mismatch"); + let test_file = write_greeting_snapshot_test_file(&dir); + fs::create_dir_all(dir.join("__snapshots__/tests")).unwrap(); + fs::write(dir.join("__snapshots__/tests/greeting.snap"), "goodbye world").unwrap(); + + let passed = TestRunner::new(vec![test_file]).run().unwrap(); + assert!(!passed, "a snapshot mismatch must fail the run"); + assert!(dir.join(".mq-test-store/tests/greeting.diff.html").exists()); + + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn test_assert_snapshot_missing_golden_fails_without_update_flag() { + let dir = temp_project_dir("snapshot_missing"); + let test_file = write_greeting_snapshot_test_file(&dir); + + let passed = TestRunner::new(vec![test_file]).run().unwrap(); + assert!(!passed, "a missing golden snapshot must fail rather than silently pass"); + assert!(!dir.join("__snapshots__/tests/greeting.snap").exists()); + + fs::remove_dir_all(&dir).ok(); + } } diff --git a/crates/mq-test/src/snapshot.rs b/crates/mq-test/src/snapshot.rs new file mode 100644 index 000000000..31d84d2f4 --- /dev/null +++ b/crates/mq-test/src/snapshot.rs @@ -0,0 +1,384 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use mq_lang::{Ident, RuntimeValue, Shared}; +use similar::{ChangeTag, TextDiff}; + +use crate::html; + +const MAX_INLINE_DIFF_LINES: usize = 40; + +fn snapshot_dir(test_file: &Path) -> PathBuf { + base_dir(test_file).join("__snapshots__").join(test_stem(test_file)) +} + +fn store_dir(test_file: &Path) -> PathBuf { + base_dir(test_file).join(".mq-test-store").join(test_stem(test_file)) +} + +fn base_dir(test_file: &Path) -> PathBuf { + match test_file.parent() { + Some(parent) if parent != Path::new("") => parent.to_path_buf(), + _ => PathBuf::from("."), + } +} + +fn test_stem(test_file: &Path) -> String { + test_file + .file_stem() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_else(|| "test".to_string()) +} + +fn sanitize_snapshot_name(name: &str) -> String { + name.chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '[' | ']') { + c + } else { + '_' + } + }) + .collect() +} + +/// Compares `actual` against the golden snapshot named `name`. Real, unmocked disk I/O — +/// unlike every other mq-test builtin, golden files must survive across runs. +/// +/// Returns the assert_eq-style convention `test.mq` expects: the passing value on +/// success, or a `{"error": true, "message": ...}` dict on failure. +pub(crate) fn check_snapshot(test_file: &Path, name: &str, actual: &str, update: bool) -> RuntimeValue { + let safe_name = sanitize_snapshot_name(name); + let snapshot_file = snapshot_dir(test_file).join(format!("{safe_name}.snap")); + + if update { + return match write_snapshot(&snapshot_file, actual) { + Ok(()) => RuntimeValue::String(actual.to_string()), + Err(e) => fail(format!( + "failed to write snapshot \"{name}\" to {}: {e}", + snapshot_file.display() + )), + }; + } + + let expected = match fs::read_to_string(&snapshot_file) { + Ok(content) => content, + Err(_) => { + let report = write_store(test_file, &safe_name, actual, &diff_lines("", actual)); + return fail(format!( + "Assertion failed: snapshot \"{name}\" does not exist yet (expected at {}).\n\n\ + Run with --update-snapshots to create it. The output that would have been \ + saved is available for review at {}", + snapshot_file.display(), + report.display() + )); + } + }; + + if expected == actual { + return RuntimeValue::String(actual.to_string()); + } + + let lines = diff_lines(&expected, actual); + let report = write_store(test_file, &safe_name, actual, &lines); + let (diff, truncated) = truncated_text_diff(&lines); + let mut message = format!("Assertion failed: snapshot \"{name}\" does not match\n\n{diff}"); + if truncated { + message.push_str(&format!("\n … diff truncated, full report at {}", report.display())); + } else { + message.push_str(&format!("\n\nFull report: {}", report.display())); + } + + fail(message) +} + +fn fail(message: String) -> RuntimeValue { + let mut map = BTreeMap::new(); + map.insert(Ident::new("error"), RuntimeValue::Boolean(true)); + map.insert(Ident::new("message"), RuntimeValue::String(message)); + RuntimeValue::Dict(Shared::new(map)) +} + +fn write_snapshot(path: &Path, content: &str) -> std::io::Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(path, content) +} + +fn write_store(test_file: &Path, safe_name: &str, actual: &str, lines: &[DiffLine]) -> PathBuf { + let dir = store_dir(test_file); + let _ = fs::create_dir_all(&dir); + let _ = fs::write(dir.join(format!("{safe_name}.actual.snap")), actual); + + let report_path = dir.join(format!("{safe_name}.diff.html")); + let _ = fs::write(&report_path, format_diff_html(safe_name, lines)); + report_path +} + +struct DiffLine { + tag: ChangeTag, + text: String, +} + +fn diff_lines(expected: &str, actual: &str) -> Vec { + TextDiff::from_lines(expected, actual) + .iter_all_changes() + .map(|change| DiffLine { + tag: change.tag(), + text: change.value().trim_end_matches('\n').to_string(), + }) + .collect() +} + +fn text_diff(lines: &[DiffLine]) -> String { + lines + .iter() + .map(|line| { + let sign = match line.tag { + ChangeTag::Delete => "- ", + ChangeTag::Insert => "+ ", + ChangeTag::Equal => " ", + }; + format!("{sign}{}", line.text) + }) + .collect::>() + .join("\n") +} + +fn truncated_text_diff(lines: &[DiffLine]) -> (String, bool) { + let truncated = lines.len() > MAX_INLINE_DIFF_LINES; + let shown = if truncated { + &lines[..MAX_INLINE_DIFF_LINES] + } else { + lines + }; + (text_diff(shown), truncated) +} + +const DIFF_HTML_STYLE: &str = r#" +:root { + color-scheme: light dark; + --bg: #ffffff; + --fg: #1a1a1a; + --muted: #6b7280; + --insert-bg: #d9f7e380; + --insert-fg: #1a7f37; + --delete-bg: #ffe3e380; + --delete-fg: #c53030; + --code-bg: #f8fafc; +} +@media (prefers-color-scheme: dark) { + :root { + --bg: #16181d; + --fg: #e6e6e6; + --muted: #9aa0a6; + --insert-bg: #12382280; + --insert-fg: #4ada91; + --delete-bg: #3a161880; + --delete-fg: #ff8080; + --code-bg: #1e293b; + } +} +* { box-sizing: border-box; } +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + margin: 2rem auto; + max-width: 960px; + color: var(--fg); + background: var(--bg); +} +h1 { font-size: 1.4rem; } +table { + border-collapse: collapse; + width: 100%; + font-family: ui-monospace, monospace; + font-size: 0.85rem; + background: var(--code-bg); + border-radius: 6px; + overflow: hidden; +} +td { padding: 0.1rem 0.6rem; white-space: pre-wrap; word-break: break-word; } +td.marker { width: 1.5rem; text-align: center; user-select: none; color: var(--muted); } +tr.insert td { background: var(--insert-bg); } +tr.insert td.marker { color: var(--insert-fg); } +tr.delete td { background: var(--delete-bg); } +tr.delete td.marker { color: var(--delete-fg); } +"#; + +/// Renders a self-contained HTML diff report, in the spirit of typst's `store/` HTML +/// diff reports — meant for reviewing a large snapshot mismatch without scrolling a +/// terminal, and as a CI artifact. +fn format_diff_html(name: &str, lines: &[DiffLine]) -> String { + let mut rows = String::new(); + + for line in lines { + let (class, marker) = match line.tag { + ChangeTag::Delete => ("delete", "-"), + ChangeTag::Insert => ("insert", "+"), + ChangeTag::Equal => ("equal", " "), + }; + rows.push_str(&format!( + " {marker}{code}\n", + code = html::escape(&line.text), + )); + } + + let body = format!( + "\x20

Snapshot diff — {escaped_name}

\n\ + \x20 \n\ + \x20 \n\ + {rows}\ + \x20 \n\ + \x20
\n", + escaped_name = html::escape(name), + ); + + html::page(&format!("mq-test snapshot diff — {name}"), DIFF_HTML_STYLE, &body) +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + fn temp_test_file(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("mq_test_snapshot_{name}_{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + dir.join("tests.mq") + } + + fn is_error_dict(value: &RuntimeValue) -> bool { + matches!(value, RuntimeValue::Dict(map) if map.contains_key(&Ident::new("error"))) + } + + #[rstest] + #[case("plain", "plain")] + #[case("a-b_c.d[0]", "a-b_c.d[0]")] + #[case("has spaces", "has_spaces")] + #[case("weird/../name", "weird_.._name")] + #[case("emoji🎉name", "emoji_name")] + fn test_sanitize_snapshot_name(#[case] input: &str, #[case] expected: &str) { + assert_eq!(sanitize_snapshot_name(input), expected); + } + + #[test] + fn test_missing_snapshot_fails_and_writes_store() { + let test_file = temp_test_file("missing"); + let result = check_snapshot(&test_file, "greeting", "hello world", false); + + assert!(is_error_dict(&result)); + let report = store_dir(&test_file).join("greeting.diff.html"); + assert!(report.exists(), "expected diff report at {}", report.display()); + assert!(!snapshot_dir(&test_file).join("greeting.snap").exists()); + + fs::remove_dir_all(test_file.parent().unwrap()).ok(); + } + + #[test] + fn test_update_creates_snapshot_and_passes() { + let test_file = temp_test_file("create"); + let result = check_snapshot(&test_file, "greeting", "hello world", true); + + assert_eq!(result, RuntimeValue::String("hello world".to_string())); + let saved = fs::read_to_string(snapshot_dir(&test_file).join("greeting.snap")).unwrap(); + assert_eq!(saved, "hello world"); + + fs::remove_dir_all(test_file.parent().unwrap()).ok(); + } + + #[test] + fn test_matching_snapshot_passes() { + let test_file = temp_test_file("match"); + check_snapshot(&test_file, "greeting", "hello world", true); + + let result = check_snapshot(&test_file, "greeting", "hello world", false); + assert_eq!(result, RuntimeValue::String("hello world".to_string())); + + fs::remove_dir_all(test_file.parent().unwrap()).ok(); + } + + #[test] + fn test_mismatched_snapshot_fails_and_writes_store() { + let test_file = temp_test_file("mismatch"); + check_snapshot(&test_file, "greeting", "hello world", true); + + let result = check_snapshot(&test_file, "greeting", "goodbye world", false); + assert!(is_error_dict(&result)); + + let actual_store = store_dir(&test_file).join("greeting.actual.snap"); + assert_eq!(fs::read_to_string(&actual_store).unwrap(), "goodbye world"); + let report = store_dir(&test_file).join("greeting.diff.html"); + let html = fs::read_to_string(&report).unwrap(); + assert!(html.contains("hello")); + assert!(html.contains("goodbye")); + + // The golden file itself must be untouched by a non-update mismatch. + assert_eq!( + fs::read_to_string(snapshot_dir(&test_file).join("greeting.snap")).unwrap(), + "hello world" + ); + + fs::remove_dir_all(test_file.parent().unwrap()).ok(); + } + + #[test] + fn test_update_overwrites_an_existing_mismatched_snapshot() { + let test_file = temp_test_file("overwrite"); + check_snapshot(&test_file, "greeting", "hello world", true); + check_snapshot(&test_file, "greeting", "goodbye world", true); + + let result = check_snapshot(&test_file, "greeting", "goodbye world", false); + assert_eq!(result, RuntimeValue::String("goodbye world".to_string())); + + fs::remove_dir_all(test_file.parent().unwrap()).ok(); + } + + #[test] + fn test_large_diff_is_truncated_in_the_failure_message_but_full_report_is_written() { + let test_file = temp_test_file("large"); + let expected: String = (0..200).map(|i| format!("line {i}\n")).collect(); + let actual: String = (0..200).map(|i| format!("line {i} changed\n")).collect(); + check_snapshot(&test_file, "big", &expected, true); + fs::write(snapshot_dir(&test_file).join("big.snap"), &expected).unwrap(); + + let result = check_snapshot(&test_file, "big", &actual, false); + match &result { + RuntimeValue::Dict(map) => { + let message = match map.get(&Ident::new("message")).unwrap() { + RuntimeValue::String(s) => s.clone(), + _ => panic!("expected string message"), + }; + assert!( + message.contains("truncated"), + "message should mention truncation: {message}" + ); + assert!( + message.lines().count() < 200, + "inline message must not dump the whole 200-line diff: {} lines", + message.lines().count() + ); + } + other => panic!("expected error dict, got {other:?}"), + } + + let report = fs::read_to_string(store_dir(&test_file).join("big.diff.html")).unwrap(); + assert!( + report.contains("line 199 changed"), + "full report must contain every changed line" + ); + + fs::remove_dir_all(test_file.parent().unwrap()).ok(); + } + + #[test] + fn test_snapshot_dir_and_store_dir_are_scoped_per_test_file_stem() { + let a = PathBuf::from("/proj/tests/a.mq"); + let b = PathBuf::from("/proj/tests/b.mq"); + assert_ne!(snapshot_dir(&a), snapshot_dir(&b)); + assert_ne!(store_dir(&a), store_dir(&b)); + assert!(snapshot_dir(&a).ends_with("__snapshots__/a")); + assert!(store_dir(&a).ends_with(".mq-test-store/a")); + } +}