diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3b5b8b..fbe762e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,11 @@ jobs: go build -o /dev/null "./${app}cmd/kit" done + # Samples are other people's apps, so this repo holds only a descriptor for + # each. A sample with a bad descriptor, or with app code in it, fails here. + - name: Check the sample descriptors + run: make check-samples + # Each demo's UI is built with pnpm and embedded in its Go binary, so this job # builds the UI, builds the binary, and checks the binary serves it. web: diff --git a/.no-mistakes.yaml b/.no-mistakes.yaml index aab3529..d0304f4 100644 --- a/.no-mistakes.yaml +++ b/.no-mistakes.yaml @@ -6,5 +6,5 @@ # checks run only where `pnpm install` has been run in web/. commands: test: 'set -e; for m in $(go list -m); do go test -count=1 "$m/..."; done; if [ -d web/node_modules ]; then pnpm --dir web test; fi' - lint: 'set -e; for m in $(go list -m); do go vet "$m/..."; done; for a in apps/*/; do [ -d "${a}cmd/kit" ] || { echo "$a has no cmd/kit"; exit 1; }; done; test -z "$(gofmt -l .)" || { echo "unformatted files (run gofmt -w .):"; gofmt -l .; exit 1; }; if [ -d web/node_modules ]; then pnpm --dir web lint && pnpm --dir web type-check; fi' + lint: 'set -e; for m in $(go list -m); do go vet "$m/..."; done; for a in apps/*/; do [ -d "${a}cmd/kit" ] || { echo "$a has no cmd/kit"; exit 1; }; done; (cd shared && go run ./cmd/check-samples ../samples); test -z "$(gofmt -l .)" || { echo "unformatted files (run gofmt -w .):"; gofmt -l .; exit 1; }; if [ -d web/node_modules ]; then pnpm --dir web lint && pnpm --dir web type-check; fi' format: "gofmt -w ." diff --git a/Makefile b/Makefile index db41878..0cb35bf 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test vet check-kits kit web +.PHONY: build test vet check-kits check-samples kit web # Every module in the Go workspace is built, vetted and tested. In a workspace, # ./... from the root matches only the root module, so each module is listed. @@ -21,6 +21,10 @@ test: check-kits: set -e; for a in apps/*/; do [ -d "$${a}cmd/kit" ] || { echo "$$a has no cmd/kit (every demo needs an offline kit)"; exit 1; }; done +# Every sample app's descriptor is well formed, and no sample holds app code. +check-samples: + cd shared && go run ./cmd/check-samples ../samples + # Builds a demo's web UI and copies it into the Go binary's embedded assets. # Run it before `make build` to get the real UI instead of the placeholder. # make web APP=acme-commerce-lenders diff --git a/README.md b/README.md index f08e7b4..3e6dc0e 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,10 @@ Code and docs: [apps/acme-commerce-lenders](apps/acme-commerce-lenders) **Apply, offer, accept and settle.** A merchant applies for working capital and receives an offer. They accept it, the advance is funded, and the settlement appears in their Xero books. +## Samples + +A sample is an open source app, written by someone else for real users, that is run against the app emulators. A demo is ours and is staged to show well. A sample has the use cases and edge cases real customers hit, so it shows where an app emulator differs from the real vendor. `samples/` records which app was run and how, and never holds the app's code. See [samples](samples). + ## How demos run Each demo is one Go binary, `cmd/kit`, that serves the demo's UI and runs its logic. It runs beside LocalStack, which hosts the app emulators and serves each one on its own host name through LocalStack's gateway, so you start LocalStack with `lstk` first. The demo calls each vendor through a local recording proxy (`shared/wire`), which is how the inspector shows what the demo sends. Each vendor's app emulator is hosted by LocalStack or run by you (`_EMULATOR_URL`), so you can build against an app emulator locally and move it to LocalStack once LocalStack includes it. @@ -30,6 +34,7 @@ Each demo is a Go app under `apps/`. The repo is a Go workspace (`go.work`), so |---|---| | `apps//` | One demo: its Go code, its docs and its embedded UI | | `shared/` | Code every demo uses: the recording proxy for vendor calls, vendor base URL config, and a launcher for running app emulators without LocalStack | +| `samples//` | One sample: a descriptor of the open source app and the output of runs, never the app's code | | `web/` | pnpm workspace for the demo UIs, built as static assets that the Go binaries embed | | `templates/demo-docs/` | The three docs every demo carries: Usage Guide, Demo Scenarios, Troubleshooting Guide | @@ -42,6 +47,7 @@ make web APP= # builds that demo's UI into its binary (needs No make build # builds every apps//cmd/ to bin/- make kit APP= # builds, then runs that demo's binary make check-kits # fails if a demo has no cmd/kit +make check-samples # fails if a sample's descriptor is malformed or a sample holds app code ``` `go test -tags live ./wire` in `shared/` also runs a check against a LocalStack you have running. diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 0000000..83506dc --- /dev/null +++ b/samples/README.md @@ -0,0 +1,81 @@ +# Samples + +A sample is an open source app, written by someone else for real users, that is run against the app emulators. It has the use cases and edge cases real customers hit, and its own tests, which we did not invent. The demos in `apps/` are different: they are staged to show well, and they are ours. + +This directory records which app was run and how. It never holds the app's code. The code stays in its own repo under its own license, and a run fetches the commit named here. + +## What a sample holds + +``` +samples// + sample.json which app, at which commit, and how it is run + README.md optional notes + results/ the output of runs (JSON) +``` + +Nothing else. `make check-samples` fails on any other file. + +## sample.json + +```json +{ + "name": "example-shop", + "repo": "https://github.com/example/shop", + "commit": "0123456789abcdef0123456789abcdef01234567", + "license": "MIT", + "vendors": ["stripe"], + "overrides": { "stripe": "STRIPE_API_BASE" }, + "run": "npm test" +} +``` + +This is an illustration, not a real sample. + +| Field | Meaning | +|---|---| +| `name` | The directory name. Lower-case words joined by hyphens. | +| `repo` | The https URL of the app's repo. | +| `commit` | The full 40-character SHA that was run. A branch or tag is rejected, because a sample must be reproducible. | +| `license` | The app's SPDX identifier, such as `MIT` or `Apache-2.0`. | +| `vendors` | The vendors whose app emulators it is run against. | +| `overrides` | For each vendor, the environment variable or setting that holds its base URL. | +| `run` | The command that runs the app or its tests, from the app's checkout. | + +## Checking + +```sh +make check-samples +``` + +## Recording a run + +Two commands in `shared/` record what an app asks of the app emulators and report what they do not cover. + +**1. Record.** `wire-proxy` listens on one port per vendor and forwards to that vendor's app emulator, the one at `_EMULATOR_URL` if set, otherwise the one LocalStack serves. Point the app's vendor base URL at the listening address, using the `overrides` in `sample.json`: + +```sh +cd shared +STRIPE_EMULATOR_VERSION= go run ./cmd/wire-proxy \ + --proxy stripe=127.0.0.1:5301 --proxy plaid=127.0.0.1:5302 \ + --events 127.0.0.1:5399 --out ../samples//results/-calls.json +``` + +Run the app or its tests, then stop the proxy with Ctrl-C. It writes every call to `--out`, with credentials in bodies masked. While it runs, `curl http://127.0.0.1:5399/events` shows the calls so far. Each call records its `origin` and, if you set `_EMULATOR_VERSION`, the app emulator's release id. An app emulator does not announce its release, so the id is the one in the release's tag, `twin--v`. + +**2. Report.** `gap-report` checks each call against the app emulators' manifests, which are `twin-/twin-manifest.json` in `WonderTwin-AI/wondertwin`: + +```sh +cd shared +go run ./cmd/gap-report --events ../samples//results/-calls.json \ + --manifest stripe=/twin-stripe/twin-manifest.json \ + --manifest plaid=/twin-plaid/twin-manifest.json +``` + +It lists each distinct call, with identifiers in the path replaced by `{}`, as `covered`, `gap` or `unchecked`, and which app emulator release it ran against. `--json` prints the same as JSON. `--fail-on-gap` exits 1 if any call is a gap. + +### What the report can and cannot tell you + +- **It reads the manifest, not the response.** Stripe's app emulator answers a route it lacks with a 404 that copies the real Stripe, so a status cannot say whether a route is missing. +- **The manifests do not share a format, so precision differs.** Xero lists routes (`PUT /api.xro/2.0/Accounts`), so a covered call is exact. Plaid lists paths (`item/get`), so it cannot tell methods apart. Stripe lists resource names (`payment_intents`), so `covered` means only that some of that resource is implemented. The `BASIS` column says which applies. +- **Identifiers are recognised by their shape.** A segment with a digit or a capital that looks like an id (`acct_3Bs...`, a GUID, a long token) becomes `{}`. An unusual identifier could be reported as its own call. +- **A vendor with no manifest is `unchecked`, never a gap.** diff --git a/shared/cmd/check-samples/main.go b/shared/cmd/check-samples/main.go new file mode 100644 index 0000000..0158cb3 --- /dev/null +++ b/shared/cmd/check-samples/main.go @@ -0,0 +1,42 @@ +// Command check-samples validates the descriptor of every sample app. +// +// check-samples [samples-dir] +// +// It exits 0 when every sample is well formed, 1 when any is not, and 2 when +// the directory cannot be read. +package main + +import ( + "fmt" + "io" + "os" + + "github.com/localstack-samples/localstack-app-emulators/shared/samples" +) + +func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } + +func run(args []string, stdout, stderr io.Writer) int { + if len(args) > 1 { + fmt.Fprintln(stderr, "usage: check-samples [samples-dir]") + return 2 + } + root := "samples" + if len(args) == 1 { + root = args[0] + } + problems, err := samples.Check(root) + if err != nil { + fmt.Fprintf(stderr, "check-samples: %v\n", err) + return 2 + } + for _, p := range problems { + fmt.Fprintln(stderr, p) + } + if len(problems) > 0 { + fmt.Fprintf(stderr, "%d problem(s) in %s\n", len(problems), root) + return 1 + } + fmt.Fprintf(stdout, "samples in %s are well formed\n", root) + return 0 +} diff --git a/shared/cmd/check-samples/main_test.go b/shared/cmd/check-samples/main_test.go new file mode 100644 index 0000000..a4b9613 --- /dev/null +++ b/shared/cmd/check-samples/main_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunExitCodes(t *testing.T) { + good := t.TempDir() + bad := t.TempDir() + if err := os.MkdirAll(filepath.Join(bad, "no-descriptor"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bad, "no-descriptor", "README.md"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + args []string + code int + out string + err string + }{ + {"an empty section passes", []string{good}, 0, "well formed", ""}, + {"a bad sample fails and is named", []string{bad}, 1, "", "no-descriptor: has no sample.json"}, + {"a missing directory is an error", []string{filepath.Join(good, "nope")}, 2, "", "check-samples:"}, + {"too many arguments", []string{"a", "b"}, 2, "", "usage:"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var out, errb bytes.Buffer + if got := run(tc.args, &out, &errb); got != tc.code { + t.Fatalf("exit %d, want %d (stderr: %s)", got, tc.code, errb.String()) + } + if !strings.Contains(out.String(), tc.out) || !strings.Contains(errb.String(), tc.err) { + t.Fatalf("stdout %q, stderr %q", out.String(), errb.String()) + } + }) + } +} diff --git a/shared/cmd/gap-report/main.go b/shared/cmd/gap-report/main.go new file mode 100644 index 0000000..5857a5e --- /dev/null +++ b/shared/cmd/gap-report/main.go @@ -0,0 +1,96 @@ +// Command gap-report says which calls an app made that the app emulators do +// not cover. +// +// gap-report --events calls.json --manifest stripe=twin-stripe/twin-manifest.json +// +// calls.json is what wire-proxy --out wrote. Each --manifest is an app +// emulator's twin-manifest.json from WonderTwin-AI/wondertwin. A vendor with no +// manifest is reported as unchecked, not as a gap. +// +// It exits 0 after printing the report. With --fail-on-gap it exits 1 when any +// call is not covered, and 2 when its input cannot be read. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "os" + "strings" + + "github.com/localstack-samples/localstack-app-emulators/shared/gaps" + "github.com/localstack-samples/localstack-app-emulators/shared/wire" +) + +func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } + +const usage = `usage: gap-report --events calls.json --manifest vendor=path [--manifest vendor=path ...] [--json] [--fail-on-gap] +` + +type manifestFlags map[string]string + +func (m manifestFlags) String() string { return "" } + +func (m manifestFlags) Set(v string) error { + vendor, path, ok := strings.Cut(v, "=") + if !ok || vendor == "" || path == "" { + return fmt.Errorf("--manifest %q must be vendor=path", v) + } + if _, dup := m[vendor]; dup { + return fmt.Errorf("--manifest names %s twice", vendor) + } + m[vendor] = path + return nil +} + +func run(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("gap-report", flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() {} + paths := manifestFlags{} + events := fs.String("events", "", "") + asJSON := fs.Bool("json", false, "") + failOnGap := fs.Bool("fail-on-gap", false, "") + fs.Var(paths, "manifest", "") + if err := fs.Parse(args); err != nil || *events == "" || fs.NArg() > 0 { + fmt.Fprint(stderr, usage) + return 2 + } + + raw, err := os.ReadFile(*events) + if err != nil { + fmt.Fprintf(stderr, "gap-report: %v\n", err) + return 2 + } + var rec struct { + Events []wire.Event `json:"events"` + } + if err := json.Unmarshal(raw, &rec); err != nil { + fmt.Fprintf(stderr, "gap-report: %s is not a recording: %v\n", *events, err) + return 2 + } + + manifests := map[string]gaps.Manifest{} + for vendor, path := range paths { + m, err := gaps.LoadManifest(path) + if err != nil { + fmt.Fprintf(stderr, "gap-report: %s manifest: %v\n", vendor, err) + return 2 + } + manifests[vendor] = m + } + + report := gaps.Build(rec.Events, manifests) + if *asJSON { + enc := json.NewEncoder(stdout) + enc.SetIndent("", " ") + _ = enc.Encode(report) + } else { + gaps.Write(stdout, report) + } + if *failOnGap && report.Gaps() > 0 { + return 1 + } + return 0 +} diff --git a/shared/cmd/gap-report/main_test.go b/shared/cmd/gap-report/main_test.go new file mode 100644 index 0000000..64b8ad6 --- /dev/null +++ b/shared/cmd/gap-report/main_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func fixtures(t *testing.T) (events, xero, stripe string) { + t.Helper() + dir := t.TempDir() + write := func(name, body string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return p + } + events = write("calls.json", `{"last":3,"generation":0,"events":[ + {"id":1,"vendor":"xero","method":"PUT","path":"/api.xro/2.0/Accounts","status":200,"emulator_version":"aaaaaaaaaaaa"}, + {"id":2,"vendor":"xero","method":"POST","path":"/api.xro/2.0/Accounts","status":405,"emulator_version":"aaaaaaaaaaaa"}, + {"id":3,"vendor":"plaid","method":"POST","path":"/item/get","status":200}]}`) + xero = write("xero.json", `{"coverage":{"resources_implemented":["PUT /api.xro/2.0/Accounts"]}}`) + stripe = write("stripe.json", `{"coverage":{"resources_implemented":["customers"]}}`) + return events, xero, stripe +} + +func TestReportPrintsGapsAndExitsZero(t *testing.T) { + events, xero, _ := fixtures(t) + var out, errb bytes.Buffer + if code := run([]string{"--events", events, "--manifest", "xero=" + xero}, &out, &errb); code != 0 { + t.Fatalf("exit %d: %s", code, errb.String()) + } + for _, want := range []string{"xero:POST /api.xro/2.0/Accounts", "unchecked", "plaid:POST /item/get", "1 not covered"} { + if !strings.Contains(out.String(), want) { + t.Errorf("want %q in:\n%s", want, out.String()) + } + } +} + +func TestFailOnGap(t *testing.T) { + events, xero, _ := fixtures(t) + var out, errb bytes.Buffer + if code := run([]string{"--events", events, "--manifest", "xero=" + xero, "--fail-on-gap"}, &out, &errb); code != 1 { + t.Fatalf("a gap should exit 1, got %d", code) + } + // With nothing to check against there is no gap to fail on. + if code := run([]string{"--events", events, "--fail-on-gap"}, &out, &errb); code != 0 { + t.Fatalf("unchecked calls are not gaps, got exit %d", code) + } +} + +func TestJSONOutput(t *testing.T) { + events, xero, _ := fixtures(t) + var out, errb bytes.Buffer + if code := run([]string{"--events", events, "--manifest", "xero=" + xero, "--json"}, &out, &errb); code != 0 { + t.Fatalf("exit %d: %s", code, errb.String()) + } + var r struct { + Calls []struct { + Vendor, Method, Path, Coverage string + } + Versions map[string][]string + } + if err := json.Unmarshal(out.Bytes(), &r); err != nil { + t.Fatalf("not JSON: %v\n%s", err, out.String()) + } + if len(r.Calls) != 3 || r.Calls[0].Coverage != "gap" || r.Versions["xero"][0] != "aaaaaaaaaaaa" { + t.Fatalf("unexpected report: %+v", r) + } +} + +func TestBadInputExitsTwo(t *testing.T) { + events, xero, _ := fixtures(t) + dir := t.TempDir() + notRecording := filepath.Join(dir, "x.json") + os.WriteFile(notRecording, []byte("{nope"), 0o644) + empty := filepath.Join(dir, "empty.json") + os.WriteFile(empty, []byte(`{"coverage":{"resources_implemented":[]}}`), 0o644) + + tests := []struct { + name string + args []string + want string + }{ + {"no events", nil, "usage:"}, + {"a stray argument", []string{"--events", events, "extra"}, "usage:"}, + {"manifest without a path", []string{"--events", events, "--manifest", "xero"}, "vendor=path"}, + {"the same vendor twice", []string{"--events", events, "--manifest", "xero=" + xero, "--manifest", "xero=" + xero}, "twice"}, + {"a missing events file", []string{"--events", filepath.Join(dir, "absent.json")}, "gap-report:"}, + {"events that are not a recording", []string{"--events", notRecording}, "not a recording"}, + {"a manifest with no coverage", []string{"--events", events, "--manifest", "xero=" + empty}, "xero manifest"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var out, errb bytes.Buffer + if code := run(tc.args, &out, &errb); code != 2 { + t.Fatalf("exit %d, want 2", code) + } + if !strings.Contains(errb.String(), tc.want) { + t.Fatalf("stderr %q should contain %q", errb.String(), tc.want) + } + }) + } +} diff --git a/shared/cmd/wire-proxy/flags.go b/shared/cmd/wire-proxy/flags.go new file mode 100644 index 0000000..090878d --- /dev/null +++ b/shared/cmd/wire-proxy/flags.go @@ -0,0 +1,15 @@ +package main + +import ( + "flag" + "io" +) + +// newFlagSet returns a flag set that reports errors to w and leaves the +// usage text to the caller. +func newFlagSet(w io.Writer) *flag.FlagSet { + fs := flag.NewFlagSet("wire-proxy", flag.ContinueOnError) + fs.SetOutput(w) + fs.Usage = func() {} + return fs +} diff --git a/shared/cmd/wire-proxy/main.go b/shared/cmd/wire-proxy/main.go new file mode 100644 index 0000000..10a0df5 --- /dev/null +++ b/shared/cmd/wire-proxy/main.go @@ -0,0 +1,252 @@ +// Command wire-proxy records the calls an app makes to its vendors. +// +// wire-proxy --proxy stripe=127.0.0.1:5301 --proxy plaid=127.0.0.1:5302 --out calls.json +// +// Each --proxy listens on an address and forwards to that vendor's app +// emulator: the one at _EMULATOR_URL if that is set, otherwise the one +// LocalStack serves. Point the app's vendor base URL at the listening address. +// +// Every call is recorded, with credentials in the bodies masked, in one +// recording shared by all the proxies. --events serves it over HTTP while the +// proxy runs, and --out writes it when the proxy stops. +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/localstack-samples/localstack-app-emulators/shared/wire" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + os.Exit(run(ctx, os.Args[1:], os.Getenv, os.Stdout, os.Stderr)) +} + +const usage = `usage: wire-proxy --proxy vendor=addr [--proxy vendor=addr ...] [--events addr] [--out file] [--keep n] + + --proxy listen on addr and forward to the vendor's app emulator (repeatable) + --events serve the recording at http://addr/events?since=N + --out write the recording to file when the proxy stops + --keep calls to keep in memory (default 20000) +` + +type proxySpec struct{ vendor, addr string } + +type proxyFlags []proxySpec + +func (p *proxyFlags) String() string { return "" } + +func (p *proxyFlags) Set(v string) error { + vendor, addr, ok := strings.Cut(v, "=") + if !ok || vendor == "" || addr == "" { + return fmt.Errorf("--proxy %q must be vendor=addr, such as stripe=127.0.0.1:5301", v) + } + for _, s := range *p { + if s.vendor == vendor { + return fmt.Errorf("--proxy names %s twice", vendor) + } + } + *p = append(*p, proxySpec{vendor, addr}) + return nil +} + +func run(ctx context.Context, args []string, getenv func(string) string, stdout, stderr io.Writer) int { + var proxies proxyFlags + cfg := config{getenv: getenv, keep: 20000} + fs := newFlagSet(stderr) + fs.Var(&proxies, "proxy", "") + fs.StringVar(&cfg.events, "events", "", "") + fs.StringVar(&cfg.out, "out", "", "") + fs.IntVar(&cfg.keep, "keep", cfg.keep, "") + if err := fs.Parse(args); err != nil { + fmt.Fprint(stderr, usage) + return 2 + } + if len(proxies) == 0 || fs.NArg() > 0 || cfg.keep < 1 { + fmt.Fprint(stderr, usage) + return 2 + } + cfg.proxies = proxies + + inst, err := start(cfg) + if err != nil { + fmt.Fprintf(stderr, "wire-proxy: %v\n", err) + return 1 + } + for _, s := range cfg.proxies { + t := inst.targets[s.vendor] + fmt.Fprintf(stdout, "%-7s http://%s -> %s %s\n", s.vendor, inst.addrs[s.vendor], t.Source, t.Upstream) + } + if inst.eventsAddr != "" { + fmt.Fprintf(stdout, "events http://%s/events\n", inst.eventsAddr) + } + + select { + case <-ctx.Done(): + case err := <-inst.failed: + fmt.Fprintf(stderr, "wire-proxy: %v\n", err) + inst.stop(context.Background()) + return 1 + } + if err := inst.stop(context.Background()); err != nil { + fmt.Fprintf(stderr, "wire-proxy: %v\n", err) + return 1 + } + if cfg.out != "" { + fmt.Fprintf(stdout, "wrote %d calls to %s\n", len(inst.rec.Events()), cfg.out) + } + return 0 +} + +type config struct { + proxies []proxySpec + events string + out string + keep int + getenv func(string) string +} + +// instance is a running wire-proxy. +type instance struct { + rec *wire.Recorder + targets map[string]wire.Target + addrs map[string]string // vendor to the address it listens on + eventsAddr string + out string + servers []*http.Server + failed chan error +} + +func start(cfg config) (*instance, error) { + inst := &instance{ + rec: wire.NewRecorder(cfg.keep), + targets: map[string]wire.Target{}, + addrs: map[string]string{}, + out: cfg.out, + failed: make(chan error, len(cfg.proxies)+1), + } + serve := func(addr string, h http.Handler) (string, error) { + ln, err := net.Listen("tcp", addr) + if err != nil { + return "", err + } + srv := &http.Server{Handler: h, ReadHeaderTimeout: 5 * time.Second} + inst.servers = append(inst.servers, srv) + go func() { + if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { + inst.failed <- err + } + }() + return ln.Addr().String(), nil + } + + for _, s := range cfg.proxies { + t := wire.TargetFromEnv(cfg.getenv, s.vendor) + addr, err := serve(s.addr, wire.NewProxy(t, inst.rec)) + if err != nil { + inst.shutdown(context.Background()) + return nil, fmt.Errorf("listen for %s on %s: %w", s.vendor, s.addr, err) + } + inst.targets[s.vendor], inst.addrs[s.vendor] = t, addr + } + if cfg.events != "" { + mux := http.NewServeMux() + mux.HandleFunc("GET /events", inst.serveEvents) + addr, err := serve(cfg.events, mux) + if err != nil { + inst.shutdown(context.Background()) + return nil, fmt.Errorf("listen for events on %s: %w", cfg.events, err) + } + inst.eventsAddr = addr + } + return inst, nil +} + +// recording is what --events serves and --out writes. It is the shape of the +// demo's GET /api/events, so one reader handles both. +type recording struct { + Events []wire.Event `json:"events"` + Last uint64 `json:"last"` + Generation uint64 `json:"generation"` +} + +func (i *instance) snapshot(since uint64) recording { + evs := i.rec.Since(since) + last := since + if len(evs) > 0 { + last = evs[len(evs)-1].ID + } + return recording{Events: evs, Last: last, Generation: i.rec.Generation()} +} + +func (i *instance) serveEvents(w http.ResponseWriter, r *http.Request) { + var since uint64 + if s := r.URL.Query().Get("since"); s != "" { + n, err := strconv.ParseUint(s, 10, 64) + if err != nil { + http.Error(w, "since must be an event id", http.StatusBadRequest) + return + } + since = n + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(i.snapshot(since)) +} + +// shutdown closes every listener and waits for calls in flight. +func (i *instance) shutdown(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + for _, s := range i.servers { + _ = s.Shutdown(ctx) + } +} + +// stop closes every listener, then writes the recording to --out if it was +// asked for. The file is written last so it holds every call that finished. +func (i *instance) stop(ctx context.Context) error { + i.shutdown(ctx) + if i.out == "" { + return nil + } + return writeAtomic(i.out, i.snapshot(0)) +} + +// writeAtomic writes v as JSON to path through a temporary file, so a reader +// never sees half a recording. +func writeAtomic(path string, v any) error { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(path), ".wire-proxy-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(append(b, '\n')); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} diff --git a/shared/cmd/wire-proxy/main_test.go b/shared/cmd/wire-proxy/main_test.go new file mode 100644 index 0000000..3e08b49 --- /dev/null +++ b/shared/cmd/wire-proxy/main_test.go @@ -0,0 +1,311 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func env(m map[string]string) func(string) string { return func(k string) string { return m[k] } } + +// upstream is an app emulator that echoes what it was asked. +func upstream(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"path":"`+r.URL.Path+`"}`) + })) + t.Cleanup(srv.Close) + return srv +} + +func startProxy(t *testing.T, cfg config) *instance { + t.Helper() + inst, err := start(cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { inst.shutdown(context.Background()) }) + return inst +} + +func get(t *testing.T, url string) string { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return string(b) +} + +func TestARecordedCallIsForwardedAndServedBack(t *testing.T) { + up := upstream(t) + inst := startProxy(t, config{ + proxies: []proxySpec{{"stripe", "127.0.0.1:0"}}, + events: "127.0.0.1:0", + keep: 100, + getenv: env(map[string]string{ + "STRIPE_EMULATOR_URL": up.URL, + "STRIPE_EMULATOR_VERSION": "0123456789ab", + }), + }) + + if got := get(t, "http://"+inst.addrs["stripe"]+"/v1/charges?limit=1"); got != `{"path":"/v1/charges"}` { + t.Fatalf("the call was not forwarded: %q", got) + } + + var rec recording + if err := json.Unmarshal([]byte(get(t, "http://"+inst.eventsAddr+"/events")), &rec); err != nil { + t.Fatal(err) + } + if len(rec.Events) != 1 { + t.Fatalf("want one recorded call, got %+v", rec) + } + ev := rec.Events[0] + if ev.Vendor != "stripe" || ev.Method != "GET" || ev.Path != "/v1/charges?limit=1" || ev.Status != 200 || + ev.Source != "local" || ev.Origin != "emulator" || ev.EmulatorVersion != "0123456789ab" { + t.Fatalf("unexpected event: %+v", ev) + } + + // since returns only what is new, and keeps the cursor when nothing is. + if err := json.Unmarshal([]byte(get(t, "http://"+inst.eventsAddr+"/events?since=1")), &rec); err != nil { + t.Fatal(err) + } + if len(rec.Events) != 0 || rec.Last != 1 { + t.Fatalf("since=1 should be empty with the cursor kept: %+v", rec) + } +} + +func TestVendorsShareOneRecording(t *testing.T) { + up := upstream(t) + inst := startProxy(t, config{ + proxies: []proxySpec{{"stripe", "127.0.0.1:0"}, {"plaid", "127.0.0.1:0"}}, + keep: 100, + getenv: env(map[string]string{"STRIPE_EMULATOR_URL": up.URL, "PLAID_EMULATOR_URL": up.URL}), + }) + if inst.addrs["stripe"] == inst.addrs["plaid"] { + t.Fatal("each vendor needs its own port") + } + get(t, "http://"+inst.addrs["stripe"]+"/v1/a") + get(t, "http://"+inst.addrs["plaid"]+"/item/get") + + evs := inst.rec.Events() + if len(evs) != 2 || evs[0].Vendor != "stripe" || evs[1].Vendor != "plaid" || evs[1].ID != 2 { + t.Fatalf("want one ordered recording of both vendors: %+v", evs) + } +} + +// The proxy is the same one the demo uses, so credentials in a body are masked +// before they are recorded or written. +func TestCredentialsAreMaskedInTheRecording(t *testing.T) { + up := upstream(t) + out := filepath.Join(t.TempDir(), "calls.json") + inst := startProxy(t, config{ + proxies: []proxySpec{{"stripe", "127.0.0.1:0"}}, + out: out, + keep: 100, + getenv: env(map[string]string{"STRIPE_EMULATOR_URL": up.URL}), + }) + resp, err := http.Post("http://"+inst.addrs["stripe"]+"/v1/x", "application/json", strings.NewReader(`{"account_number":"000123456789"}`)) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if err := inst.stop(context.Background()); err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(b), "000123456789") { + t.Fatalf("the account number reached the recording:\n%s", b) + } +} + +func TestStopWritesTheRecordingToOut(t *testing.T) { + up := upstream(t) + out := filepath.Join(t.TempDir(), "nested", "calls.json") + inst := startProxy(t, config{ + proxies: []proxySpec{{"stripe", "127.0.0.1:0"}}, + out: out, + keep: 100, + getenv: env(map[string]string{"STRIPE_EMULATOR_URL": up.URL}), + }) + get(t, "http://"+inst.addrs["stripe"]+"/v1/a") + get(t, "http://"+inst.addrs["stripe"]+"/v1/b") + if err := inst.stop(context.Background()); err != nil { + t.Fatal(err) + } + + var rec recording + b, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &rec); err != nil { + t.Fatalf("the file is not a recording: %v", err) + } + if len(rec.Events) != 2 || rec.Last != 2 { + t.Fatalf("want both calls in the file: %+v", rec) + } + if entries, _ := os.ReadDir(filepath.Dir(out)); len(entries) != 1 { + t.Fatalf("a temporary file was left behind: %v", entries) + } +} + +func TestStopWithoutOutWritesNothing(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + up := upstream(t) + inst := startProxy(t, config{ + proxies: []proxySpec{{"stripe", "127.0.0.1:0"}}, + keep: 100, + getenv: env(map[string]string{"STRIPE_EMULATOR_URL": up.URL}), + }) + if err := inst.stop(context.Background()); err != nil { + t.Fatal(err) + } + if entries, _ := os.ReadDir(dir); len(entries) != 0 { + t.Fatalf("wrote files nobody asked for: %v", entries) + } +} + +func TestAnUnreachableAppEmulatorIsRecordedAsAFailedCall(t *testing.T) { + inst := startProxy(t, config{ + proxies: []proxySpec{{"xero", "127.0.0.1:0"}}, + keep: 100, + getenv: env(map[string]string{"XERO_EMULATOR_URL": "http://127.0.0.1:1"}), + }) + resp, err := http.Get("http://" + inst.addrs["xero"] + "/x") + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusBadGateway { + t.Fatalf("want 502, got %d", resp.StatusCode) + } + evs := inst.rec.Events() + if len(evs) != 1 || evs[0].Error == "" { + t.Fatalf("the failure should be recorded: %+v", evs) + } +} + +func TestBadEventsCursorIs400(t *testing.T) { + up := upstream(t) + inst := startProxy(t, config{ + proxies: []proxySpec{{"stripe", "127.0.0.1:0"}}, + events: "127.0.0.1:0", + keep: 100, + getenv: env(map[string]string{"STRIPE_EMULATOR_URL": up.URL}), + }) + resp, err := http.Get("http://" + inst.eventsAddr + "/events?since=abc") + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != 400 { + t.Fatalf("want 400, got %d", resp.StatusCode) + } +} + +func TestStartFailsAndWritesNothingWhenAPortIsTaken(t *testing.T) { + taken, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer taken.Close() + out := filepath.Join(t.TempDir(), "calls.json") + _, err = start(config{ + proxies: []proxySpec{{"stripe", "127.0.0.1:0"}, {"plaid", taken.Addr().String()}}, + out: out, + keep: 100, + getenv: env(nil), + }) + if err == nil || !strings.Contains(err.Error(), "plaid") { + t.Fatalf("want an error naming plaid, got %v", err) + } + if _, statErr := os.Stat(out); statErr == nil { + t.Fatal("a failed start must not write the recording") + } +} + +func TestRunRejectsBadArguments(t *testing.T) { + tests := []struct { + name string + args []string + want string + }{ + {"no proxy", nil, "usage:"}, + {"a proxy without an address", []string{"--proxy", "stripe"}, "vendor=addr"}, + {"the same vendor twice", []string{"--proxy", "stripe=127.0.0.1:0", "--proxy", "stripe=127.0.0.1:0"}, "twice"}, + {"a stray argument", []string{"--proxy", "stripe=127.0.0.1:0", "extra"}, "usage:"}, + {"keep must be positive", []string{"--proxy", "stripe=127.0.0.1:0", "--keep", "0"}, "usage:"}, + {"an unknown flag", []string{"--nope"}, "usage:"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var out, errb bytes.Buffer + if code := run(context.Background(), tc.args, env(nil), &out, &errb); code != 2 { + t.Fatalf("exit %d, want 2", code) + } + if !strings.Contains(errb.String(), tc.want) { + t.Fatalf("stderr %q should contain %q", errb.String(), tc.want) + } + }) + } +} + +func TestRunServesUntilStoppedThenWritesOut(t *testing.T) { + up := upstream(t) + out := filepath.Join(t.TempDir(), "calls.json") + ctx, cancel := context.WithCancel(context.Background()) + var stdout, stderr syncBuffer + done := make(chan int, 1) + go func() { + done <- run(ctx, []string{"--proxy", "stripe=127.0.0.1:0", "--out", out}, env(map[string]string{"STRIPE_EMULATOR_URL": up.URL}), &stdout, &stderr) + }() + + var addr string + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + // The first line is "stripe http://ADDR -> local URL". + if f := strings.Fields(stdout.String()); len(f) >= 2 { + addr = strings.TrimPrefix(f[1], "http://") + break + } + time.Sleep(10 * time.Millisecond) + } + if addr == "" { + t.Fatalf("never announced where it listens; stderr: %s", stderr.String()) + } + get(t, "http://"+addr+"/v1/a") + cancel() + + select { + case code := <-done: + if code != 0 { + t.Fatalf("exit %d: %s", code, stderr.String()) + } + case <-time.After(10 * time.Second): + t.Fatal("did not stop") + } + b, err := os.ReadFile(out) + if err != nil || !strings.Contains(string(b), `"/v1/a"`) { + t.Fatalf("recording not written: %v\n%s", err, b) + } + if !strings.Contains(stdout.String(), "wrote 1 calls to") { + t.Fatalf("stdout: %s", stdout.String()) + } +} diff --git a/shared/cmd/wire-proxy/syncbuf_test.go b/shared/cmd/wire-proxy/syncbuf_test.go new file mode 100644 index 0000000..2ff10fd --- /dev/null +++ b/shared/cmd/wire-proxy/syncbuf_test.go @@ -0,0 +1,24 @@ +package main + +import ( + "strings" + "sync" +) + +// syncBuffer is a buffer a test can read while run is still writing to it. +type syncBuffer struct { + mu sync.Mutex + b strings.Builder +} + +func (s *syncBuffer) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.Write(p) +} + +func (s *syncBuffer) String() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.String() +} diff --git a/shared/gaps/gaps.go b/shared/gaps/gaps.go new file mode 100644 index 0000000..e4b0ff2 --- /dev/null +++ b/shared/gaps/gaps.go @@ -0,0 +1,313 @@ +// Package gaps says which of the calls an app made are not covered by the app +// emulators it was run against. +// +// Coverage comes from each app emulator's twin-manifest.json, in +// WonderTwin-AI/wondertwin, under coverage.resources_implemented. It cannot +// come from a response's status: Stripe's app emulator answers a call it lacks +// with a 404 that copies the real Stripe's answer for a URL that does not +// exist, so a 404 does not say which of the two it is. +// +// The manifests do not share a format, so each entry is read as one of three +// kinds and a call is matched the way its kind allows: +// +// "PUT /api.xro/2.0/Accounts" a route: method and path ({} matches one segment) +// "item/get" a path: any method +// "payment_intents" a resource name: a path's first resource, any method +// +// A route answers for the call exactly. A path cannot tell methods apart. A +// resource name cannot tell operations apart, so "covered" means only that the +// app emulator implements some of that resource. +package gaps + +import ( + "encoding/json" + "fmt" + "os" + "regexp" + "sort" + "strings" + + "github.com/localstack-samples/localstack-app-emulators/shared/wire" +) + +// Coverage is whether an app emulator covers a call. +type Coverage string + +const ( + Covered Coverage = "covered" + Gap Coverage = "gap" // the app emulator's manifest does not list it + Unchecked Coverage = "unchecked" // no manifest was given for the vendor +) + +// Basis is how precisely a manifest can answer for a call. +type Basis string + +const ( + BasisRoute Basis = "route" // method and path + BasisPath Basis = "path" // path, any method + BasisResource Basis = "resource" // resource name only + BasisNone Basis = "" +) + +// Manifest is the coverage of one vendor's app emulator. +type Manifest struct { + routes []route + paths [][]string + resources map[string]bool +} + +type route struct { + method string + segs []string +} + +// LoadManifest reads coverage.resources_implemented from a twin-manifest.json. +func LoadManifest(path string) (Manifest, error) { + raw, err := os.ReadFile(path) + if err != nil { + return Manifest{}, err + } + var doc struct { + Coverage struct { + Implemented []string `json:"resources_implemented"` + } `json:"coverage"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + return Manifest{}, fmt.Errorf("%s: %w", path, err) + } + if len(doc.Coverage.Implemented) == 0 { + return Manifest{}, fmt.Errorf("%s has no coverage.resources_implemented", path) + } + return ParseManifest(doc.Coverage.Implemented), nil +} + +var methodPrefix = regexp.MustCompile(`^([A-Z]+) (/.*)$`) + +// ParseManifest reads a list of resources_implemented entries. +func ParseManifest(entries []string) Manifest { + m := Manifest{resources: map[string]bool{}} + for _, e := range entries { + e = strings.TrimSpace(e) + switch { + case methodPrefix.MatchString(e): + g := methodPrefix.FindStringSubmatch(e) + m.routes = append(m.routes, route{method: g[1], segs: segments(g[2])}) + case strings.Contains(e, "/"): + m.paths = append(m.paths, segments(e)) + case e != "": + m.resources[e] = true + } + } + return m +} + +func segments(p string) []string { + p = strings.Trim(p, "/") + if p == "" { + return nil + } + return strings.Split(p, "/") +} + +func equalSegs(pattern, got []string) bool { + if len(pattern) != len(got) { + return false + } + for i := range pattern { + if pattern[i] != "{}" && pattern[i] != got[i] { + return false + } + } + return true +} + +// Covers reports whether the manifest covers a call, and how precisely it can +// say so. The path must not include a query string. +func (m Manifest) Covers(method, path string) (bool, Basis) { + segs := segments(path) + for _, r := range m.routes { + if r.method == method && equalSegs(r.segs, segs) { + return true, BasisRoute + } + } + for _, p := range m.paths { + if equalSegs(p, segs) { + return true, BasisPath + } + } + for _, name := range resourceNames(segs) { + if m.resources[name] { + return true, BasisResource + } + } + // What the manifest could have answered for, so a gap says how sure it is. + switch { + case len(m.routes) > 0: + return false, BasisRoute + case len(m.paths) > 0: + return false, BasisPath + case len(m.resources) > 0: + return false, BasisResource + } + return false, BasisNone +} + +var ( + versionSeg = regexp.MustCompile(`^v[0-9]+$`) + guid = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + digits = regexp.MustCompile(`^[0-9]+$`) + prefixedID = regexp.MustCompile(`^[a-z]{2,}_[A-Za-z0-9]{6,}$`) + longToken = regexp.MustCompile(`^[A-Za-z0-9]{20,}$`) +) + +// idLike reports whether a path segment looks like an identifier rather than a +// resource name. It is a heuristic. Resource names are lower-case words, and +// the identifiers the vendors issue carry a digit or a capital, so a +// prefixed identifier must too. +func idLike(seg string) bool { + switch { + case guid.MatchString(seg), digits.MatchString(seg): + return true + case prefixedID.MatchString(seg): + return strings.ToLower(seg) != seg || strings.ContainsAny(seg, "0123456789") + case longToken.MatchString(seg): + return true + } + return false +} + +// Normalize replaces each identifier in a path with {}, so calls that differ +// only by identifier are one call. +func Normalize(path string) string { + segs := segments(path) + for i, s := range segs { + if idLike(s) { + segs[i] = "{}" + } + } + return "/" + strings.Join(segs, "/") +} + +// resourceNames lists the names a resource-level manifest could use for a +// path: with the version and identifiers dropped, the first resource, the last +// one and all of them joined by underscores. "/v1/accounts/acct_1/external_accounts" +// gives "accounts", "external_accounts" and "accounts_external_accounts". +func resourceNames(segs []string) []string { + var words []string + for i, s := range segs { + if i == 0 && versionSeg.MatchString(s) { + continue + } + if idLike(s) { + continue + } + words = append(words, s) + } + if len(words) == 0 { + return nil + } + names := []string{words[0], words[len(words)-1], strings.Join(words, "_")} + if len(words) >= 2 { + names = append(names, words[0]+"_"+words[1]) + } + return names +} + +// Call is one distinct call the app made, with how often. +type Call struct { + Vendor string `json:"vendor"` + Method string `json:"method"` + Path string `json:"path"` // identifiers replaced with {} + Count int `json:"count"` + Failed int `json:"failed"` // answered with 4xx or 5xx, or not answered + Coverage Coverage `json:"coverage"` + Basis Basis `json:"basis,omitempty"` +} + +// Report is what a run of an app asked of the app emulators. +type Report struct { + Calls []Call `json:"calls"` + // Versions lists the app emulator releases the calls were made against, per + // vendor, so a report can be reproduced. "unknown" means none was recorded. + Versions map[string][]string `json:"versions"` +} + +// Gaps is the number of distinct calls the app emulators do not cover. +func (r Report) Gaps() int { + n := 0 + for _, c := range r.Calls { + if c.Coverage == Gap { + n++ + } + } + return n +} + +// Build checks each recorded call against its vendor's manifest. A vendor with +// no manifest is Unchecked, never a gap: the report does not guess. +func Build(events []wire.Event, manifests map[string]Manifest) Report { + type key struct{ vendor, method, path string } + calls := map[key]*Call{} + versions := map[string]map[string]bool{} + + for _, e := range events { + path := e.Path + if i := strings.IndexByte(path, '?'); i >= 0 { + path = path[:i] + } + k := key{e.Vendor, e.Method, Normalize(path)} + c, ok := calls[k] + if !ok { + c = &Call{Vendor: e.Vendor, Method: e.Method, Path: k.path, Coverage: Unchecked} + if m, have := manifests[e.Vendor]; have { + covered, basis := m.Covers(e.Method, path) + c.Basis = basis + c.Coverage = Gap + if covered { + c.Coverage = Covered + } + } + calls[k] = c + } + c.Count++ + if e.Error != "" || e.Status >= 400 { + c.Failed++ + } + + if versions[e.Vendor] == nil { + versions[e.Vendor] = map[string]bool{} + } + v := e.EmulatorVersion + if v == "" { + v = "unknown" + } + versions[e.Vendor][v] = true + } + + r := Report{Versions: map[string][]string{}} + for _, c := range calls { + r.Calls = append(r.Calls, *c) + } + // Gaps first, since they are what the report is for. + order := map[Coverage]int{Gap: 0, Unchecked: 1, Covered: 2} + sort.Slice(r.Calls, func(i, j int) bool { + a, b := r.Calls[i], r.Calls[j] + if order[a.Coverage] != order[b.Coverage] { + return order[a.Coverage] < order[b.Coverage] + } + if a.Vendor != b.Vendor { + return a.Vendor < b.Vendor + } + if a.Path != b.Path { + return a.Path < b.Path + } + return a.Method < b.Method + }) + for vendor, set := range versions { + for v := range set { + r.Versions[vendor] = append(r.Versions[vendor], v) + } + sort.Strings(r.Versions[vendor]) + } + return r +} diff --git a/shared/gaps/gaps_test.go b/shared/gaps/gaps_test.go new file mode 100644 index 0000000..7f1e5e0 --- /dev/null +++ b/shared/gaps/gaps_test.go @@ -0,0 +1,209 @@ +package gaps + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/localstack-samples/localstack-app-emulators/shared/wire" +) + +// The three manifest formats seen in WonderTwin-AI/wondertwin. +var ( + xero = ParseManifest([]string{"GET /api.xro/2.0/Accounts", "PUT /api.xro/2.0/Accounts", "GET /api.xro/2.0/Invoices/{}"}) + plaid = ParseManifest([]string{"item/get", "sandbox/public_token/create"}) + stripe = ParseManifest([]string{"customers", "payment_intents", "external_accounts"}) +) + +func TestCoversByRoute(t *testing.T) { + tests := []struct { + method, path string + covered bool + }{ + {"GET", "/api.xro/2.0/Accounts", true}, + {"PUT", "/api.xro/2.0/Accounts", true}, + {"POST", "/api.xro/2.0/Accounts", false}, // same path, a method the manifest lacks + {"GET", "/api.xro/2.0/Invoices/8d1f1b5e-35b4-4e4d-9f8e-0d8b3f6a7c11", true}, + {"GET", "/api.xro/2.0/Invoices", false}, // the list is a different route from one invoice + {"GET", "/api.xro/2.0/NotAThing", false}, + } + for _, tc := range tests { + got, basis := xero.Covers(tc.method, tc.path) + if got != tc.covered || basis != BasisRoute { + t.Errorf("%s %s: covered=%v basis=%q, want covered=%v basis=route", tc.method, tc.path, got, basis, tc.covered) + } + } +} + +func TestCoversByPathIgnoresTheMethod(t *testing.T) { + for _, method := range []string{"GET", "POST"} { + if got, basis := plaid.Covers(method, "/item/get"); !got || basis != BasisPath { + t.Errorf("%s /item/get: covered=%v basis=%q", method, got, basis) + } + } + if got, basis := plaid.Covers("POST", "/accounts/get"); got || basis != BasisPath { + t.Errorf("an unlisted path should be an uncovered path-level answer: covered=%v basis=%q", got, basis) + } +} + +func TestCoversByResourceName(t *testing.T) { + tests := []struct { + path string + covered bool + }{ + {"/v1/payment_intents", true}, + {"/v1/payment_intents/pi_3Nq8Z2LkdIwHu7ix", true}, + {"/v1/payment_intents/pi_3Nq8Z2LkdIwHu7ix/confirm", true}, + {"/v1/customers", true}, + // The last resource counts, so a nested one is found by its own name. + {"/v1/accounts/acct_1Nq8Z2LkdIwHu7ix/external_accounts", true}, + {"/v1/radar/value_lists", false}, + {"/v1/refunds", false}, + } + for _, tc := range tests { + got, basis := stripe.Covers("GET", tc.path) + if got != tc.covered || basis != BasisResource { + t.Errorf("%s: covered=%v basis=%q, want covered=%v basis=resource", tc.path, got, basis, tc.covered) + } + } +} + +func TestResourceNamesDoNotMistakeAnUnderscoredNameForAnID(t *testing.T) { + // "payment_intents" has the shape of a prefixed ID, but IDs carry a digit or + // a capital and resource names are lower-case words. + for _, name := range []string{"payment_intents", "external_accounts", "financial_connections", "public_token"} { + if idLike(name) { + t.Errorf("%q is a resource name, not an ID", name) + } + } + for _, id := range []string{"acct_3BsYdLdigjpvPgKzcNdCWOQl", "ch_q7BU0ZeG73uY827vdkPgZAap", "8d1f1b5e-35b4-4e4d-9f8e-0d8b3f6a7c11", "12345", "pi_abc123"} { + if !idLike(id) { + t.Errorf("%q is an ID", id) + } + } +} + +func TestNormalizeReplacesIdentifiers(t *testing.T) { + tests := map[string]string{ + "/v1/charges": "/v1/charges", + "/v1/accounts/acct_3BsYdLdigjpvPgKzcNdCWOQl": "/v1/accounts/{}", + "/api.xro/2.0/Invoices/8d1f1b5e-35b4-4e4d-9f8e-0d8b3f6a7c11": "/api.xro/2.0/Invoices/{}", + "/item/get": "/item/get", + } + for in, want := range tests { + if got := Normalize(in); got != want { + t.Errorf("Normalize(%q) = %q, want %q", in, got, want) + } + } +} + +func ev(vendor, method, path string, status int, version string) wire.Event { + return wire.Event{Vendor: vendor, Method: method, Path: path, Status: status, EmulatorVersion: version} +} + +func TestBuildGroupsCallsAndMarksGaps(t *testing.T) { + events := []wire.Event{ + ev("xero", "PUT", "/api.xro/2.0/Accounts", 200, "aaaaaaaaaaaa"), + ev("xero", "POST", "/api.xro/2.0/Accounts", 405, "aaaaaaaaaaaa"), + ev("stripe", "GET", "/v1/charges?limit=100", 200, ""), + ev("stripe", "GET", "/v1/charges?limit=100&starting_after=ch_q7BU0ZeG73uY827vdkPgZAap", 200, ""), + ev("stripe", "GET", "/v1/accounts/acct_3BsYdLdigjpvPgKzcNdCWOQl", 200, ""), + ev("stripe", "GET", "/v1/accounts/acct_9ZxYdLdigjpvPgKzcNdCWOQl", 404, ""), + ev("plaid", "POST", "/item/get", 200, ""), // no plaid manifest was given + } + r := Build(events, map[string]Manifest{"xero": xero, "stripe": stripe}) + + got := map[string]Call{} + for _, c := range r.Calls { + got[c.Vendor+":"+c.Method+" "+c.Path] = c + } + // Two charge reads differ only by cursor and query, so they are one call. + if c := got["stripe:GET /v1/charges"]; c.Count != 2 || c.Coverage != Gap { + t.Errorf("charges: %+v", c) + } + // Two accounts differ only by ID, and one of them failed. + if c := got["stripe:GET /v1/accounts/{}"]; c.Count != 2 || c.Failed != 1 || c.Coverage != Gap { + t.Errorf("accounts: %+v", c) + } + if c := got["xero:PUT /api.xro/2.0/Accounts"]; c.Coverage != Covered || c.Basis != BasisRoute { + t.Errorf("a covered route: %+v", c) + } + if c := got["xero:POST /api.xro/2.0/Accounts"]; c.Coverage != Gap || c.Failed != 1 { + t.Errorf("a method the manifest lacks: %+v", c) + } + if c := got["plaid:POST /item/get"]; c.Coverage != Unchecked || c.Basis != BasisNone { + t.Errorf("a vendor with no manifest must be unchecked, not a gap: %+v", c) + } + // customers and payment_intents are listed, but charges and accounts are not. + if r.Gaps() != 3 { + t.Errorf("want 3 gaps, got %d: %+v", r.Gaps(), r.Calls) + } + // Gaps come first. + if r.Calls[0].Coverage != Gap || r.Calls[len(r.Calls)-1].Coverage == Gap { + t.Errorf("gaps should sort first: %+v", r.Calls) + } + if v := r.Versions["xero"]; len(v) != 1 || v[0] != "aaaaaaaaaaaa" { + t.Errorf("xero versions: %v", v) + } + if v := r.Versions["stripe"]; len(v) != 1 || v[0] != "unknown" { + t.Errorf("a missing version is reported as unknown: %v", v) + } +} + +func TestAnErrorCountsAsFailedEvenWithoutAStatus(t *testing.T) { + e := wire.Event{Vendor: "xero", Method: "GET", Path: "/api.xro/2.0/Accounts", Error: "connection refused"} + r := Build([]wire.Event{e}, map[string]Manifest{"xero": xero}) + if r.Calls[0].Failed != 1 { + t.Fatalf("%+v", r.Calls[0]) + } +} + +func TestLoadManifest(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return p + } + m, err := LoadManifest(write("ok.json", `{"coverage":{"resources_implemented":["GET /a/{}"]}}`)) + if err != nil { + t.Fatal(err) + } + if got, _ := m.Covers("GET", "/a/1"); !got { + t.Fatal("the loaded manifest should cover GET /a/1") + } + for name, body := range map[string]string{ + "empty.json": `{"coverage":{"resources_implemented":[]}}`, + "missing.json": `{}`, + "bad.json": `{nope`, + } { + if _, err := LoadManifest(write(name, body)); err == nil { + t.Errorf("%s should be rejected: an empty manifest would make every call a gap", name) + } + } + if _, err := LoadManifest(filepath.Join(dir, "absent.json")); err == nil { + t.Error("a missing file should be an error") + } +} + +func TestWritePrintsGapsVersionsAndTheCaveat(t *testing.T) { + r := Build([]wire.Event{ + ev("xero", "POST", "/api.xro/2.0/Accounts", 405, "aaaaaaaaaaaa"), + ev("xero", "PUT", "/api.xro/2.0/Accounts", 200, "aaaaaaaaaaaa"), + }, map[string]Manifest{"xero": xero}) + var b bytes.Buffer + Write(&b, r) + out := b.String() + for _, want := range []string{"gap", "xero:POST /api.xro/2.0/Accounts", "covered", "2 distinct calls, 1 not covered", "xero app emulator: aaaaaaaaaaaa", "resource-level"} { + if !strings.Contains(out, want) { + t.Errorf("want %q in:\n%s", want, out) + } + } + if strings.Index(out, "gap") > strings.Index(out, "covered") { + t.Errorf("gaps should be listed before covered calls:\n%s", out) + } +} diff --git a/shared/gaps/render.go b/shared/gaps/render.go new file mode 100644 index 0000000..a1f5aea --- /dev/null +++ b/shared/gaps/render.go @@ -0,0 +1,31 @@ +package gaps + +import ( + "fmt" + "io" + "sort" + "strings" + "text/tabwriter" +) + +// Write prints the report as a table, gaps first, then the app emulator +// releases it was made against. +func Write(w io.Writer, r Report) { + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, "COVERAGE\tCALL\tCOUNT\tFAILED\tBASIS") + for _, c := range r.Calls { + fmt.Fprintf(tw, "%s\t%s:%s %s\t%d\t%d\t%s\n", c.Coverage, c.Vendor, c.Method, c.Path, c.Count, c.Failed, c.Basis) + } + tw.Flush() + + fmt.Fprintf(w, "\n%d distinct calls, %d not covered\n", len(r.Calls), r.Gaps()) + vendors := make([]string, 0, len(r.Versions)) + for v := range r.Versions { + vendors = append(vendors, v) + } + sort.Strings(vendors) + for _, v := range vendors { + fmt.Fprintf(w, "%s app emulator: %s\n", v, strings.Join(r.Versions[v], ", ")) + } + fmt.Fprintln(w, "A resource-level basis cannot tell which operations of a resource are implemented.") +} diff --git a/shared/samples/samples.go b/shared/samples/samples.go new file mode 100644 index 0000000..eee0bfe --- /dev/null +++ b/shared/samples/samples.go @@ -0,0 +1,184 @@ +// Package samples checks the descriptors of sample apps. +// +// A sample is an open source app, written by someone else for real users, that +// is run against the app emulators. This repo records which app, at which +// commit, and how it was pointed at the app emulators. It never holds the app's +// code: the code stays under its own license in its own repo, and a run fetches +// the pinned commit. +package samples + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// File is the name of a sample's descriptor. +const File = "sample.json" + +// Descriptor says which app a sample is and how to run it against the app +// emulators. +type Descriptor struct { + Name string `json:"name"` // the directory name + Repo string `json:"repo"` // https URL of the app's repo + Commit string `json:"commit"` // the full commit SHA that was run + License string `json:"license"` // the app's SPDX license identifier + Vendors []string `json:"vendors"` // the vendors whose app emulators it is run against + // Overrides says, for each vendor, how the app is pointed at an app emulator: + // the environment variable or setting that holds the vendor's base URL. + Overrides map[string]string `json:"overrides"` + Run string `json:"run"` // the command that runs the app or its tests, from the app's checkout +} + +var ( + slug = regexp.MustCompile(`^[a-z][a-z0-9-]*$`) + sha = regexp.MustCompile(`^[0-9a-f]{40}$`) + spdx = regexp.MustCompile(`^[A-Za-z0-9.+-]+( (AND|OR|WITH) [A-Za-z0-9.+-]+)*$`) + repoURL = regexp.MustCompile(`^https://[a-z0-9.-]+/[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$`) +) + +// Validate lists what is wrong with d, given the directory name it sits in. +func (d Descriptor) Validate(dir string) []string { + var p []string + add := func(format string, args ...any) { p = append(p, fmt.Sprintf(format, args...)) } + + if d.Name != dir { + add("name %q must be the directory name %q", d.Name, dir) + } + if !slug.MatchString(dir) { + add("directory name %q must be lower-case words joined by hyphens", dir) + } + if !repoURL.MatchString(d.Repo) { + add("repo %q must be an https URL of the form https://host/owner/name", d.Repo) + } + if !sha.MatchString(d.Commit) { + add("commit %q must be a full 40-character lower-case SHA, not a branch or a tag", d.Commit) + } + if !validLicense(d.License) { + add("license %q must be an SPDX identifier such as MIT or Apache-2.0", d.License) + } + if len(d.Vendors) == 0 { + add("vendors must name at least one vendor") + } + seen := map[string]bool{} + for _, v := range d.Vendors { + if !slug.MatchString(v) { + add("vendor %q must be lower-case words joined by hyphens", v) + } + if seen[v] { + add("vendor %q is listed twice", v) + } + seen[v] = true + if strings.TrimSpace(d.Overrides[v]) == "" { + add("overrides must say how %s is pointed at its app emulator", v) + } + } + for v := range d.Overrides { + if !seen[v] { + add("overrides names %q, which is not in vendors", v) + } + } + if strings.TrimSpace(d.Run) == "" { + add("run must be the command that runs the app or its tests") + } + sort.Strings(p) + return p +} + +// validLicense accepts an SPDX identifier or expression. The operators are not +// identifiers, so "OR" on its own is not a license. +func validLicense(l string) bool { + if !spdx.MatchString(l) { + return false + } + switch l { + case "AND", "OR", "WITH": + return false + } + return true +} + +// allowed is what a sample's directory may hold. Anything else is app code or +// output that belongs elsewhere. +func allowed(name string, isDir bool) bool { + switch { + case name == File || name == "README.md": + return !isDir + case name == "results": + return isDir + } + return false +} + +// Problem is one thing wrong in the samples tree. +type Problem struct { + Sample string // the sample's directory name, or "" for the tree itself + Detail string +} + +func (p Problem) String() string { + if p.Sample == "" { + return p.Detail + } + return p.Sample + ": " + p.Detail +} + +// Check validates every sample under root. A sample is a directory holding +// anything at all, so one with no descriptor is a problem, not an omission. +func Check(root string) ([]Problem, error) { + entries, err := os.ReadDir(root) + if err != nil { + return nil, err + } + var out []Problem + for _, e := range entries { + if !e.IsDir() { + if e.Name() != "README.md" { + out = append(out, Problem{Detail: fmt.Sprintf("%s is not a sample directory", e.Name())}) + } + continue + } + out = append(out, checkSample(root, e.Name())...) + } + return out, nil +} + +func checkSample(root, name string) []Problem { + dir := filepath.Join(root, name) + var out []Problem + add := func(format string, args ...any) { + out = append(out, Problem{Sample: name, Detail: fmt.Sprintf(format, args...)}) + } + + entries, err := os.ReadDir(dir) + if err != nil { + add("cannot read: %v", err) + return out + } + for _, e := range entries { + if !allowed(e.Name(), e.IsDir()) { + add("%s does not belong here. A sample holds %s, README.md and results/, never the app's code", e.Name(), File) + } + } + + raw, err := os.ReadFile(filepath.Join(dir, File)) + if err != nil { + add("has no %s", File) + return out + } + var d Descriptor + dec := json.NewDecoder(strings.NewReader(string(raw))) + dec.DisallowUnknownFields() + if err := dec.Decode(&d); err != nil { + add("%s is not valid: %v", File, err) + return out + } + for _, detail := range d.Validate(name) { + add("%s", detail) + } + return out +} diff --git a/shared/samples/samples_test.go b/shared/samples/samples_test.go new file mode 100644 index 0000000..2b9f89f --- /dev/null +++ b/shared/samples/samples_test.go @@ -0,0 +1,174 @@ +package samples + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func good() Descriptor { + return Descriptor{ + Name: "medusa-checkout", + Repo: "https://github.com/example/shop", + Commit: strings.Repeat("a", 40), + License: "MIT", + Vendors: []string{"stripe"}, + Overrides: map[string]string{"stripe": "STRIPE_API_BASE"}, + Run: "npm test", + } +} + +func TestAGoodDescriptorHasNoProblems(t *testing.T) { + if p := good().Validate("medusa-checkout"); len(p) != 0 { + t.Fatalf("unexpected problems: %v", p) + } +} + +func TestValidateNamesEachProblem(t *testing.T) { + tests := []struct { + name string + mutate func(*Descriptor) + dir string + want string + }{ + {"name must match the directory", func(d *Descriptor) { d.Name = "other" }, "medusa-checkout", `must be the directory name`}, + {"directory is a slug", func(d *Descriptor) { d.Name = "Bad_Name" }, "Bad_Name", `lower-case words`}, + {"repo is https", func(d *Descriptor) { d.Repo = "git@github.com:example/shop.git" }, "medusa-checkout", `https URL`}, + {"a branch is not a commit", func(d *Descriptor) { d.Commit = "main" }, "medusa-checkout", `full 40-character`}, + {"a short SHA is not a commit", func(d *Descriptor) { d.Commit = "abc1234" }, "medusa-checkout", `full 40-character`}, + {"an upper-case SHA is rejected", func(d *Descriptor) { d.Commit = strings.Repeat("A", 40) }, "medusa-checkout", `full 40-character`}, + {"license is required", func(d *Descriptor) { d.License = "" }, "medusa-checkout", `SPDX`}, + {"vendors are required", func(d *Descriptor) { d.Vendors, d.Overrides = nil, nil }, "medusa-checkout", `at least one vendor`}, + {"every vendor needs an override", func(d *Descriptor) { d.Overrides = map[string]string{} }, "medusa-checkout", `how stripe is pointed`}, + {"a blank override does not count", func(d *Descriptor) { d.Overrides["stripe"] = " " }, "medusa-checkout", `how stripe is pointed`}, + {"an override needs a vendor", func(d *Descriptor) { d.Overrides["plaid"] = "PLAID_HOST" }, "medusa-checkout", `"plaid", which is not in vendors`}, + {"a vendor twice", func(d *Descriptor) { d.Vendors = []string{"stripe", "stripe"} }, "medusa-checkout", `listed twice`}, + {"run is required", func(d *Descriptor) { d.Run = "" }, "medusa-checkout", `command that runs`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + d := good() + tc.mutate(&d) + got := strings.Join(d.Validate(tc.dir), "\n") + if !strings.Contains(got, tc.want) { + t.Fatalf("want a problem containing %q, got:\n%s", tc.want, got) + } + }) + } +} + +func TestLicenseAcceptsSPDXExpressions(t *testing.T) { + for _, l := range []string{"MIT", "Apache-2.0", "GPL-3.0-or-later", "MIT OR Apache-2.0"} { + d := good() + d.License = l + if p := d.Validate("medusa-checkout"); len(p) != 0 { + t.Errorf("%q should be accepted: %v", l, p) + } + } + for _, l := range []string{"see LICENSE file", "MIT;rm", "OR"} { + d := good() + d.License = l + if p := d.Validate("medusa-checkout"); len(p) == 0 { + t.Errorf("%q should be rejected", l) + } + } +} + +func write(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func descriptorJSON(t *testing.T, d Descriptor) string { + t.Helper() + b, err := json.Marshal(d) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestCheckAcceptsAWellFormedTree(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "README.md"), "# samples") + write(t, filepath.Join(root, "medusa-checkout", File), descriptorJSON(t, good())) + write(t, filepath.Join(root, "medusa-checkout", "README.md"), "notes") + write(t, filepath.Join(root, "medusa-checkout", "results", "2026-11-04.json"), "{}") + if p, err := Check(root); err != nil || len(p) != 0 { + t.Fatalf("err=%v problems=%v", err, p) + } +} + +func TestCheckAcceptsNoSamples(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "README.md"), "# samples") + if p, err := Check(root); err != nil || len(p) != 0 { + t.Fatalf("an empty section is fine: err=%v problems=%v", err, p) + } +} + +// The reason a sample holds no app code is the app's license, so a checkout +// that slips in must fail the check. +func TestCheckRejectsAppCodeInASample(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "medusa-checkout", File), descriptorJSON(t, good())) + write(t, filepath.Join(root, "medusa-checkout", "src", "app.ts"), "export {}") + write(t, filepath.Join(root, "medusa-checkout", "package.json"), "{}") + p, err := Check(root) + if err != nil { + t.Fatal(err) + } + var got []string + for _, x := range p { + got = append(got, x.String()) + } + joined := strings.Join(got, "\n") + for _, want := range []string{"src does not belong here", "package.json does not belong here"} { + if !strings.Contains(joined, want) { + t.Errorf("want %q in:\n%s", want, joined) + } + } +} + +func TestCheckReportsMissingAndMalformedDescriptors(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "no-descriptor", "README.md"), "x") + write(t, filepath.Join(root, "bad-json", File), "{not json") + write(t, filepath.Join(root, "extra-field", File), `{"name":"extra-field","surprise":1}`) + p, err := Check(root) + if err != nil { + t.Fatal(err) + } + var got []string + for _, x := range p { + got = append(got, x.String()) + } + joined := strings.Join(got, "\n") + for _, want := range []string{"no-descriptor: has no sample.json", "bad-json: sample.json is not valid", "extra-field: sample.json is not valid"} { + if !strings.Contains(joined, want) { + t.Errorf("want %q in:\n%s", want, joined) + } + } +} + +func TestCheckRejectsStrayFilesAtTheTop(t *testing.T) { + root := t.TempDir() + write(t, filepath.Join(root, "notes.txt"), "x") + p, _ := Check(root) + if len(p) != 1 || !strings.Contains(p[0].String(), "notes.txt is not a sample directory") { + t.Fatalf("got %v", p) + } +} + +func TestCheckFailsWhenTheRootIsMissing(t *testing.T) { + if _, err := Check(filepath.Join(t.TempDir(), "nope")); err == nil { + t.Fatal("a missing samples directory should be an error") + } +}