Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ jobs:
go build -o /dev/null "./${app}cmd/kit"
done

# Samples are other people's apps, so this repo holds only a descriptor for
# each. A sample with a bad descriptor, or with app code in it, fails here.
- name: Check the sample descriptors
run: make check-samples

# Each demo's UI is built with pnpm and embedded in its Go binary, so this job
# builds the UI, builds the binary, and checks the binary serves it.
web:
Expand Down
2 changes: 1 addition & 1 deletion .no-mistakes.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@
# checks run only where `pnpm install` has been run in web/.
commands:
test: 'set -e; for m in $(go list -m); do go test -count=1 "$m/..."; done; if [ -d web/node_modules ]; then pnpm --dir web test; fi'
lint: 'set -e; for m in $(go list -m); do go vet "$m/..."; done; for a in apps/*/; do [ -d "${a}cmd/kit" ] || { echo "$a has no cmd/kit"; exit 1; }; done; test -z "$(gofmt -l .)" || { echo "unformatted files (run gofmt -w .):"; gofmt -l .; exit 1; }; if [ -d web/node_modules ]; then pnpm --dir web lint && pnpm --dir web type-check; fi'
lint: 'set -e; for m in $(go list -m); do go vet "$m/..."; done; for a in apps/*/; do [ -d "${a}cmd/kit" ] || { echo "$a has no cmd/kit"; exit 1; }; done; (cd shared && go run ./cmd/check-samples ../samples); test -z "$(gofmt -l .)" || { echo "unformatted files (run gofmt -w .):"; gofmt -l .; exit 1; }; if [ -d web/node_modules ]; then pnpm --dir web lint && pnpm --dir web type-check; fi'
format: "gofmt -w ."
6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: build test vet check-kits kit web
.PHONY: build test vet check-kits check-samples kit web

# Every module in the Go workspace is built, vetted and tested. In a workspace,
# ./... from the root matches only the root module, so each module is listed.
Expand All @@ -21,6 +21,10 @@ test:
check-kits:
set -e; for a in apps/*/; do [ -d "$${a}cmd/kit" ] || { echo "$$a has no cmd/kit (every demo needs an offline kit)"; exit 1; }; done

# Every sample app's descriptor is well formed, and no sample holds app code.
check-samples:
cd shared && go run ./cmd/check-samples ../samples

# Builds a demo's web UI and copies it into the Go binary's embedded assets.
# Run it before `make build` to get the real UI instead of the placeholder.
# make web APP=acme-commerce-lenders
Expand Down
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ Code and docs: [apps/acme-commerce-lenders](apps/acme-commerce-lenders)

**Apply, offer, accept and settle.** A merchant applies for working capital and receives an offer. They accept it, the advance is funded, and the settlement appears in their Xero books.

## Samples

A sample is an open source app, written by someone else for real users, that is run against the app emulators. A demo is ours and is staged to show well. A sample has the use cases and edge cases real customers hit, so it shows where an app emulator differs from the real vendor. `samples/` records which app was run and how, and never holds the app's code. See [samples](samples).

## How demos run

Each demo is one Go binary, `cmd/kit`, that serves the demo's UI and runs its logic. It runs beside LocalStack, which hosts the app emulators and serves each one on its own host name through LocalStack's gateway, so you start LocalStack with `lstk` first. The demo calls each vendor through a local recording proxy (`shared/wire`), which is how the inspector shows what the demo sends. Each vendor's app emulator is hosted by LocalStack or run by you (`<VENDOR>_EMULATOR_URL`), so you can build against an app emulator locally and move it to LocalStack once LocalStack includes it.
Expand All @@ -30,6 +34,7 @@ Each demo is a Go app under `apps/`. The repo is a Go workspace (`go.work`), so
|---|---|
| `apps/<demo>/` | One demo: its Go code, its docs and its embedded UI |
| `shared/` | Code every demo uses: the recording proxy for vendor calls, vendor base URL config, and a launcher for running app emulators without LocalStack |
| `samples/<sample>/` | One sample: a descriptor of the open source app and the output of runs, never the app's code |
| `web/` | pnpm workspace for the demo UIs, built as static assets that the Go binaries embed |
| `templates/demo-docs/` | The three docs every demo carries: Usage Guide, Demo Scenarios, Troubleshooting Guide |

Expand All @@ -42,6 +47,7 @@ make web APP=<demo> # builds that demo's UI into its binary (needs No
make build # builds every apps/<demo>/cmd/<name> to bin/<demo>-<name>
make kit APP=<demo> # builds, then runs that demo's binary
make check-kits # fails if a demo has no cmd/kit
make check-samples # fails if a sample's descriptor is malformed or a sample holds app code
```

`go test -tags live ./wire` in `shared/` also runs a check against a LocalStack you have running.
81 changes: 81 additions & 0 deletions samples/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
# Samples

A sample is an open source app, written by someone else for real users, that is run against the app emulators. It has the use cases and edge cases real customers hit, and its own tests, which we did not invent. The demos in `apps/` are different: they are staged to show well, and they are ours.

This directory records which app was run and how. It never holds the app's code. The code stays in its own repo under its own license, and a run fetches the commit named here.

## What a sample holds

```
samples/<name>/
sample.json which app, at which commit, and how it is run
README.md optional notes
results/ the output of runs (JSON)
```

Nothing else. `make check-samples` fails on any other file.

## sample.json

```json
{
"name": "example-shop",
"repo": "https://github.com/example/shop",
"commit": "0123456789abcdef0123456789abcdef01234567",
"license": "MIT",
"vendors": ["stripe"],
"overrides": { "stripe": "STRIPE_API_BASE" },
"run": "npm test"
}
```

This is an illustration, not a real sample.

| Field | Meaning |
|---|---|
| `name` | The directory name. Lower-case words joined by hyphens. |
| `repo` | The https URL of the app's repo. |
| `commit` | The full 40-character SHA that was run. A branch or tag is rejected, because a sample must be reproducible. |
| `license` | The app's SPDX identifier, such as `MIT` or `Apache-2.0`. |
| `vendors` | The vendors whose app emulators it is run against. |
| `overrides` | For each vendor, the environment variable or setting that holds its base URL. |
| `run` | The command that runs the app or its tests, from the app's checkout. |

## Checking

```sh
make check-samples
```

## Recording a run

Two commands in `shared/` record what an app asks of the app emulators and report what they do not cover.

**1. Record.** `wire-proxy` listens on one port per vendor and forwards to that vendor's app emulator, the one at `<VENDOR>_EMULATOR_URL` if set, otherwise the one LocalStack serves. Point the app's vendor base URL at the listening address, using the `overrides` in `sample.json`:

```sh
cd shared
STRIPE_EMULATOR_VERSION=<release id> go run ./cmd/wire-proxy \
--proxy stripe=127.0.0.1:5301 --proxy plaid=127.0.0.1:5302 \
--events 127.0.0.1:5399 --out ../samples/<name>/results/<date>-calls.json
```

Run the app or its tests, then stop the proxy with Ctrl-C. It writes every call to `--out`, with credentials in bodies masked. While it runs, `curl http://127.0.0.1:5399/events` shows the calls so far. Each call records its `origin` and, if you set `<VENDOR>_EMULATOR_VERSION`, the app emulator's release id. An app emulator does not announce its release, so the id is the one in the release's tag, `twin-<name>-v<id>`.

**2. Report.** `gap-report` checks each call against the app emulators' manifests, which are `twin-<name>/twin-manifest.json` in `WonderTwin-AI/wondertwin`:

```sh
cd shared
go run ./cmd/gap-report --events ../samples/<name>/results/<date>-calls.json \
--manifest stripe=<path>/twin-stripe/twin-manifest.json \
--manifest plaid=<path>/twin-plaid/twin-manifest.json
```

It lists each distinct call, with identifiers in the path replaced by `{}`, as `covered`, `gap` or `unchecked`, and which app emulator release it ran against. `--json` prints the same as JSON. `--fail-on-gap` exits 1 if any call is a gap.

### What the report can and cannot tell you

- **It reads the manifest, not the response.** Stripe's app emulator answers a route it lacks with a 404 that copies the real Stripe, so a status cannot say whether a route is missing.
- **The manifests do not share a format, so precision differs.** Xero lists routes (`PUT /api.xro/2.0/Accounts`), so a covered call is exact. Plaid lists paths (`item/get`), so it cannot tell methods apart. Stripe lists resource names (`payment_intents`), so `covered` means only that some of that resource is implemented. The `BASIS` column says which applies.
- **Identifiers are recognised by their shape.** A segment with a digit or a capital that looks like an id (`acct_3Bs...`, a GUID, a long token) becomes `{}`. An unusual identifier could be reported as its own call.
- **A vendor with no manifest is `unchecked`, never a gap.**
42 changes: 42 additions & 0 deletions shared/cmd/check-samples/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Command check-samples validates the descriptor of every sample app.
//
// check-samples [samples-dir]
//
// It exits 0 when every sample is well formed, 1 when any is not, and 2 when
// the directory cannot be read.
package main

import (
"fmt"
"io"
"os"

"github.com/localstack-samples/localstack-app-emulators/shared/samples"
)

func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }

func run(args []string, stdout, stderr io.Writer) int {
if len(args) > 1 {
fmt.Fprintln(stderr, "usage: check-samples [samples-dir]")
return 2
}
root := "samples"
if len(args) == 1 {
root = args[0]
}
problems, err := samples.Check(root)
if err != nil {
fmt.Fprintf(stderr, "check-samples: %v\n", err)
return 2
}
for _, p := range problems {
fmt.Fprintln(stderr, p)
}
if len(problems) > 0 {
fmt.Fprintf(stderr, "%d problem(s) in %s\n", len(problems), root)
return 1
}
fmt.Fprintf(stdout, "samples in %s are well formed\n", root)
return 0
}
44 changes: 44 additions & 0 deletions shared/cmd/check-samples/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package main

import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
)

func TestRunExitCodes(t *testing.T) {
good := t.TempDir()
bad := t.TempDir()
if err := os.MkdirAll(filepath.Join(bad, "no-descriptor"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(bad, "no-descriptor", "README.md"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}

tests := []struct {
name string
args []string
code int
out string
err string
}{
{"an empty section passes", []string{good}, 0, "well formed", ""},
{"a bad sample fails and is named", []string{bad}, 1, "", "no-descriptor: has no sample.json"},
{"a missing directory is an error", []string{filepath.Join(good, "nope")}, 2, "", "check-samples:"},
{"too many arguments", []string{"a", "b"}, 2, "", "usage:"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var out, errb bytes.Buffer
if got := run(tc.args, &out, &errb); got != tc.code {
t.Fatalf("exit %d, want %d (stderr: %s)", got, tc.code, errb.String())
}
if !strings.Contains(out.String(), tc.out) || !strings.Contains(errb.String(), tc.err) {
t.Fatalf("stdout %q, stderr %q", out.String(), errb.String())
}
})
}
}
96 changes: 96 additions & 0 deletions shared/cmd/gap-report/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
// Command gap-report says which calls an app made that the app emulators do
// not cover.
//
// gap-report --events calls.json --manifest stripe=twin-stripe/twin-manifest.json
//
// calls.json is what wire-proxy --out wrote. Each --manifest is an app
// emulator's twin-manifest.json from WonderTwin-AI/wondertwin. A vendor with no
// manifest is reported as unchecked, not as a gap.
//
// It exits 0 after printing the report. With --fail-on-gap it exits 1 when any
// call is not covered, and 2 when its input cannot be read.
package main

import (
"encoding/json"
"flag"
"fmt"
"io"
"os"
"strings"

"github.com/localstack-samples/localstack-app-emulators/shared/gaps"
"github.com/localstack-samples/localstack-app-emulators/shared/wire"
)

func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }

const usage = `usage: gap-report --events calls.json --manifest vendor=path [--manifest vendor=path ...] [--json] [--fail-on-gap]
`

type manifestFlags map[string]string

func (m manifestFlags) String() string { return "" }

func (m manifestFlags) Set(v string) error {
vendor, path, ok := strings.Cut(v, "=")
if !ok || vendor == "" || path == "" {
return fmt.Errorf("--manifest %q must be vendor=path", v)
}
if _, dup := m[vendor]; dup {
return fmt.Errorf("--manifest names %s twice", vendor)
}
m[vendor] = path
return nil
}

func run(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("gap-report", flag.ContinueOnError)
fs.SetOutput(stderr)
fs.Usage = func() {}
paths := manifestFlags{}
events := fs.String("events", "", "")
asJSON := fs.Bool("json", false, "")
failOnGap := fs.Bool("fail-on-gap", false, "")
fs.Var(paths, "manifest", "")
if err := fs.Parse(args); err != nil || *events == "" || fs.NArg() > 0 {
fmt.Fprint(stderr, usage)
return 2
}

raw, err := os.ReadFile(*events)
if err != nil {
fmt.Fprintf(stderr, "gap-report: %v\n", err)
return 2
}
var rec struct {
Events []wire.Event `json:"events"`
}
if err := json.Unmarshal(raw, &rec); err != nil {
fmt.Fprintf(stderr, "gap-report: %s is not a recording: %v\n", *events, err)
return 2
}

manifests := map[string]gaps.Manifest{}
for vendor, path := range paths {
m, err := gaps.LoadManifest(path)
if err != nil {
fmt.Fprintf(stderr, "gap-report: %s manifest: %v\n", vendor, err)
return 2
}
manifests[vendor] = m
}

report := gaps.Build(rec.Events, manifests)
if *asJSON {
enc := json.NewEncoder(stdout)
enc.SetIndent("", " ")
_ = enc.Encode(report)
} else {
gaps.Write(stdout, report)
}
if *failOnGap && report.Gaps() > 0 {
return 1
}
return 0
}
Loading
Loading