From b0698e0a66a48e370c5e10f22dd3060a20158d74 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Apr 2026 20:37:22 +0000 Subject: [PATCH] test(cli): move tests/ out of tools/, separate marmot vs dm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tools/` is for libraries (arti-build); shell-based interop harnesses that drive the `amy` binary belong with the CLI module. New layout: cli/tests/ ├── lib.sh # shared logging + result tracking ├── headless/helpers.sh # shared amy_a / amy_json / assertions ├── marmot/ # MLS group-messaging interop (vs whitenoise-rs) │ ├── marmot-interop.sh │ ├── marmot-interop-headless.sh │ ├── setup.sh │ ├── tests-create.sh │ ├── tests-manage.sh │ ├── tests-extras.sh │ └── patches/ └── dm/ # NIP-17 DM interop (amy ↔ amy) ├── dm-interop-headless.sh ├── setup.sh └── tests-dm.sh Per-suite changes: - Each suite owns its own setup.sh; previously the Marmot setup was at headless/setup.sh and the DM setup at headless/setup-dm.sh, which implied shared infra they don't actually share. - `cli/tests/lib.sh` and `cli/tests/headless/helpers.sh` are the only shared bits across suites. - The DM harness still reuses Marmot's `start_local_relay` / `stop_local_relay` (sourced via `../marmot/setup.sh`) — same relay binary, no duplication. - All sourcing paths updated; REPO_ROOT now climbs three levels (was two when the harness lived under `tools/`); patches/ is now a sibling of marmot/setup.sh rather than under marmot/headless/. - `.gitignore` updated to cover both suites' state dirs. cli/ROADMAP.md and cli/tests/README.md updated to point at the new paths. No behavioural change — every test runs the same code; only the file layout and source paths moved. --- cli/ROADMAP.md | 10 ++-- cli/tests/.gitignore | 3 ++ {tools/marmot-interop => cli/tests}/README.md | 47 ++++++++++++++----- .../tests/dm}/dm-interop-headless.sh | 34 +++++++------- .../setup-dm.sh => cli/tests/dm/setup.sh | 4 +- .../headless => cli/tests/dm}/tests-dm.sh | 2 +- .../tests}/headless/helpers.sh | 2 +- {tools/marmot-interop => cli/tests}/lib.sh | 0 .../tests/marmot}/marmot-interop-headless.sh | 27 ++++++----- .../tests/marmot}/marmot-interop.sh | 5 +- .../patches/whitenoise-defaults-env.patch | 0 .../patches/whitenoise-discovery-env.patch | 0 .../patches/whitenoise-mock-keyring.patch | 0 .../whitenoise-skip-unprocessable-retry.patch | 0 .../headless => cli/tests/marmot}/setup.sh | 4 +- .../tests/marmot}/tests-create.sh | 2 +- .../tests/marmot}/tests-extras.sh | 2 +- .../tests/marmot}/tests-manage.sh | 2 +- tools/marmot-interop/.gitignore | 2 - 19 files changed, 87 insertions(+), 59 deletions(-) create mode 100644 cli/tests/.gitignore rename {tools/marmot-interop => cli/tests}/README.md (80%) rename {tools/marmot-interop => cli/tests/dm}/dm-interop-headless.sh (70%) rename tools/marmot-interop/headless/setup-dm.sh => cli/tests/dm/setup.sh (96%) rename {tools/marmot-interop/headless => cli/tests/dm}/tests-dm.sh (99%) rename {tools/marmot-interop => cli/tests}/headless/helpers.sh (95%) rename {tools/marmot-interop => cli/tests}/lib.sh (100%) rename {tools/marmot-interop => cli/tests/marmot}/marmot-interop-headless.sh (85%) rename {tools/marmot-interop => cli/tests/marmot}/marmot-interop.sh (99%) rename {tools/marmot-interop/headless => cli/tests/marmot}/patches/whitenoise-defaults-env.patch (100%) rename {tools/marmot-interop/headless => cli/tests/marmot}/patches/whitenoise-discovery-env.patch (100%) rename {tools/marmot-interop/headless => cli/tests/marmot}/patches/whitenoise-mock-keyring.patch (100%) rename {tools/marmot-interop/headless => cli/tests/marmot}/patches/whitenoise-skip-unprocessable-retry.patch (100%) rename {tools/marmot-interop/headless => cli/tests/marmot}/setup.sh (99%) rename {tools/marmot-interop/headless => cli/tests/marmot}/tests-create.sh (99%) rename {tools/marmot-interop/headless => cli/tests/marmot}/tests-extras.sh (99%) rename {tools/marmot-interop/headless => cli/tests/marmot}/tests-manage.sh (99%) delete mode 100644 tools/marmot-interop/.gitignore diff --git a/cli/ROADMAP.md b/cli/ROADMAP.md index 61c227102..8237e15c0 100644 --- a/cli/ROADMAP.md +++ b/cli/ROADMAP.md @@ -95,11 +95,11 @@ move anything, re-audit — you're probably duplicating logic. 8. **Distribution** — Homebrew + Scoop + `.deb` in the same release pipeline as desktop. Plan: `cli/plans/2026-04-21-cli-distribution.md`. 9. **Test suite** — end-to-end against a local relay. Marmot interop is - covered by `tools/marmot-interop/marmot-interop-headless.sh`; NIP-17 - DM interop between two `amy` clients is covered by - `tools/marmot-interop/dm-interop-headless.sh` (text + file + strict - 10050 + fallback + cursor-advance). Neither runs in CI yet (both - need Rust + ~3 min cold relay build). + covered by `cli/tests/marmot/marmot-interop-headless.sh`; NIP-17 DM + interop between two `amy` clients is covered by + `cli/tests/dm/dm-interop-headless.sh` (text + file + strict 10050 + + fallback + cursor-advance). Neither runs in CI yet (both need Rust + + ~3 min cold relay build). 10. **Everything else in the matrix.** --- diff --git a/cli/tests/.gitignore b/cli/tests/.gitignore new file mode 100644 index 000000000..f7bc17b06 --- /dev/null +++ b/cli/tests/.gitignore @@ -0,0 +1,3 @@ +marmot/state/ +marmot/state-headless/ +dm/state-dm-headless/ diff --git a/tools/marmot-interop/README.md b/cli/tests/README.md similarity index 80% rename from tools/marmot-interop/README.md rename to cli/tests/README.md index 2fca06f1b..f544930b9 100644 --- a/tools/marmot-interop/README.md +++ b/cli/tests/README.md @@ -1,22 +1,45 @@ -# Interop Test Harnesses +# amy CLI test harnesses -Two flavours of the Marmot harness, same scenarios: +Shell-based end-to-end harnesses that drive the `amy` CLI binary against a +loopback `nostr-rs-relay`. Layout: -- **`marmot-interop.sh`** — interactive. Drives B/C via `wn` and **prompts the - human** to perform each Amethyst-side step in the mobile UI (Identity A). - Use this for final UI verification. -- **`marmot-interop-headless.sh`** — zero prompts. Drives A via the `amy` CLI - (`./gradlew :cli:installDist`) and B/C via `wn`. Runs every scenario - end-to-end and exits with a pass/fail summary. Use this for CI and for - iterating on the Nostr/Marmot plumbing without needing to touch a phone. +``` +cli/tests/ +├── lib.sh # shared logging, results, assertions +├── headless/ # shared bits used by every harness +│ └── helpers.sh +├── marmot/ # Marmot / MLS group-messaging interop +│ ├── marmot-interop.sh # interactive — prompts Amethyst Android UI +│ ├── marmot-interop-headless.sh # zero-prompt +│ ├── setup.sh # preflight + wn + relay + identities +│ ├── tests-create.sh # tests 01–05 +│ ├── tests-manage.sh # tests 06–08, 11 +│ ├── tests-extras.sh # tests 09, 10, 12, 13 +│ └── patches/ # whitenoise-rs harness patches +└── dm/ # NIP-17 DM interop (amy ↔ amy) + ├── dm-interop-headless.sh + ├── setup.sh # preflight + identities + └── tests-dm.sh +``` + +The Marmot harnesses come in two flavours, same scenarios: + +- **`marmot/marmot-interop.sh`** — interactive. Drives B/C via `wn` and + **prompts the human** to perform each Amethyst-side step in the mobile UI + (Identity A). Use this for final UI verification. +- **`marmot/marmot-interop-headless.sh`** — zero prompts. Drives A via the + `amy` CLI (`./gradlew :cli:installDist`) and B/C via `wn`. Runs every + scenario end-to-end and exits with a pass/fail summary. Use this for CI + and for iterating on the Nostr/Marmot plumbing without needing to touch a + phone. A third, slimmer harness covers the NIP-17 DM surface: -- **`dm-interop-headless.sh`** — two `amy` processes (Identity A and +- **`dm/dm-interop-headless.sh`** — two `amy` processes (Identity A and Identity D) exchange NIP-17 DMs through the loopback nostr-rs-relay. No whitenoise-rs required — only `amy` and the relay binary (which is shared with the Marmot harness's checkout at - `state-headless/nostr-rs-relay/`). + `marmot/state-headless/nostr-rs-relay/`). Both Marmot harnesses validate Amethyst against **whitenoise-rs** (https://github.com/marmot-protocol/whitenoise-rs), the reference Rust @@ -42,7 +65,7 @@ the end of the run. | 12 | Offline catch-up / replay | – | | 13 | KeyPackage rotation | – | -### DM (amy ↔ amy, NIP-17) — `dm-interop-headless.sh` +### DM (amy ↔ amy, NIP-17) — `dm/dm-interop-headless.sh` | # | Test | |---|---| diff --git a/tools/marmot-interop/dm-interop-headless.sh b/cli/tests/dm/dm-interop-headless.sh similarity index 70% rename from tools/marmot-interop/dm-interop-headless.sh rename to cli/tests/dm/dm-interop-headless.sh index 36414c256..587f1b5fa 100755 --- a/tools/marmot-interop/dm-interop-headless.sh +++ b/cli/tests/dm/dm-interop-headless.sh @@ -11,7 +11,8 @@ set -uo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd -- "$SCRIPT_DIR/../.." && pwd)" +REPO_ROOT="$(cd -- "$SCRIPT_DIR/../../.." && pwd)" +TESTS_DIR="$(cd -- "$SCRIPT_DIR/.." && pwd)" STATE_DIR="$SCRIPT_DIR/state-dm-headless" LOG_DIR="$STATE_DIR/logs" A_DIR="$STATE_DIR/A" @@ -26,7 +27,7 @@ AMY_BIN="$REPO_ROOT/cli/build/install/amy/bin/amy" # Share the nostr-rs-relay checkout with the Marmot harness to avoid # rebuilding it twice. Override RELAY_REPO / RELAY_DATA if you want full # isolation between runs. -RELAY_REPO="${RELAY_REPO:-$SCRIPT_DIR/state-headless/nostr-rs-relay}" +RELAY_REPO="${RELAY_REPO:-$TESTS_DIR/marmot/state-headless/nostr-rs-relay}" RELAY_BIN="$RELAY_REPO/target/release/nostr-rs-relay" RELAY_DATA="$STATE_DIR/relay" RELAY_PORT="${RELAY_PORT:-8090}" @@ -54,22 +55,23 @@ mkdir -p "$STATE_DIR" "$LOG_DIR" "$A_DIR" "$D_DIR" : >"$LOG_FILE" : >"$RESULTS_FILE" -# Reuse the logging + result helpers from the Marmot harness. lib.sh -# also declares wn-specific helpers; they're harmless when unused. -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" +# Reuse the logging + result helpers shared between every harness. +# lib.sh declares wn-specific helpers too — harmless when unused. +# shellcheck source=../lib.sh +source "$TESTS_DIR/lib.sh" -# Reuse start_local_relay / stop_local_relay from setup.sh. preflight() -# there also builds whitenoise-rs, which we don't need — setup-dm.sh +# Reuse start_local_relay / stop_local_relay from the Marmot harness's +# setup.sh — the relay lifecycle is identical. preflight() there also +# builds whitenoise-rs, which we don't need; setup.sh in this dir # defines a slimmer preflight_dm(). -# shellcheck source=headless/setup.sh -source "$SCRIPT_DIR/headless/setup.sh" -# shellcheck source=headless/setup-dm.sh -source "$SCRIPT_DIR/headless/setup-dm.sh" -# shellcheck source=headless/helpers.sh -source "$SCRIPT_DIR/headless/helpers.sh" -# shellcheck source=headless/tests-dm.sh -source "$SCRIPT_DIR/headless/tests-dm.sh" +# shellcheck source=../marmot/setup.sh +source "$TESTS_DIR/marmot/setup.sh" +# shellcheck source=setup.sh +source "$SCRIPT_DIR/setup.sh" +# shellcheck source=../headless/helpers.sh +source "$TESTS_DIR/headless/helpers.sh" +# shellcheck source=tests-dm.sh +source "$SCRIPT_DIR/tests-dm.sh" cleanup() { local rc=$? diff --git a/tools/marmot-interop/headless/setup-dm.sh b/cli/tests/dm/setup.sh similarity index 96% rename from tools/marmot-interop/headless/setup-dm.sh rename to cli/tests/dm/setup.sh index 6bb7e764a..148982375 100644 --- a/tools/marmot-interop/headless/setup-dm.sh +++ b/cli/tests/dm/setup.sh @@ -1,6 +1,6 @@ # shellcheck shell=bash # -# headless/setup-dm.sh — amy-only preflight + identity bootstrap for the +# setup.sh — amy-only preflight + identity bootstrap for the # NIP-17 DM interop harness. Much slimmer than the Marmot setup: # # - Builds `amy` (same retry-on-503 logic as setup.sh). @@ -11,7 +11,7 @@ # recipient-inbox routing has something to resolve to. # # The heavy `start_local_relay` / `stop_local_relay` helpers live in -# headless/setup.sh and are sourced by the top-level harness. +# the Marmot harness's setup.sh and are sourced by the top-level harness. # --- preflight (amy + relay only, no wn / Marmot patches) ------------------- preflight_dm() { diff --git a/tools/marmot-interop/headless/tests-dm.sh b/cli/tests/dm/tests-dm.sh similarity index 99% rename from tools/marmot-interop/headless/tests-dm.sh rename to cli/tests/dm/tests-dm.sh index a693ca675..945ee22d6 100644 --- a/tools/marmot-interop/headless/tests-dm.sh +++ b/cli/tests/dm/tests-dm.sh @@ -1,6 +1,6 @@ # shellcheck shell=bash # -# headless/tests-dm.sh — NIP-17 DM interop tests for two `amy` clients. +# tests-dm.sh — NIP-17 DM interop tests for two `amy` clients. # # Identity A (sender) and Identity D (recipient) each live in their own # --data-dir and share one loopback nostr-rs-relay. Tests cover: diff --git a/tools/marmot-interop/headless/helpers.sh b/cli/tests/headless/helpers.sh similarity index 95% rename from tools/marmot-interop/headless/helpers.sh rename to cli/tests/headless/helpers.sh index 7a7e10a65..7c9e55dc0 100644 --- a/tools/marmot-interop/headless/helpers.sh +++ b/cli/tests/headless/helpers.sh @@ -1,6 +1,6 @@ # shellcheck shell=bash # -# headless/helpers.sh — thin wrappers that keep the per-test code tight. +# helpers.sh — thin wrappers that keep the per-test code tight. # --- amy wrapper ------------------------------------------------------------- amy_a() { "$AMY_BIN" --data-dir "$A_DIR" "$@"; } diff --git a/tools/marmot-interop/lib.sh b/cli/tests/lib.sh similarity index 100% rename from tools/marmot-interop/lib.sh rename to cli/tests/lib.sh diff --git a/tools/marmot-interop/marmot-interop-headless.sh b/cli/tests/marmot/marmot-interop-headless.sh similarity index 85% rename from tools/marmot-interop/marmot-interop-headless.sh rename to cli/tests/marmot/marmot-interop-headless.sh index df2bd8308..c89ce806c 100755 --- a/tools/marmot-interop/marmot-interop-headless.sh +++ b/cli/tests/marmot/marmot-interop-headless.sh @@ -14,7 +14,8 @@ set -uo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd -- "$SCRIPT_DIR/../.." && pwd)" +REPO_ROOT="$(cd -- "$SCRIPT_DIR/../../.." && pwd)" +TESTS_DIR="$(cd -- "$SCRIPT_DIR/.." && pwd)" STATE_DIR="$SCRIPT_DIR/state-headless" LOG_DIR="$STATE_DIR/logs" A_DIR="$STATE_DIR/A" @@ -66,19 +67,19 @@ mkdir -p "$STATE_DIR" "$LOG_DIR" "$A_DIR" "$B_DIR/logs" "$C_DIR/logs" : >"$RESULTS_FILE" # Reuse colours / logging / dump_daemon_diagnostics from the interactive harness. -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" +# shellcheck source=../lib.sh +source "$TESTS_DIR/lib.sh" -# shellcheck source=headless/setup.sh -source "$SCRIPT_DIR/headless/setup.sh" -# shellcheck source=headless/helpers.sh -source "$SCRIPT_DIR/headless/helpers.sh" -# shellcheck source=headless/tests-create.sh -source "$SCRIPT_DIR/headless/tests-create.sh" -# shellcheck source=headless/tests-manage.sh -source "$SCRIPT_DIR/headless/tests-manage.sh" -# shellcheck source=headless/tests-extras.sh -source "$SCRIPT_DIR/headless/tests-extras.sh" +# shellcheck source=setup.sh +source "$SCRIPT_DIR/setup.sh" +# shellcheck source=../headless/helpers.sh +source "$TESTS_DIR/headless/helpers.sh" +# shellcheck source=tests-create.sh +source "$SCRIPT_DIR/tests-create.sh" +# shellcheck source=tests-manage.sh +source "$SCRIPT_DIR/tests-manage.sh" +# shellcheck source=tests-extras.sh +source "$SCRIPT_DIR/tests-extras.sh" # Make sure Ctrl+C / SIGTERM / SIGHUP all run the full cleanup path — # otherwise wnd is nohup'd and keeps running after the script dies, diff --git a/tools/marmot-interop/marmot-interop.sh b/cli/tests/marmot/marmot-interop.sh similarity index 99% rename from tools/marmot-interop/marmot-interop.sh rename to cli/tests/marmot/marmot-interop.sh index f8f7e4702..bfca09ffa 100755 --- a/tools/marmot-interop/marmot-interop.sh +++ b/cli/tests/marmot/marmot-interop.sh @@ -10,6 +10,7 @@ set -uo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +TESTS_DIR="$(cd -- "$SCRIPT_DIR/.." && pwd)" STATE_DIR="$SCRIPT_DIR/state" LOG_DIR="$STATE_DIR/logs" B_DIR="$STATE_DIR/B" @@ -75,8 +76,8 @@ mkdir -p "$STATE_DIR" "$LOG_DIR" "$B_DIR/logs" "$C_DIR/logs" : >"$LOG_FILE" : >"$RESULTS_FILE" -# shellcheck source=lib.sh -source "$SCRIPT_DIR/lib.sh" +# shellcheck source=../lib.sh +source "$TESTS_DIR/lib.sh" # --- preflight --------------------------------------------------------------- preflight() { diff --git a/tools/marmot-interop/headless/patches/whitenoise-defaults-env.patch b/cli/tests/marmot/patches/whitenoise-defaults-env.patch similarity index 100% rename from tools/marmot-interop/headless/patches/whitenoise-defaults-env.patch rename to cli/tests/marmot/patches/whitenoise-defaults-env.patch diff --git a/tools/marmot-interop/headless/patches/whitenoise-discovery-env.patch b/cli/tests/marmot/patches/whitenoise-discovery-env.patch similarity index 100% rename from tools/marmot-interop/headless/patches/whitenoise-discovery-env.patch rename to cli/tests/marmot/patches/whitenoise-discovery-env.patch diff --git a/tools/marmot-interop/headless/patches/whitenoise-mock-keyring.patch b/cli/tests/marmot/patches/whitenoise-mock-keyring.patch similarity index 100% rename from tools/marmot-interop/headless/patches/whitenoise-mock-keyring.patch rename to cli/tests/marmot/patches/whitenoise-mock-keyring.patch diff --git a/tools/marmot-interop/headless/patches/whitenoise-skip-unprocessable-retry.patch b/cli/tests/marmot/patches/whitenoise-skip-unprocessable-retry.patch similarity index 100% rename from tools/marmot-interop/headless/patches/whitenoise-skip-unprocessable-retry.patch rename to cli/tests/marmot/patches/whitenoise-skip-unprocessable-retry.patch diff --git a/tools/marmot-interop/headless/setup.sh b/cli/tests/marmot/setup.sh similarity index 99% rename from tools/marmot-interop/headless/setup.sh rename to cli/tests/marmot/setup.sh index 66e3fff77..e475f9027 100644 --- a/tools/marmot-interop/headless/setup.sh +++ b/cli/tests/marmot/setup.sh @@ -1,6 +1,6 @@ # shellcheck shell=bash # -# headless/setup.sh — preflight + daemon lifecycle + identity bootstrap. +# setup.sh — preflight + daemon lifecycle + identity bootstrap. # Sourced from marmot-interop-headless.sh. # --- preflight --------------------------------------------------------------- @@ -90,7 +90,7 @@ preflight() { if [[ ! -f "$marker" ]]; then step "patching whitenoise-rs: $name" if ( cd "$WN_REPO" && patch -p1 --forward --reject-file=- \ - <"$SCRIPT_DIR/headless/patches/$name" >>"$LOG_FILE" 2>&1 ); then + <"$SCRIPT_DIR/patches/$name" >>"$LOG_FILE" 2>&1 ); then touch "$marker" # Invalidate the previous build so the patched source is picked up. rm -f "$WN_BIN" "$WND_BIN" diff --git a/tools/marmot-interop/headless/tests-create.sh b/cli/tests/marmot/tests-create.sh similarity index 99% rename from tools/marmot-interop/headless/tests-create.sh rename to cli/tests/marmot/tests-create.sh index 33c898c35..218851be0 100644 --- a/tools/marmot-interop/headless/tests-create.sh +++ b/cli/tests/marmot/tests-create.sh @@ -1,6 +1,6 @@ # shellcheck shell=bash # -# headless/tests-create.sh — tests 01..05. +# tests-create.sh — tests 01..05. # Focus: KeyPackage discovery, group creation, invites, initial messages. test_01_keypackage_discovery() { diff --git a/tools/marmot-interop/headless/tests-extras.sh b/cli/tests/marmot/tests-extras.sh similarity index 99% rename from tools/marmot-interop/headless/tests-extras.sh rename to cli/tests/marmot/tests-extras.sh index 6e8495856..b6b026a5a 100644 --- a/tools/marmot-interop/headless/tests-extras.sh +++ b/cli/tests/marmot/tests-extras.sh @@ -1,6 +1,6 @@ # shellcheck shell=bash # -# headless/tests-extras.sh — tests 09, 10, 12, 13. +# tests-extras.sh — tests 09, 10, 12, 13. # Focus: reactions/replies, concurrent commits, offline catchup, KP rotation. test_09_reply_react_unreact() { diff --git a/tools/marmot-interop/headless/tests-manage.sh b/cli/tests/marmot/tests-manage.sh similarity index 99% rename from tools/marmot-interop/headless/tests-manage.sh rename to cli/tests/marmot/tests-manage.sh index fd14c49f9..7923811ab 100644 --- a/tools/marmot-interop/headless/tests-manage.sh +++ b/cli/tests/marmot/tests-manage.sh @@ -1,6 +1,6 @@ # shellcheck shell=bash # -# headless/tests-manage.sh — tests 06, 07, 08, 11. +# tests-manage.sh — tests 06, 07, 08, 11. # Focus: removal, metadata rename, admin promote/demote, leave. test_06_member_removal() { diff --git a/tools/marmot-interop/.gitignore b/tools/marmot-interop/.gitignore deleted file mode 100644 index 1bcd48904..000000000 --- a/tools/marmot-interop/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -state/ -state-headless/