From 8c4a82ec35c1d88ff145f08e6e5286651ddd2a07 Mon Sep 17 00:00:00 2001 From: John Lancaster <32917998+jsl12@users.noreply.github.com> Date: Wed, 24 Jun 2026 14:51:26 -0500 Subject: [PATCH] testing improvements --- docs/step3.md | 94 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 21 deletions(-) diff --git a/docs/step3.md b/docs/step3.md index c7097af..408c23e 100644 --- a/docs/step3.md +++ b/docs/step3.md @@ -40,8 +40,14 @@ This corresponds to MVP Step 3 from `docs/mvp.md`: - `src/transcription/services/__init__.py` (optional export) ### Tests -- `tests/test_providers_openrouter.py` -- `tests/test_transcription_service.py` +- `tests/providers/test_openrouter.py` +- `tests/services/test_transcription.py` + +### Test directory convention +- Mirror source domains under `tests/`. +- Provider adapter tests live under `tests/providers/`. +- Service-layer tests live under `tests/services/`. +- Prefer one focused test module per production module (for Step 3: `test_openrouter.py`, `test_transcription.py`). --- @@ -133,33 +139,79 @@ This corresponds to MVP Step 3 from `docs/mvp.md`: --- -## Phase E — Tests (mocked, deterministic) +## Phase E — Tests (two-phase scaffold -> fill) -## `tests/test_providers_openrouter.py` -- [ ] test adapter initializes from settings -- [ ] test model fallback when `provider_model is None` -- [ ] test referer/title options are included when set -- [ ] test successful SDK response parses transcript text -- [ ] test SDK exception maps to `ProviderError` -- [ ] test empty/invalid response maps to `ProviderError` +### Required execution resources -## `tests/test_transcription_service.py` -- [ ] test prompt loader reads canonical prompt file -- [ ] test missing prompt raises `PromptLoadError` -- [ ] test transcription function loads file and calls provider once -- [ ] test image path missing raises clear error -- [ ] test provider error is propagated/wrapped predictably -- [ ] test returned result includes transcript text and metadata +Load and reference these directly during test planning/implementation so the two-phase flow is enforced: -> Keep these unit tests mocked (no real OpenRouter calls in default suite). +- [ ] `resource://catalog/prompts/pytest-scaffold` +- [ ] `resource://prompts/pytest-scaffold/document` +- [ ] `resource://catalog/prompts/pytest-fill-scaffold` +- [ ] `resource://prompts/pytest-fill-scaffold/document` + +### Phase E1 — Scaffold test structure first + +Prompt: `resource://catalog/prompts/pytest-scaffold` + +Suggested arguments: +- [ ] `target_modules` = `src/transcription/providers/openrouter.py`, `src/transcription/services/transcription.py` +- [ ] `mode` = `scaffold` +- [ ] `path_strategy` = `src-to-tests-mirror` +- [ ] `naming_style` = `concise-behavior` + +Expected scaffold outcomes: +- [ ] `tests/providers/test_openrouter.py` exists with class/method skeletons and one-line docstrings +- [ ] `tests/services/test_transcription.py` exists with class/method skeletons and one-line docstrings +- [ ] collection succeeds on scaffold-only tests + +Scaffold coverage targets: +- [ ] adapter initializes from settings +- [ ] model fallback when `provider_model is None` +- [ ] referer/title options included when set +- [ ] successful SDK response parses transcript text +- [ ] SDK exception maps to `ProviderError` +- [ ] empty/invalid response maps to `ProviderError` +- [ ] prompt loader reads canonical prompt file +- [ ] missing prompt raises `PromptLoadError` +- [ ] transcription function loads file and calls provider once +- [ ] image path missing raises clear error +- [ ] provider error is propagated/wrapped predictably +- [ ] returned result includes transcript text and metadata + +### Phase E2 — Fill scaffolded tests with assertions + +Prompt: `resource://catalog/prompts/pytest-fill-scaffold` + +Suggested arguments: +- [ ] `target_files` = `tests/providers/test_openrouter.py`, `tests/services/test_transcription.py` +- [ ] `stack` = `pure-python` +- [ ] `strategy` = `minimal` +- [ ] `marker_lane` = `unit` + +Fill constraints: +- [ ] preserve scaffold class/method names and one-line docstrings +- [ ] keep mocks to an absolute minimum; mock only network boundaries and non-deterministic failures +- [ ] keep one behavior target per test method + +> Default suite should remain deterministic and fast, but mocking should be minimal and intentional. + +### Optional real-endpoint validation lane + +- [ ] Add an opt-in integration lane for real provider calls (for example `@pytest.mark.integration` and `@pytest.mark.live_api`). +- [ ] Gate live tests behind explicit env vars (for example `OPENROUTER_API_KEY`, optional `RUN_LIVE_API_TESTS=1`). +- [ ] Exclude live tests from default CI/local runs unless explicitly requested. +- [ ] Keep at least one thin smoke path that can validate request/response compatibility against the real endpoint. --- ## Phase F — Verification commands -- [ ] `uv run pytest tests/test_providers_openrouter.py -q` -- [ ] `uv run pytest tests/test_transcription_service.py -q` -- [ ] `uv run pytest -q` +- [ ] E1 scaffold validation: `uv run pytest --collect-only -q` +- [ ] E2 fill validation (unit lane): `uv run pytest -m unit -q` +- [ ] E2 targeted provider file: `uv run pytest tests/providers/test_openrouter.py -q` +- [ ] E2 targeted service file: `uv run pytest tests/services/test_transcription.py -q` +- [ ] E2 final full-suite check: `uv run pytest -q` ---