generated from john/python-template
Jim LancasterandCopilot App
5566f48fc0
Quality Gate / gate (push) Failing after 11s
Update the reviewer skill so its procedure matches how this repo actually works: - Route review reports to docs/reviews/ and mark them non-canonical, resolving the conflict where reports landed in the same docs/ tree they resolve findings against. - Pin verification commands to uv (uv run ruff check / ty check / pytest -m 'not external'). - Record the pytest contract: strict markers, strict asyncio mode, and the never-awaited-coroutine warning promoted to an error. - Convert the deterministic checks to a table with an Enforced by column; three checks are unenforced and one only partial, which are now findings by construction. - Add a consequence-based severity rubric and a Direction column for bidirectional drift. - Escalate test-suite concerns to test-effectiveness-auditor. Also fix tests/test_db.py, which was missing 'from sqlalchemy import text' while using it in 14 places. Three tests were failing with NameError. Wrapped the pre-existing long lines in the same file so it lints clean. Document the deliberate nicegui==3.13.0 pin in pyproject.toml, a new runbook dependency upgrade policy, and the reviewer skill, so the pin is not flagged as a defect or widened as incidental cleanup. Co-authored-by: Copilot App <[email protected]>
383 lines
15 KiB
Python
383 lines
15 KiB
Python
"""Tests for the database runtime and V2 schema bootstrap behavior."""
|
|
|
|
import warnings
|
|
|
|
import pytest
|
|
import pytest_asyncio
|
|
import sqlalchemy as sa
|
|
from sqlalchemy import inspect
|
|
from sqlalchemy import text
|
|
from sqlalchemy.dialects import postgresql
|
|
from sqlalchemy.exc import SAWarning
|
|
from sqlmodel import SQLModel
|
|
from sqlmodel import select
|
|
from sqlmodel.ext.asyncio.session import AsyncSession
|
|
|
|
from transcription.config import Settings
|
|
from transcription.config import SqliteSettings
|
|
from transcription.db import create_all
|
|
from transcription.db import dispose_database_runtime
|
|
from transcription.db import initialize_database_runtime
|
|
from transcription.db import reconcile_canonical_media_paths
|
|
from transcription.db import reconcile_legacy_job_source_columns
|
|
from transcription.db import reconcile_person_name_columns
|
|
from transcription.db import session_scope
|
|
from transcription.db.models import DocumentType
|
|
from transcription.db.models import PersonRole
|
|
from transcription.db.models import Source
|
|
|
|
|
|
@pytest_asyncio.fixture(autouse=True)
|
|
async def _reset_database_runtime():
|
|
await dispose_database_runtime()
|
|
yield
|
|
await dispose_database_runtime()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_all_creates_expected_tables(tmp_path):
|
|
settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
database=SqliteSettings(path=str(tmp_path / "schema.db")),
|
|
environment="test",
|
|
)
|
|
runtime = initialize_database_runtime(settings=settings)
|
|
|
|
try:
|
|
await create_all(engine=runtime.engine)
|
|
async with runtime.engine.connect() as conn:
|
|
table_names = set(await conn.run_sync(lambda c: inspect(c).get_table_names()))
|
|
|
|
assert "document" in table_names
|
|
assert "document_type" in table_names
|
|
assert "tag" in table_names
|
|
assert "person" in table_names
|
|
assert "photo" in table_names
|
|
assert "person_role" in table_names
|
|
assert "document_person" in table_names
|
|
assert "document_tag" in table_names
|
|
assert "person_tag" in table_names
|
|
assert "job" in table_names
|
|
assert "source" in table_names
|
|
assert "job_source" in table_names
|
|
assert "execution_attempt" in table_names
|
|
assert "revision" not in table_names
|
|
finally:
|
|
await dispose_database_runtime()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_session_yields_async_session(tmp_path):
|
|
settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
database=SqliteSettings(path=str(tmp_path / "session.db")),
|
|
environment="test",
|
|
)
|
|
initialize_database_runtime(settings=settings)
|
|
|
|
try:
|
|
async with session_scope(settings=settings) as session:
|
|
assert isinstance(session, AsyncSession)
|
|
value = (await session.exec(select(1))).one()
|
|
assert value == 1
|
|
finally:
|
|
await dispose_database_runtime()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runtime_rejects_reinitialization_for_different_database(tmp_path):
|
|
"""An existing process runtime cannot silently switch database targets."""
|
|
first_settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
database=SqliteSettings(path=str(tmp_path / "first.db")),
|
|
environment="test",
|
|
)
|
|
second_settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
database=SqliteSettings(path=str(tmp_path / "second.db")),
|
|
environment="test",
|
|
)
|
|
initialize_database_runtime(settings=first_settings)
|
|
|
|
try:
|
|
with pytest.raises(RuntimeError, match="already initialized for a different database"):
|
|
initialize_database_runtime(settings=second_settings)
|
|
finally:
|
|
await dispose_database_runtime()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_all_seeds_default_registry_rows(tmp_path):
|
|
settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
database=SqliteSettings(path=str(tmp_path / "seed.db")),
|
|
environment="test",
|
|
)
|
|
runtime = initialize_database_runtime(settings=settings)
|
|
|
|
try:
|
|
await create_all(engine=runtime.engine)
|
|
async with AsyncSession(runtime.engine, expire_on_commit=False) as session:
|
|
role_keys = set((await session.exec(select(PersonRole.semantic_key))).all())
|
|
type_keys = set((await session.exec(select(DocumentType.semantic_key))).all())
|
|
|
|
assert {"author", "recipient", "mentioned"}.issubset(role_keys)
|
|
assert {"book", "letter", "postcard", "photo", "journal", "form"}.issubset(type_keys)
|
|
finally:
|
|
await dispose_database_runtime()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_all_declares_hot_path_indexes(tmp_path):
|
|
"""Worker and detail-page filters must be index-backed in a freshly created schema."""
|
|
settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
database=SqliteSettings(path=str(tmp_path / "indexes.db")),
|
|
environment="test",
|
|
)
|
|
runtime = initialize_database_runtime(settings=settings)
|
|
|
|
try:
|
|
await create_all(engine=runtime.engine)
|
|
async with runtime.engine.connect() as connection:
|
|
|
|
def collect(sync_connection) -> tuple[dict[str, list[list[str]]], list[list[str]]]:
|
|
database = inspect(sync_connection)
|
|
indexes = {
|
|
table: [index["column_names"] for index in database.get_indexes(table)]
|
|
for table in (
|
|
"job",
|
|
"source",
|
|
"job_source",
|
|
"document",
|
|
"document_person",
|
|
"document_tag",
|
|
"person_tag",
|
|
)
|
|
}
|
|
job_source_unique = [
|
|
constraint["column_names"]
|
|
for constraint in database.get_unique_constraints("job_source")
|
|
]
|
|
document_tag_unique = [
|
|
constraint["column_names"]
|
|
for constraint in database.get_unique_constraints("document_tag")
|
|
]
|
|
person_tag_unique = [
|
|
constraint["column_names"]
|
|
for constraint in database.get_unique_constraints("person_tag")
|
|
]
|
|
return indexes, job_source_unique, document_tag_unique, person_tag_unique
|
|
|
|
indexes, job_source_unique, document_tag_unique, person_tag_unique = await connection.run_sync(collect)
|
|
|
|
assert ["status", "date_created"] in indexes["job"]
|
|
assert ["document_id"] in indexes["job"]
|
|
assert ["document_id"] in indexes["source"]
|
|
assert ["preferred_execution_attempt_id"] in indexes["source"]
|
|
assert ["job_id"] in indexes["job_source"]
|
|
assert ["source_id"] in indexes["job_source"]
|
|
assert ["job_id", "source_id"] in job_source_unique
|
|
assert ["document_type_id"] in indexes["document"]
|
|
for column in ("document_id", "person_id", "role_id"):
|
|
assert [column] in indexes["document_person"]
|
|
assert ["document_id"] in indexes["document_tag"]
|
|
assert ["tag_id"] in indexes["document_tag"]
|
|
assert ["document_id", "tag_id"] in document_tag_unique
|
|
assert ["person_id"] in indexes["person_tag"]
|
|
assert ["tag_id"] in indexes["person_tag"]
|
|
assert ["person_id", "tag_id"] in person_tag_unique
|
|
finally:
|
|
await dispose_database_runtime()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reconcile_person_name_columns_backfills_split_names(tmp_path):
|
|
settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
database=SqliteSettings(path=str(tmp_path / "legacy-person-name.db")),
|
|
environment="test",
|
|
)
|
|
runtime = initialize_database_runtime(settings=settings)
|
|
|
|
try:
|
|
await create_all(engine=runtime.engine)
|
|
async with runtime.engine.begin() as connection:
|
|
await connection.execute(text('alter table "person" add column "full_name" varchar'))
|
|
await connection.execute(
|
|
text(
|
|
'insert into "person" (id, full_name, given_names, last_name, created_at, updated_at) '
|
|
"values (:id, :full_name, '', '', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)"
|
|
),
|
|
{"id": "55" * 16, "full_name": "Ada Lovelace"},
|
|
)
|
|
|
|
changed = await reconcile_person_name_columns(engine=runtime.engine)
|
|
assert changed >= 1
|
|
|
|
async with runtime.engine.connect() as connection:
|
|
row = (
|
|
await connection.execute(
|
|
text('select given_names, last_name from "person" where id = :id'),
|
|
{"id": "55" * 16},
|
|
)
|
|
).one()
|
|
assert row[0] == "Ada"
|
|
assert row[1] == "Lovelace"
|
|
finally:
|
|
await dispose_database_runtime()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reconcile_legacy_job_source_columns_drops_executed_at(tmp_path):
|
|
settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
database=SqliteSettings(path=str(tmp_path / "legacy-column.db")),
|
|
environment="test",
|
|
)
|
|
runtime = initialize_database_runtime(settings=settings)
|
|
|
|
try:
|
|
await create_all(engine=runtime.engine)
|
|
async with runtime.engine.begin() as connection:
|
|
await connection.execute(text("PRAGMA foreign_keys=OFF"))
|
|
await connection.execute(text('alter table "job_source" rename to "job_source_current"'))
|
|
await connection.execute(
|
|
text(
|
|
'create table "job_source" ('
|
|
'id char(32) not null primary key, '
|
|
'job_id char(32) not null, '
|
|
'source_id char(32) not null, '
|
|
'status varchar(11) not null, '
|
|
'executed_at datetime not null, '
|
|
'constraint "uq_job_source_job_source" unique ("job_id", "source_id"), '
|
|
'foreign key("job_id") references "job" ("id"), '
|
|
'foreign key("source_id") references "source" ("id")'
|
|
")"
|
|
)
|
|
)
|
|
await connection.execute(text('drop table "job_source_current"'))
|
|
await connection.execute(text("PRAGMA foreign_keys=ON"))
|
|
|
|
dropped = await reconcile_legacy_job_source_columns(engine=runtime.engine)
|
|
assert dropped == 1
|
|
|
|
async with runtime.engine.connect() as connection:
|
|
columns = await connection.run_sync(lambda c: [col["name"] for col in inspect(c).get_columns("job_source")])
|
|
assert "executed_at" not in columns
|
|
finally:
|
|
await dispose_database_runtime()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reconcile_canonical_media_paths_normalizes_source_and_photo_paths(tmp_path):
|
|
settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
database=SqliteSettings(path=str(tmp_path / "canonical-paths.db")),
|
|
environment="test",
|
|
)
|
|
runtime = initialize_database_runtime(settings=settings)
|
|
|
|
try:
|
|
await create_all(engine=runtime.engine)
|
|
async with runtime.engine.begin() as connection:
|
|
await connection.execute(
|
|
text(
|
|
'insert into "person" (id, given_names, last_name, created_at, updated_at) '
|
|
'values (:id, :given_names, :last_name, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)'
|
|
),
|
|
{"id": "11" * 16, "given_names": "Portrait", "last_name": "Person"},
|
|
)
|
|
await connection.execute(
|
|
text(
|
|
'insert into "photo" (id, person_id, path, is_primary, created_at, updated_at) '
|
|
'values (:id, :person_id, :path, :is_primary, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)'
|
|
),
|
|
{
|
|
"id": "44" * 16,
|
|
"person_id": "11" * 16,
|
|
"path": "data\\photos\\seeded.png",
|
|
"is_primary": 1,
|
|
},
|
|
)
|
|
await connection.execute(
|
|
text(
|
|
'insert into "document" (id, name, created_at, updated_at) '
|
|
'values (:id, :name, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)'
|
|
),
|
|
{"id": "22" * 16, "name": "Doc"},
|
|
)
|
|
await connection.execute(
|
|
text(
|
|
'insert into "source" (id, document_id, page_number, upload_name, filename, '
|
|
"file_path, file_hash, file_size_bytes, date_uploaded) "
|
|
"values (:id, :document_id, 1, :upload_name, :filename, :file_path, "
|
|
":file_hash, :file_size_bytes, CURRENT_TIMESTAMP)"
|
|
),
|
|
{
|
|
"id": "33" * 16,
|
|
"document_id": "22" * 16,
|
|
"upload_name": "page.png",
|
|
"filename": "page.png",
|
|
"file_path": "data\\documents\\doc-1\\page.png",
|
|
"file_hash": "a" * 64,
|
|
"file_size_bytes": 1,
|
|
},
|
|
)
|
|
|
|
changed = await reconcile_canonical_media_paths(engine=runtime.engine)
|
|
assert changed == 2
|
|
|
|
async with runtime.engine.connect() as connection:
|
|
source_path = (
|
|
await connection.execute(text('select file_path from "source" where id = :id'), {"id": "33" * 16})
|
|
).scalar_one()
|
|
photo_path = (
|
|
await connection.execute(text('select path from "photo" where id = :id'), {"id": "44" * 16})
|
|
).scalar_one()
|
|
assert source_path == "documents/doc-1/page.png"
|
|
assert photo_path == "photos/seeded.png"
|
|
finally:
|
|
await dispose_database_runtime()
|
|
|
|
|
|
def test_metadata_has_no_unresolvable_table_cycle():
|
|
"""create_all must be able to order every table, including on PostgreSQL."""
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("error", SAWarning)
|
|
ordered = [table.name for table in SQLModel.metadata.sorted_tables]
|
|
|
|
assert ordered.index("source") < ordered.index("execution_attempt")
|
|
|
|
|
|
def test_preferred_execution_attempt_id_column_matches_model_declaration():
|
|
"""The self-referential provenance column is a real UUID, not an opaque CHAR(32)."""
|
|
column = SQLModel.metadata.tables["source"].c["preferred_execution_attempt_id"]
|
|
|
|
assert isinstance(column.type, sa.Uuid)
|
|
assert column.type.compile(dialect=postgresql.dialect()) == "UUID"
|
|
assert Source.model_fields["preferred_execution_attempt_id"].annotation is not None
|
|
|
|
foreign_key = next(iter(column.foreign_keys))
|
|
assert foreign_key.use_alter is True
|
|
assert foreign_key.column is SQLModel.metadata.tables["execution_attempt"].c["id"]
|
|
|
|
|
|
def test_bootstrap_policy_production_defaults_false():
|
|
settings = Settings(openrouter_api_key="test-key", environment="production")
|
|
assert settings.should_bootstrap_schema is False
|
|
|
|
|
|
def test_bootstrap_policy_development_defaults_true():
|
|
settings = Settings(openrouter_api_key="test-key", environment="development")
|
|
assert settings.should_bootstrap_schema is True
|
|
|
|
|
|
def test_bootstrap_policy_explicit_override_true():
|
|
settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
environment="production",
|
|
bootstrap_schema_on_startup=True,
|
|
)
|
|
assert settings.should_bootstrap_schema is True
|