generated from john/python-template
94 lines
3.4 KiB
Python
94 lines
3.4 KiB
Python
"""Tests for prompt artifacts in prompts/."""
|
|
|
|
import hashlib
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
|
|
from transcription.config import Settings
|
|
from transcription.services.sources import PromptExecution
|
|
from transcription.services.sources import PromptLoadError
|
|
from transcription.services.sources import build_prompt_execution
|
|
from transcription.services.sources import load_prompt_text
|
|
|
|
PROMPT_PATH = Path("prompts/transcribe_document.md")
|
|
|
|
|
|
def _prompt_text() -> str:
|
|
"""Read prompt text from the canonical prompt file."""
|
|
return PROMPT_PATH.read_text(encoding="utf-8")
|
|
|
|
|
|
class TestPromptArtifact:
|
|
"""Verify prompt artifact presence and baseline semantics."""
|
|
|
|
def test_prompt_file_exists(self):
|
|
"""Canonical transcription prompt file exists."""
|
|
assert PROMPT_PATH.exists()
|
|
|
|
def test_prompt_file_is_not_empty(self):
|
|
"""Canonical prompt file has non-whitespace content."""
|
|
text = _prompt_text()
|
|
assert text.strip()
|
|
|
|
def test_prompt_mentions_verbatim_behavior(self):
|
|
"""Prompt explicitly enforces verbatim transcription behavior."""
|
|
text = _prompt_text().lower()
|
|
assert "verbatim" in text
|
|
assert "do not summarize" in text
|
|
|
|
def test_prompt_includes_uncertainty_and_illegible_markers(self):
|
|
"""Prompt contains conventions for uncertainty and illegible text."""
|
|
text = _prompt_text().lower()
|
|
assert "[boston?]" in text
|
|
assert "[illegible]" in text
|
|
|
|
def test_prompt_includes_deleted_and_inserted_conventions(self):
|
|
"""Prompt contains conventions for deleted and inserted text."""
|
|
text = _prompt_text().lower()
|
|
assert "[deleted:" in text
|
|
assert "[inserted:" in text
|
|
|
|
|
|
class TestPromptConfiguration:
|
|
def test_builds_validated_immutable_prompt_provenance(self, tmp_path):
|
|
prompt_text = "Transcribe this document verbatim."
|
|
(tmp_path / "custom.md").write_text(prompt_text, encoding="utf-8")
|
|
settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
prompt_dir=tmp_path,
|
|
default_prompt_name="custom.md",
|
|
transcription_temperature=0.2,
|
|
transcription_top_p=0.9,
|
|
)
|
|
|
|
execution = build_prompt_execution(settings=settings)
|
|
|
|
assert execution.prompt_hash == hashlib.sha256(prompt_text.encode()).hexdigest()
|
|
assert execution.temperature == 0.2
|
|
assert execution.top_p == 0.9
|
|
with pytest.raises(ValidationError):
|
|
execution.prompt_name = "changed.md"
|
|
|
|
def test_rejects_prompt_path_traversal_even_with_direct_loader_call(self, tmp_path):
|
|
outside_prompt = tmp_path / "outside.md"
|
|
prompt_dir = tmp_path / "prompts"
|
|
prompt_dir.mkdir()
|
|
outside_prompt.write_text("secret", encoding="utf-8")
|
|
settings = Settings(openrouter_api_key="test-key", prompt_dir=prompt_dir)
|
|
|
|
with pytest.raises(PromptLoadError):
|
|
load_prompt_text(prompt_name="../outside.md", settings=settings)
|
|
|
|
def test_prompt_execution_rejects_invalid_provenance_hash(self):
|
|
with pytest.raises(ValidationError):
|
|
PromptExecution(
|
|
prompt_name="prompt.md",
|
|
prompt_hash="not-a-sha256",
|
|
system_prompt=None,
|
|
user_prompt="text",
|
|
temperature=None,
|
|
top_p=None,
|
|
)
|