generated from john/python-template
This commit is contained in:
@@ -39,8 +39,8 @@ async def read_document_source_media(
|
||||
if source.document_id != document_id:
|
||||
raise HTTPException(status_code=404, detail="Source not found for Document")
|
||||
|
||||
path = Path(source.file_path).resolve()
|
||||
upload_root = service.settings.upload_dir.resolve()
|
||||
path = (upload_root / Path(source.file_path)).resolve()
|
||||
try:
|
||||
path.relative_to(upload_root)
|
||||
except ValueError as exc:
|
||||
|
||||
@@ -24,7 +24,7 @@ from .config import get_settings
|
||||
from .db import create_all
|
||||
from .db import dispose_database_runtime
|
||||
from .db import initialize_database_runtime
|
||||
from .db import normalize_legacy_status_spellings
|
||||
from .db import reconcile_canonical_media_paths
|
||||
from .db import reconcile_legacy_job_source_columns
|
||||
from .services import ServiceBundle
|
||||
from .ui import register_pages
|
||||
@@ -45,7 +45,7 @@ async def _lifespan(app: FastAPI):
|
||||
if settings.should_bootstrap_schema:
|
||||
await create_all(engine=app.state.runtime.engine)
|
||||
await reconcile_legacy_job_source_columns(engine=app.state.runtime.engine)
|
||||
await normalize_legacy_status_spellings(engine=app.state.runtime.engine)
|
||||
await reconcile_canonical_media_paths(engine=app.state.runtime.engine)
|
||||
|
||||
settings.upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
settings.prompt_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from .operations import create_all
|
||||
from .operations import normalize_legacy_status_spellings
|
||||
from .operations import reconcile_canonical_media_paths
|
||||
from .operations import reconcile_legacy_job_source_columns
|
||||
from .runtime import dispose_database_runtime
|
||||
from .runtime import initialize_database_runtime
|
||||
@@ -10,7 +10,7 @@ __all__ = [
|
||||
"create_all",
|
||||
"dispose_database_runtime",
|
||||
"initialize_database_runtime",
|
||||
"normalize_legacy_status_spellings",
|
||||
"reconcile_canonical_media_paths",
|
||||
"reconcile_legacy_job_source_columns",
|
||||
"session_scope",
|
||||
"transaction_scope",
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import URL
|
||||
from sqlalchemy import MetaData
|
||||
from sqlalchemy import Table
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import inspect as sqlalchemy_inspect
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlmodel import SQLModel
|
||||
|
||||
from transcription.config import Settings
|
||||
from transcription.config import get_settings
|
||||
|
||||
# Register table metadata.
|
||||
from transcription.db import models as _models # noqa: F401
|
||||
from transcription.db.engine import get_database_url
|
||||
|
||||
EXPORT_TABLE_ORDER = (
|
||||
"document_type",
|
||||
"person_role",
|
||||
"document",
|
||||
"person",
|
||||
"document_person",
|
||||
"job",
|
||||
"source",
|
||||
"job_source",
|
||||
"execution_attempt",
|
||||
)
|
||||
|
||||
BYTES_FIELDS = {"transport_body"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MigrationPaths:
|
||||
source_db_url: str
|
||||
target_db_url: str
|
||||
source_upload_dir: Path
|
||||
target_upload_dir: Path
|
||||
bundle_dir: Path
|
||||
|
||||
|
||||
def export_bundle(*, source_db_url: str, source_upload_dir: Path, bundle_dir: Path) -> None:
|
||||
bundle_dir.mkdir(parents=True, exist_ok=True)
|
||||
export_json = bundle_dir / "database.json"
|
||||
uploads_bundle_dir = bundle_dir / "uploads"
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"schema_name": "transcription.export-import",
|
||||
"schema_version": "1",
|
||||
"created_at": datetime.now().isoformat(),
|
||||
"tables": {},
|
||||
}
|
||||
|
||||
engine = create_engine(source_db_url)
|
||||
try:
|
||||
inspector = sqlalchemy_inspect(engine)
|
||||
source_tables = set(inspector.get_table_names())
|
||||
metadata = MetaData()
|
||||
metadata.reflect(bind=engine)
|
||||
current_metadata = SQLModel.metadata
|
||||
|
||||
with engine.connect() as connection:
|
||||
for table_name in EXPORT_TABLE_ORDER:
|
||||
if table_name not in source_tables:
|
||||
payload["tables"][table_name] = []
|
||||
continue
|
||||
|
||||
source_table = metadata.tables[table_name]
|
||||
target_table = current_metadata.tables[table_name]
|
||||
export_columns = [
|
||||
column.name for column in target_table.columns if column.name in source_table.columns
|
||||
]
|
||||
rows = connection.execute(select(*(source_table.c[name] for name in export_columns))).mappings().all()
|
||||
payload["tables"][table_name] = [
|
||||
_serialize_row(row, table_name=table_name, source_upload_dir=source_upload_dir) for row in rows
|
||||
]
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
export_json.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||
|
||||
if uploads_bundle_dir.exists():
|
||||
shutil.rmtree(uploads_bundle_dir)
|
||||
if source_upload_dir.exists():
|
||||
shutil.copytree(source_upload_dir, uploads_bundle_dir)
|
||||
else:
|
||||
uploads_bundle_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def import_bundle(*, target_db_url: str, target_upload_dir: Path, bundle_dir: Path) -> None:
|
||||
export_json = bundle_dir / "database.json"
|
||||
uploads_bundle_dir = bundle_dir / "uploads"
|
||||
payload = json.loads(export_json.read_text(encoding="utf-8"))
|
||||
|
||||
engine = create_engine(target_db_url)
|
||||
try:
|
||||
SQLModel.metadata.create_all(engine)
|
||||
with engine.begin() as connection:
|
||||
for table_name in reversed(EXPORT_TABLE_ORDER):
|
||||
table = SQLModel.metadata.tables[table_name]
|
||||
connection.execute(table.delete())
|
||||
|
||||
for table_name in EXPORT_TABLE_ORDER:
|
||||
rows = payload.get("tables", {}).get(table_name, [])
|
||||
if not rows:
|
||||
continue
|
||||
table = SQLModel.metadata.tables[table_name]
|
||||
connection.execute(table.insert(), [_deserialize_row(row, table) for row in rows])
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
if target_upload_dir.exists():
|
||||
shutil.rmtree(target_upload_dir)
|
||||
target_upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
if uploads_bundle_dir.exists():
|
||||
shutil.copytree(uploads_bundle_dir, target_upload_dir, dirs_exist_ok=True)
|
||||
|
||||
|
||||
def migrate_via_bundle(paths: MigrationPaths) -> None:
|
||||
export_bundle(
|
||||
source_db_url=paths.source_db_url,
|
||||
source_upload_dir=paths.source_upload_dir,
|
||||
bundle_dir=paths.bundle_dir,
|
||||
)
|
||||
import_bundle(
|
||||
target_db_url=paths.target_db_url,
|
||||
target_upload_dir=paths.target_upload_dir,
|
||||
bundle_dir=paths.bundle_dir,
|
||||
)
|
||||
|
||||
|
||||
def sqlite_url_from_path(path: Path) -> str:
|
||||
return URL.create(drivername="sqlite", database=str(path)).render_as_string(hide_password=False)
|
||||
|
||||
|
||||
def default_sync_db_url(settings: Settings | None = None) -> str:
|
||||
runtime_settings = settings or get_settings()
|
||||
return get_database_url(runtime_settings).replace("+aiosqlite", "").replace("+asyncpg", "").replace("+psycopg", "")
|
||||
|
||||
|
||||
def _serialize_row(row: dict[str, Any], *, table_name: str, source_upload_dir: Path) -> dict[str, Any]:
|
||||
serialized: dict[str, Any] = {}
|
||||
for key, value in row.items():
|
||||
serialized_value = _serialize_value(key, value)
|
||||
if table_name == "source" and key == "file_path" and isinstance(serialized_value, str):
|
||||
serialized[key] = _canonical_media_relative_path(
|
||||
serialized_value,
|
||||
source_upload_dir=source_upload_dir,
|
||||
preferred_prefix="documents/",
|
||||
)
|
||||
continue
|
||||
if table_name == "person" and key == "portrait_path" and isinstance(serialized_value, str):
|
||||
serialized[key] = _canonical_media_relative_path(
|
||||
serialized_value,
|
||||
source_upload_dir=source_upload_dir,
|
||||
preferred_prefix="persons/",
|
||||
)
|
||||
continue
|
||||
serialized[key] = serialized_value
|
||||
return serialized
|
||||
|
||||
|
||||
def _serialize_value(key: str, value: Any) -> Any:
|
||||
if isinstance(value, UUID):
|
||||
return str(value)
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, bytes):
|
||||
return {"encoding": "base64", "data": base64.b64encode(value).decode("ascii")}
|
||||
if isinstance(value, dict):
|
||||
return {str(k): _serialize_value("", v) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_serialize_value("", item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _deserialize_row(row: dict[str, Any], table: Table) -> dict[str, Any]:
|
||||
deserialized: dict[str, Any] = {}
|
||||
for key, value in row.items():
|
||||
if key in BYTES_FIELDS and isinstance(value, dict) and value.get("encoding") == "base64":
|
||||
deserialized[key] = base64.b64decode(value["data"])
|
||||
continue
|
||||
if key in table.columns:
|
||||
try:
|
||||
python_type: type[Any] = table.columns[key].type.python_type
|
||||
except NotImplementedError:
|
||||
deserialized[key] = value
|
||||
continue
|
||||
deserialized[key] = _deserialize_value(python_type, value)
|
||||
return deserialized
|
||||
|
||||
|
||||
def _deserialize_value(python_type: type[Any], value: Any) -> Any:
|
||||
if value is None:
|
||||
return None
|
||||
if python_type is UUID and isinstance(value, str):
|
||||
return UUID(value)
|
||||
if python_type is datetime and isinstance(value, str):
|
||||
return datetime.fromisoformat(value)
|
||||
if python_type is date and isinstance(value, str):
|
||||
return date.fromisoformat(value)
|
||||
return value
|
||||
|
||||
|
||||
def _canonical_media_relative_path(value: str, *, source_upload_dir: Path, preferred_prefix: str) -> str:
|
||||
normalized = value.strip().replace("\\", "/")
|
||||
lowered = normalized.casefold()
|
||||
upload_root = source_upload_dir.resolve().as_posix().casefold().rstrip("/")
|
||||
if lowered.startswith(upload_root + "/"):
|
||||
normalized = normalized[len(source_upload_dir.resolve().as_posix()) + 1 :]
|
||||
lowered = normalized.casefold()
|
||||
|
||||
if lowered.startswith("/uploads/"):
|
||||
normalized = normalized[len("/uploads/") :]
|
||||
lowered = normalized.casefold()
|
||||
elif lowered.startswith("uploads/"):
|
||||
normalized = normalized[len("uploads/") :]
|
||||
lowered = normalized.casefold()
|
||||
elif lowered.startswith("data/"):
|
||||
normalized = normalized[len("data/") :]
|
||||
lowered = normalized.casefold()
|
||||
|
||||
if preferred_prefix == "persons/" and lowered.startswith("portraits/"):
|
||||
normalized = "persons/" + normalized[len("portraits/") :]
|
||||
lowered = normalized.casefold()
|
||||
|
||||
for prefix in ("documents/", "persons/"):
|
||||
marker = f"/{prefix}"
|
||||
index = lowered.find(marker)
|
||||
if index >= 0:
|
||||
normalized = normalized[index + 1 :]
|
||||
lowered = normalized.casefold()
|
||||
break
|
||||
|
||||
if not lowered.startswith(preferred_prefix):
|
||||
return normalized
|
||||
return Path(normalized).as_posix()
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import inspect as sqlalchemy_inspect
|
||||
from sqlalchemy import text
|
||||
@@ -12,7 +13,6 @@ from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from .engine import resolve_engine
|
||||
from .models import DocumentType
|
||||
from .models import JobSourceStatus
|
||||
from .models import PersonRole
|
||||
from .registries import BUILT_IN_DOCUMENT_TYPES
|
||||
from .registries import BUILT_IN_PERSON_ROLES
|
||||
@@ -32,56 +32,6 @@ async def create_all(*, engine: AsyncEngine | None = None) -> None:
|
||||
logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url)
|
||||
|
||||
|
||||
async def normalize_legacy_status_spellings(*, engine: AsyncEngine | None = None) -> int:
|
||||
"""Normalize legacy enum-name spellings to canonical enum values.
|
||||
|
||||
Historical databases may carry ``TRANSCRIBED``/``FAILED``-style enum *names*
|
||||
in ``job_source.status`` or ``execution_attempt.status``. Runtime models
|
||||
expect canonical lowercase values, so stale rows must be normalized before
|
||||
ORM reads.
|
||||
"""
|
||||
active_engine = engine or resolve_engine()
|
||||
if not hasattr(active_engine, "begin"):
|
||||
return 0
|
||||
replacements = {
|
||||
status.name: status.value
|
||||
for status in JobSourceStatus
|
||||
if status.name != status.value
|
||||
}
|
||||
|
||||
def _normalize(sync_connection) -> int:
|
||||
inspector = sqlalchemy_inspect(sync_connection)
|
||||
table_names = set(inspector.get_table_names())
|
||||
if not table_names:
|
||||
return 0
|
||||
fixed = 0
|
||||
for table_name in ("job_source", "execution_attempt"):
|
||||
if table_name not in table_names:
|
||||
continue
|
||||
for legacy, canonical in replacements.items():
|
||||
result = sync_connection.execute(
|
||||
text(f'update "{table_name}" set status = :canonical where status = :legacy'),
|
||||
{"canonical": canonical, "legacy": legacy},
|
||||
)
|
||||
fixed += result.rowcount or 0
|
||||
return fixed
|
||||
|
||||
async with active_engine.begin() as connection:
|
||||
fixed_rows = await connection.run_sync(_normalize)
|
||||
if fixed_rows:
|
||||
logger.warning("Normalized %s legacy status row(s) to canonical spellings", fixed_rows)
|
||||
return fixed_rows
|
||||
|
||||
|
||||
LEGACY_JOB_SOURCE_COLUMNS = (
|
||||
"raw_transcription",
|
||||
"ai_metadata",
|
||||
"raw_api_response",
|
||||
"error_detail",
|
||||
"executed_at",
|
||||
)
|
||||
|
||||
|
||||
async def reconcile_legacy_job_source_columns(*, engine: AsyncEngine | None = None) -> int:
|
||||
"""Remove stale V4.6 ``job_source`` evidence columns from existing databases.
|
||||
|
||||
@@ -100,7 +50,13 @@ async def reconcile_legacy_job_source_columns(*, engine: AsyncEngine | None = No
|
||||
return 0
|
||||
present_columns = {column["name"] for column in inspector.get_columns("job_source")}
|
||||
dropped = 0
|
||||
for column_name in LEGACY_JOB_SOURCE_COLUMNS:
|
||||
for column_name in (
|
||||
"raw_transcription",
|
||||
"ai_metadata",
|
||||
"raw_api_response",
|
||||
"error_detail",
|
||||
"executed_at",
|
||||
):
|
||||
if column_name not in present_columns:
|
||||
continue
|
||||
sync_connection.execute(text(f'alter table "job_source" drop column "{column_name}"'))
|
||||
@@ -114,6 +70,95 @@ async def reconcile_legacy_job_source_columns(*, engine: AsyncEngine | None = No
|
||||
return dropped_columns
|
||||
|
||||
|
||||
async def reconcile_canonical_media_paths(*, engine: AsyncEngine | None = None) -> int:
|
||||
"""Normalize stored media paths to upload-root-relative POSIX form."""
|
||||
active_engine = engine or resolve_engine()
|
||||
if not hasattr(active_engine, "begin"):
|
||||
return 0
|
||||
|
||||
def _reconcile(sync_connection) -> int:
|
||||
rows_changed = 0
|
||||
inspector = sqlalchemy_inspect(sync_connection)
|
||||
table_names = set(inspector.get_table_names())
|
||||
if "source" in table_names:
|
||||
rows = sync_connection.execute(text('select id, file_path from "source" where file_path is not null')).mappings().all()
|
||||
for row in rows:
|
||||
original = str(row["file_path"])
|
||||
normalized = _canonical_relative_path(original, preferred_prefix="documents/")
|
||||
if normalized is None or normalized == original:
|
||||
continue
|
||||
sync_connection.execute(
|
||||
text('update "source" set file_path = :file_path where id = :id'),
|
||||
{"id": row["id"], "file_path": normalized},
|
||||
)
|
||||
rows_changed += 1
|
||||
|
||||
if "person" in table_names:
|
||||
rows = sync_connection.execute(
|
||||
text('select id, portrait_path from "person" where portrait_path is not null')
|
||||
).mappings().all()
|
||||
for row in rows:
|
||||
original = str(row["portrait_path"])
|
||||
normalized = _canonical_relative_path(original, preferred_prefix="persons/")
|
||||
if normalized is None or normalized == original:
|
||||
continue
|
||||
sync_connection.execute(
|
||||
text('update "person" set portrait_path = :portrait_path where id = :id'),
|
||||
{"id": row["id"], "portrait_path": normalized},
|
||||
)
|
||||
rows_changed += 1
|
||||
return rows_changed
|
||||
|
||||
async with active_engine.begin() as connection:
|
||||
rows_changed = await connection.run_sync(_reconcile)
|
||||
if rows_changed:
|
||||
logger.warning("Normalized %s media-path row(s) to canonical relative format", rows_changed)
|
||||
return rows_changed
|
||||
|
||||
|
||||
def _canonical_relative_path(value: str, *, preferred_prefix: str) -> str | None:
|
||||
normalized = value.strip().replace("\\", "/")
|
||||
if not normalized:
|
||||
return None
|
||||
|
||||
lowered = normalized.casefold()
|
||||
if lowered.startswith("http://") or lowered.startswith("https://") or lowered.startswith("data:"):
|
||||
return None
|
||||
|
||||
if lowered.startswith("/uploads/"):
|
||||
normalized = normalized[len("/uploads/") :]
|
||||
lowered = normalized.casefold()
|
||||
elif lowered.startswith("uploads/"):
|
||||
normalized = normalized[len("uploads/") :]
|
||||
lowered = normalized.casefold()
|
||||
elif lowered.startswith("data/"):
|
||||
normalized = normalized[len("data/") :]
|
||||
lowered = normalized.casefold()
|
||||
|
||||
for prefix in ("documents/", "persons/", "portraits/"):
|
||||
marker = f"/{prefix}"
|
||||
index = lowered.find(marker)
|
||||
if index >= 0:
|
||||
normalized = normalized[index + 1 :]
|
||||
lowered = normalized.casefold()
|
||||
break
|
||||
if lowered.startswith(prefix):
|
||||
break
|
||||
|
||||
if preferred_prefix == "persons/" and lowered.startswith("portraits/"):
|
||||
normalized = "persons/" + normalized[len("portraits/") :]
|
||||
lowered = normalized.casefold()
|
||||
|
||||
if not lowered.startswith(preferred_prefix):
|
||||
return None
|
||||
|
||||
# Collapse any accidental "." segments while preserving relative semantics.
|
||||
collapsed = Path(normalized).as_posix()
|
||||
if collapsed.startswith("../") or collapsed == "..":
|
||||
return None
|
||||
return collapsed
|
||||
|
||||
|
||||
async def seed_registry_defaults(*, engine: AsyncEngine | None = None) -> None:
|
||||
"""Seed default registry rows for role and document type taxonomies."""
|
||||
active_engine = engine or resolve_engine()
|
||||
|
||||
@@ -54,16 +54,6 @@ class RegistrySummary:
|
||||
is_built_in: bool
|
||||
reference_count: int
|
||||
|
||||
@property
|
||||
def document_count(self) -> int:
|
||||
"""Backward-compatible alias for document-type settings consumers."""
|
||||
return self.reference_count
|
||||
|
||||
@property
|
||||
def link_count(self) -> int:
|
||||
"""Backward-compatible alias for person-role settings consumers."""
|
||||
return self.reference_count
|
||||
|
||||
|
||||
class RegistryService[ModelT: RegistryEntry](ServiceBase):
|
||||
"""Generic create/read/update/delete behavior for a registry table.
|
||||
|
||||
@@ -95,14 +95,14 @@ class ProviderInput:
|
||||
media_type: str
|
||||
|
||||
|
||||
def build_provider_input(source: Source) -> ProviderInput:
|
||||
def build_provider_input(source: Source, *, upload_dir: Path) -> ProviderInput:
|
||||
"""Describe the stored Source bytes that a provider request will carry.
|
||||
|
||||
Stored pages are normalized upright at ingest, so the file on disk is the
|
||||
exact payload sent to the provider and ``file_hash`` already identifies it.
|
||||
"""
|
||||
return ProviderInput(
|
||||
path=Path(source.file_path),
|
||||
path=(upload_dir / Path(source.file_path)).resolve(),
|
||||
digest_sha256=source.file_hash.lower(),
|
||||
byte_size=source.file_size_bytes,
|
||||
media_type=source_mime_type(source.file_path),
|
||||
|
||||
@@ -115,6 +115,7 @@ async def create_document_job(
|
||||
stored_path=stored_path,
|
||||
file_hash=stored.file_hash,
|
||||
file_size_bytes=stored.file_size_bytes,
|
||||
upload_dir=runtime_settings.upload_dir,
|
||||
prompt_execution=prompt_execution,
|
||||
)
|
||||
except Exception as exc:
|
||||
@@ -191,6 +192,7 @@ async def create_job_for_document(
|
||||
stored_sources=stored_sources,
|
||||
provider=provider,
|
||||
model=model,
|
||||
upload_dir=runtime_settings.upload_dir,
|
||||
prompt_execution=prompt_execution,
|
||||
)
|
||||
except Exception as exc:
|
||||
@@ -219,6 +221,7 @@ async def _create_document_job_records(
|
||||
stored_path: Path,
|
||||
file_hash: str,
|
||||
file_size_bytes: int,
|
||||
upload_dir: Path,
|
||||
prompt_execution,
|
||||
) -> tuple[Document, Job]:
|
||||
document = Document(
|
||||
@@ -246,7 +249,7 @@ async def _create_document_job_records(
|
||||
page_number=1,
|
||||
upload_name=Path(original_filename).name,
|
||||
filename=stored_path.name,
|
||||
file_path=str(stored_path),
|
||||
file_path=_upload_relative_path(stored_path=stored_path, upload_dir=upload_dir),
|
||||
file_hash=file_hash,
|
||||
file_size_bytes=file_size_bytes,
|
||||
)
|
||||
@@ -274,6 +277,7 @@ async def _create_job_for_document_records(
|
||||
stored_sources: Sequence[PendingStoredSource],
|
||||
provider: str | None,
|
||||
model: str | None,
|
||||
upload_dir: Path,
|
||||
prompt_execution,
|
||||
) -> tuple[Job, list[UUID]]:
|
||||
document = await session.get(Document, document_id)
|
||||
@@ -309,7 +313,10 @@ async def _create_job_for_document_records(
|
||||
page_number=next_page_number + page_offset,
|
||||
upload_name=Path(stored_source.original_filename).name,
|
||||
filename=stored_source.stored_path.name,
|
||||
file_path=str(stored_source.stored_path),
|
||||
file_path=_upload_relative_path(
|
||||
stored_path=stored_source.stored_path,
|
||||
upload_dir=upload_dir,
|
||||
),
|
||||
file_hash=stored_source.file_hash,
|
||||
file_size_bytes=stored_source.file_size_bytes,
|
||||
)
|
||||
@@ -338,6 +345,10 @@ def _best_effort_delete(path: Path) -> None:
|
||||
logger.warning("Failed to clean up Source file after database error: %s", path)
|
||||
|
||||
|
||||
def _upload_relative_path(*, stored_path: Path, upload_dir: Path) -> str:
|
||||
return stored_path.resolve().relative_to(upload_dir.resolve()).as_posix()
|
||||
|
||||
|
||||
async def store_source_file(
|
||||
*,
|
||||
filename: str,
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
import inspect
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC
|
||||
@@ -24,7 +23,6 @@ from ..errors import format_error_detail
|
||||
from ..providers import ProviderError
|
||||
from ..providers import RequestManifest
|
||||
from ..providers import SourceEvidenceReference
|
||||
from ..providers import TranscriptionProvider
|
||||
from ..providers import TranscriptionResult
|
||||
from ..providers import TransportEvidence
|
||||
from . import ServiceBundle
|
||||
@@ -225,7 +223,7 @@ async def process_queued_job( # noqa: PLR0915
|
||||
provider_input = None
|
||||
page_outcome: _SuccessfulPage | _FailedPage
|
||||
try:
|
||||
provider_input = build_provider_input(source)
|
||||
provider_input = build_provider_input(source, upload_dir=runtime_settings.upload_dir)
|
||||
source_reference = SourceEvidenceReference(
|
||||
source_id=source.id,
|
||||
digest_sha256=provider_input.digest_sha256,
|
||||
@@ -238,9 +236,12 @@ async def process_queued_job( # noqa: PLR0915
|
||||
# raised before this point, which would otherwise leave it unbound.
|
||||
monotonic_started_at = asyncio.get_running_loop().time()
|
||||
result = await asyncio.wait_for(
|
||||
_call_transcriber(
|
||||
input_path=provider_input.path,
|
||||
prompt_execution=prompt_execution,
|
||||
transcribe_document_image(
|
||||
provider_input.path,
|
||||
prompt_name=prompt_execution.prompt_name,
|
||||
prompt_text=prompt_execution.user_prompt,
|
||||
temperature=prompt_execution.temperature,
|
||||
top_p=prompt_execution.top_p,
|
||||
settings=runtime_settings,
|
||||
provider=provider,
|
||||
source_reference=source_reference,
|
||||
@@ -697,36 +698,3 @@ def _find_provider_error(exc: BaseException) -> ProviderError | None:
|
||||
return current
|
||||
current = current.__cause__ or current.__context__
|
||||
return None
|
||||
|
||||
|
||||
async def _call_transcriber(
|
||||
*,
|
||||
input_path,
|
||||
prompt_execution: PromptExecution,
|
||||
settings: Settings,
|
||||
provider: TranscriptionProvider,
|
||||
source_reference: SourceEvidenceReference,
|
||||
requested_model: str | None,
|
||||
) -> TranscriptionResult:
|
||||
"""Call the current transcriber while supporting legacy injected test doubles."""
|
||||
if "source_reference" in inspect.signature(transcribe_document_image).parameters:
|
||||
return await transcribe_document_image(
|
||||
input_path,
|
||||
prompt_name=prompt_execution.prompt_name,
|
||||
prompt_text=prompt_execution.user_prompt,
|
||||
temperature=prompt_execution.temperature,
|
||||
top_p=prompt_execution.top_p,
|
||||
settings=settings,
|
||||
provider=provider,
|
||||
source_reference=source_reference,
|
||||
requested_model=requested_model,
|
||||
)
|
||||
return await transcribe_document_image(
|
||||
input_path,
|
||||
prompt_name=prompt_execution.prompt_name,
|
||||
prompt_text=prompt_execution.user_prompt,
|
||||
temperature=prompt_execution.temperature,
|
||||
top_p=prompt_execution.top_p,
|
||||
settings=settings,
|
||||
provider=provider,
|
||||
)
|
||||
|
||||
@@ -4,10 +4,10 @@ from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
from urllib.parse import unquote
|
||||
|
||||
_ABSOLUTE_SCHEMES = ("http://", "https://", "data:")
|
||||
_UPLOAD_ROUTE_PREFIX = "/uploads/"
|
||||
_CANONICAL_PREFIXES = ("documents/", "persons/")
|
||||
|
||||
|
||||
def absolute_upload_url(path: str, *, base_url: str) -> str:
|
||||
@@ -18,13 +18,7 @@ def absolute_upload_url(path: str, *, base_url: str) -> str:
|
||||
|
||||
|
||||
def resolve_media_url(path: str | None, *, upload_dir: Path, base_url: str) -> str | None:
|
||||
"""Map a stored media path onto a served upload URL.
|
||||
|
||||
Stored paths have accumulated several shapes over the life of the schema:
|
||||
absolute filesystem paths, paths relative to the working directory, paths
|
||||
relative to the upload root, and paths that already carry an upload route.
|
||||
All of them must still resolve, so each shape is tried in turn.
|
||||
"""
|
||||
"""Map a canonical stored relative media path onto a served upload URL."""
|
||||
candidate = (path or "").strip()
|
||||
if not candidate:
|
||||
return None
|
||||
@@ -34,56 +28,23 @@ def resolve_media_url(path: str | None, *, upload_dir: Path, base_url: str) -> s
|
||||
if lowered.startswith(_ABSOLUTE_SCHEMES):
|
||||
return normalized
|
||||
if normalized.startswith(_UPLOAD_ROUTE_PREFIX):
|
||||
upload_relative = normalized.removeprefix(_UPLOAD_ROUTE_PREFIX)
|
||||
return _resolve_upload_relative(upload_relative, upload_dir=upload_dir, base_url=base_url)
|
||||
|
||||
resolved_upload_dir = upload_dir.resolve()
|
||||
path_obj = Path(candidate)
|
||||
|
||||
if path_obj.is_absolute():
|
||||
absolute_candidates = [path_obj.resolve()]
|
||||
else:
|
||||
absolute_candidates = [
|
||||
(Path.cwd() / path_obj).resolve(),
|
||||
(resolved_upload_dir / path_obj).resolve(),
|
||||
]
|
||||
|
||||
for absolute_candidate in absolute_candidates:
|
||||
try:
|
||||
relative = absolute_candidate.relative_to(resolved_upload_dir).as_posix()
|
||||
except ValueError:
|
||||
continue
|
||||
if absolute_candidate.is_file():
|
||||
return absolute_upload_url(f"/uploads/{quote(relative)}", base_url=base_url)
|
||||
|
||||
upload_name = resolved_upload_dir.name.casefold()
|
||||
normalized_parts = Path(normalized).parts
|
||||
lowered_parts = [part.casefold() for part in normalized_parts]
|
||||
if upload_name in lowered_parts:
|
||||
index = lowered_parts.index(upload_name)
|
||||
relative = Path(*normalized_parts[index + 1 :]).as_posix()
|
||||
if relative:
|
||||
return _resolve_upload_relative(relative, upload_dir=upload_dir, base_url=base_url)
|
||||
|
||||
if lowered.startswith("uploads/"):
|
||||
upload_relative = normalized.split("/", 1)[1] if "/" in normalized else ""
|
||||
return _resolve_upload_relative(upload_relative, upload_dir=upload_dir, base_url=base_url)
|
||||
if lowered.startswith("data/"):
|
||||
relative = normalized.split("/", 1)[1] if "/" in normalized else ""
|
||||
return _resolve_upload_relative(relative, upload_dir=upload_dir, base_url=base_url)
|
||||
if lowered.startswith(("documents/", "persons/")):
|
||||
return _resolve_upload_relative(normalized.removeprefix(_UPLOAD_ROUTE_PREFIX), upload_dir=upload_dir, base_url=base_url)
|
||||
if lowered.startswith(_CANONICAL_PREFIXES):
|
||||
return _resolve_upload_relative(normalized, upload_dir=upload_dir, base_url=base_url)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_upload_relative(relative: str, *, upload_dir: Path, base_url: str) -> str | None:
|
||||
candidate = unquote(relative).strip().replace("\\", "/")
|
||||
candidate = relative.strip().replace("\\", "/")
|
||||
if not candidate:
|
||||
return None
|
||||
resolved_upload_dir = upload_dir.resolve()
|
||||
try:
|
||||
absolute_candidate = (resolved_upload_dir / Path(candidate)).resolve()
|
||||
normalized_relative = Path(candidate).as_posix()
|
||||
if not normalized_relative.casefold().startswith(_CANONICAL_PREFIXES):
|
||||
return None
|
||||
absolute_candidate = (resolved_upload_dir / Path(normalized_relative)).resolve()
|
||||
safe_relative = absolute_candidate.relative_to(resolved_upload_dir).as_posix()
|
||||
except ValueError:
|
||||
return None
|
||||
@@ -94,6 +55,7 @@ def _resolve_upload_relative(relative: str, *, upload_dir: Path, base_url: str)
|
||||
|
||||
def public_media_path_label(path: str | None, *, upload_dir: Path) -> str:
|
||||
"""Return a safe, non-local path label for UI metadata display."""
|
||||
_ = upload_dir
|
||||
candidate = (path or "").strip()
|
||||
if not candidate:
|
||||
return "unknown"
|
||||
@@ -103,23 +65,6 @@ def public_media_path_label(path: str | None, *, upload_dir: Path) -> str:
|
||||
|
||||
if normalized.startswith(_UPLOAD_ROUTE_PREFIX):
|
||||
return normalized
|
||||
if lowered.startswith("uploads/"):
|
||||
return f"/{normalized}"
|
||||
if lowered.startswith("data/"):
|
||||
relative = normalized.split("/", 1)[1] if "/" in normalized else ""
|
||||
return f"/uploads/{relative}" if relative else "/uploads"
|
||||
if lowered.startswith(("documents/", "persons/")):
|
||||
if lowered.startswith(_CANONICAL_PREFIXES):
|
||||
return f"/uploads/{normalized}"
|
||||
|
||||
resolved_upload_dir = upload_dir.resolve()
|
||||
path_obj = Path(candidate)
|
||||
if path_obj.is_absolute():
|
||||
absolute_candidate = path_obj.resolve()
|
||||
try:
|
||||
relative = absolute_candidate.relative_to(resolved_upload_dir).as_posix()
|
||||
return f"/uploads/{relative}"
|
||||
except ValueError:
|
||||
# Never expose non-managed absolute filesystem paths.
|
||||
return path_obj.name or "unknown"
|
||||
|
||||
return f"/uploads/{quote(path_obj.name)}"
|
||||
return Path(normalized).name or "unknown"
|
||||
|
||||
@@ -62,7 +62,7 @@ def register_page(*, settings: Settings) -> None: # noqa: PLR0915
|
||||
{
|
||||
"id": str(item.id),
|
||||
"label": item.label,
|
||||
"document_count": item.document_count,
|
||||
"reference_count": item.reference_count,
|
||||
"is_active": item.is_active,
|
||||
"is_built_in": item.is_built_in,
|
||||
}
|
||||
@@ -70,7 +70,7 @@ def register_page(*, settings: Settings) -> None: # noqa: PLR0915
|
||||
]
|
||||
table = render_registry_table(
|
||||
rows,
|
||||
count_field="document_count",
|
||||
count_field="reference_count",
|
||||
count_label="Documents",
|
||||
)
|
||||
|
||||
@@ -179,7 +179,7 @@ def register_page(*, settings: Settings) -> None: # noqa: PLR0915
|
||||
{
|
||||
"id": str(role.id),
|
||||
"label": role.label,
|
||||
"link_count": role.link_count,
|
||||
"reference_count": role.reference_count,
|
||||
"is_active": role.is_active,
|
||||
"is_built_in": role.is_built_in,
|
||||
}
|
||||
@@ -187,7 +187,7 @@ def register_page(*, settings: Settings) -> None: # noqa: PLR0915
|
||||
]
|
||||
table = render_registry_table(
|
||||
rows,
|
||||
count_field="link_count",
|
||||
count_field="reference_count",
|
||||
count_label="Links",
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user