generated from john/python-template
ruff and settings
This commit is contained in:
@@ -4,10 +4,13 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from fastapi import FastAPI, Request
|
from fastapi import FastAPI
|
||||||
|
from fastapi import Request
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
from transcription.errors import AppError, ErrorCategory, build_error_envelope
|
from transcription.errors import AppError
|
||||||
|
from transcription.errors import ErrorCategory
|
||||||
|
from transcription.errors import build_error_envelope
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|||||||
+17
-18
@@ -3,21 +3,20 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from threading import Event, Thread
|
from threading import Event
|
||||||
|
from threading import Thread
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
|
|
||||||
from transcription.api.errors import register_error_handlers
|
from .api.errors import register_error_handlers
|
||||||
from transcription.api.health import router as health_router
|
from .api.health import router as health_router
|
||||||
from transcription.config import get_settings, setup_logging
|
from .config import get_settings
|
||||||
from transcription.db import (
|
from .config import setup_logging
|
||||||
create_all,
|
from .db import cleanup_database
|
||||||
dispose_database_runtime,
|
from .db import create_all
|
||||||
initialize_database_runtime,
|
from .db import get_engine
|
||||||
should_bootstrap_schema,
|
from .ui import register_pages
|
||||||
)
|
from .worker import run_worker_loop
|
||||||
from transcription.ui import register_pages
|
|
||||||
from transcription.worker import run_worker_loop
|
|
||||||
|
|
||||||
|
|
||||||
def _start_worker(app: FastAPI) -> None:
|
def _start_worker(app: FastAPI) -> None:
|
||||||
@@ -25,7 +24,7 @@ def _start_worker(app: FastAPI) -> None:
|
|||||||
worker_thread = Thread(
|
worker_thread = Thread(
|
||||||
target=run_worker_loop,
|
target=run_worker_loop,
|
||||||
kwargs={
|
kwargs={
|
||||||
"engine": app.state.db_runtime.engine,
|
"engine": app.state.db_conn,
|
||||||
"stop_event": stop_event,
|
"stop_event": stop_event,
|
||||||
"poll_interval_seconds": 1.0,
|
"poll_interval_seconds": 1.0,
|
||||||
},
|
},
|
||||||
@@ -52,10 +51,11 @@ async def _lifespan(app: FastAPI):
|
|||||||
|
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
app.state.settings = settings
|
app.state.settings = settings
|
||||||
app.state.db_runtime = initialize_database_runtime(settings=settings)
|
engine = get_engine()
|
||||||
|
app.state.db_conn = engine
|
||||||
|
|
||||||
if should_bootstrap_schema(settings):
|
if settings.should_bootstrap_schema:
|
||||||
create_all(engine=app.state.db_runtime.engine)
|
create_all(engine=engine)
|
||||||
|
|
||||||
settings.upload_dir.mkdir(parents=True, exist_ok=True)
|
settings.upload_dir.mkdir(parents=True, exist_ok=True)
|
||||||
settings.prompt_dir.mkdir(parents=True, exist_ok=True)
|
settings.prompt_dir.mkdir(parents=True, exist_ok=True)
|
||||||
@@ -65,7 +65,7 @@ async def _lifespan(app: FastAPI):
|
|||||||
yield
|
yield
|
||||||
finally:
|
finally:
|
||||||
_stop_worker(app)
|
_stop_worker(app)
|
||||||
dispose_database_runtime()
|
cleanup_database()
|
||||||
|
|
||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
@@ -75,4 +75,3 @@ def create_app() -> FastAPI:
|
|||||||
register_pages(app)
|
register_pages(app)
|
||||||
app.include_router(health_router)
|
app.include_router(health_router)
|
||||||
return app
|
return app
|
||||||
|
|
||||||
|
|||||||
+31
-12
@@ -5,14 +5,16 @@ once at startup. Provider-specific defaults (model names, base URLs)
|
|||||||
are resolved by the provider adapters, not here.
|
are resolved by the provider adapters, not here.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
import logging.config
|
import logging.config
|
||||||
|
from contextvars import ContextVar
|
||||||
from enum import StrEnum
|
from enum import StrEnum
|
||||||
from functools import lru_cache
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
|
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings
|
||||||
|
from pydantic_settings import SettingsConfigDict
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class Provider(StrEnum):
|
class Provider(StrEnum):
|
||||||
@@ -48,6 +50,24 @@ class Settings(BaseSettings):
|
|||||||
worker_max_retries: int = 0
|
worker_max_retries: int = 0
|
||||||
worker_retry_backoff_seconds: float = 0.0
|
worker_retry_backoff_seconds: float = 0.0
|
||||||
|
|
||||||
|
@property
|
||||||
|
def should_bootstrap_schema(self) -> bool:
|
||||||
|
"""Return whether startup should auto-create schema for this environment."""
|
||||||
|
if self.bootstrap_schema_on_startup is not None:
|
||||||
|
return self.bootstrap_schema_on_startup
|
||||||
|
return self.environment in {"development", "test"}
|
||||||
|
|
||||||
|
|
||||||
|
_settings: ContextVar[Settings | None] = ContextVar("settings", default=None)
|
||||||
|
|
||||||
|
|
||||||
|
def get_settings() -> Settings:
|
||||||
|
settings = _settings.get()
|
||||||
|
if settings is None:
|
||||||
|
settings = Settings() # pyright: ignore[reportCallIssue]
|
||||||
|
_settings.set(settings)
|
||||||
|
return settings
|
||||||
|
|
||||||
|
|
||||||
LOGGING_CONFIG: dict[str, object] = {
|
LOGGING_CONFIG: dict[str, object] = {
|
||||||
"version": 1,
|
"version": 1,
|
||||||
@@ -69,18 +89,17 @@ LOGGING_CONFIG: dict[str, object] = {
|
|||||||
"level": "INFO",
|
"level": "INFO",
|
||||||
"handlers": ["console"],
|
"handlers": ["console"],
|
||||||
},
|
},
|
||||||
|
"loggers": {
|
||||||
|
"transcription": {
|
||||||
|
"level": "DEBUG",
|
||||||
|
"handlers": ["console"],
|
||||||
|
"propagate": False,
|
||||||
|
}
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@lru_cache(maxsize=1)
|
|
||||||
def get_settings() -> Settings:
|
|
||||||
"""Return the singleton Settings instance.
|
|
||||||
|
|
||||||
Cached so the entire application shares one validated config.
|
|
||||||
"""
|
|
||||||
return Settings()
|
|
||||||
|
|
||||||
|
|
||||||
def setup_logging() -> None:
|
def setup_logging() -> None:
|
||||||
"""Configure root logging once at startup."""
|
"""Configure root logging once at startup."""
|
||||||
logging.config.dictConfig(LOGGING_CONFIG)
|
logging.config.dictConfig(LOGGING_CONFIG)
|
||||||
|
logger.debug("Logging configured")
|
||||||
|
|||||||
+24
-53
@@ -7,70 +7,44 @@ startup/shutdown behavior is predictable and lifespan-managed.
|
|||||||
import contextlib
|
import contextlib
|
||||||
import logging
|
import logging
|
||||||
from collections.abc import Generator
|
from collections.abc import Generator
|
||||||
from dataclasses import dataclass
|
from functools import cache
|
||||||
|
|
||||||
from sqlalchemy import inspect, text
|
from sqlalchemy import inspect
|
||||||
|
from sqlalchemy import text
|
||||||
from sqlalchemy.engine import Engine
|
from sqlalchemy.engine import Engine
|
||||||
from sqlmodel import Session, SQLModel, create_engine
|
from sqlmodel import Session
|
||||||
|
from sqlmodel import SQLModel
|
||||||
|
from sqlmodel import create_engine
|
||||||
|
|
||||||
from transcription.config import Settings, get_settings
|
from .config import Settings
|
||||||
|
from .config import get_settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class DatabaseRuntime:
|
|
||||||
"""Process-level database runtime resources."""
|
|
||||||
|
|
||||||
engine: Engine
|
|
||||||
|
|
||||||
|
|
||||||
_runtime: DatabaseRuntime | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def _build_engine(settings: Settings) -> Engine:
|
def _build_engine(settings: Settings) -> Engine:
|
||||||
connect_args: dict[str, object] = {}
|
connect_args: dict[str, object] = {}
|
||||||
if settings.database_url.startswith("sqlite"):
|
if settings.database_url.startswith("sqlite"):
|
||||||
connect_args["check_same_thread"] = False
|
connect_args["check_same_thread"] = False
|
||||||
return create_engine(
|
return create_engine(
|
||||||
settings.database_url,
|
url=settings.database_url,
|
||||||
echo=False,
|
echo=False,
|
||||||
connect_args=connect_args,
|
connect_args=connect_args,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def initialize_database_runtime(*, settings: Settings | None = None) -> DatabaseRuntime:
|
@cache
|
||||||
"""Initialize and cache the process database runtime once."""
|
def get_engine() -> Engine:
|
||||||
global _runtime
|
"""Return a new SQLAlchemy engine for the current settings."""
|
||||||
|
settings = get_settings()
|
||||||
if _runtime is not None:
|
logger.debug("Creating new SQLAlchemy engine for database_url=%s", settings.database_url)
|
||||||
return _runtime
|
return _build_engine(settings)
|
||||||
|
|
||||||
runtime_settings = settings or get_settings()
|
|
||||||
_runtime = DatabaseRuntime(engine=_build_engine(runtime_settings))
|
|
||||||
return _runtime
|
|
||||||
|
|
||||||
|
|
||||||
def get_database_runtime() -> DatabaseRuntime:
|
def cleanup_database() -> None:
|
||||||
"""Return initialized database runtime, creating it if needed."""
|
"""Cleanup database runtime resources."""
|
||||||
if _runtime is None:
|
engine = get_engine()
|
||||||
return initialize_database_runtime()
|
engine.dispose()
|
||||||
return _runtime
|
|
||||||
|
|
||||||
|
|
||||||
def dispose_database_runtime() -> None:
|
|
||||||
"""Dispose process database runtime resources."""
|
|
||||||
global _runtime
|
|
||||||
if _runtime is not None:
|
|
||||||
_runtime.engine.dispose()
|
|
||||||
_runtime = None
|
|
||||||
|
|
||||||
|
|
||||||
def should_bootstrap_schema(settings: Settings) -> bool:
|
|
||||||
"""Return whether startup should auto-create schema for this environment."""
|
|
||||||
if settings.bootstrap_schema_on_startup is not None:
|
|
||||||
return settings.bootstrap_schema_on_startup
|
|
||||||
return settings.environment in {"development", "test"}
|
|
||||||
|
|
||||||
|
|
||||||
def create_all(*, engine: Engine | None = None) -> None:
|
def create_all(*, engine: Engine | None = None) -> None:
|
||||||
@@ -78,9 +52,10 @@ def create_all(*, engine: Engine | None = None) -> None:
|
|||||||
# Import models so SQLModel metadata is fully registered before bootstrap.
|
# Import models so SQLModel metadata is fully registered before bootstrap.
|
||||||
from transcription import models as _models # noqa: F401
|
from transcription import models as _models # noqa: F401
|
||||||
|
|
||||||
active_engine = engine or get_database_runtime().engine
|
active_engine = engine or get_engine()
|
||||||
SQLModel.metadata.create_all(active_engine)
|
SQLModel.metadata.create_all(active_engine)
|
||||||
_ensure_sqlite_compat_columns(active_engine)
|
_ensure_sqlite_compat_columns(active_engine)
|
||||||
|
logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url)
|
||||||
|
|
||||||
|
|
||||||
def _ensure_sqlite_compat_columns(engine: Engine) -> None:
|
def _ensure_sqlite_compat_columns(engine: Engine) -> None:
|
||||||
@@ -100,17 +75,13 @@ def _ensure_sqlite_compat_columns(engine: Engine) -> None:
|
|||||||
columns = {column["name"] for column in inspector.get_columns("job")}
|
columns = {column["name"] for column in inspector.get_columns("job")}
|
||||||
if "retry_count" not in columns:
|
if "retry_count" not in columns:
|
||||||
with engine.begin() as connection:
|
with engine.begin() as connection:
|
||||||
connection.execute(
|
connection.execute(text("ALTER TABLE job ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0"))
|
||||||
text("ALTER TABLE job ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0")
|
logger.warning("Applied SQLite compatibility schema patch table=job column=retry_count default=0")
|
||||||
)
|
|
||||||
logger.warning(
|
|
||||||
"Applied SQLite compatibility schema patch table=job column=retry_count default=0"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
def get_session(*, engine: Engine | None = None) -> Generator[Session]:
|
def get_session(*, engine: Engine | None = None) -> Generator[Session]:
|
||||||
"""Yield a database session and ensure cleanup."""
|
"""Yield a database session and ensure cleanup."""
|
||||||
active_engine = engine or get_database_runtime().engine
|
active_engine = engine or get_engine()
|
||||||
with Session(active_engine) as session:
|
with Session(active_engine) as session:
|
||||||
yield session
|
yield session
|
||||||
|
|||||||
@@ -3,7 +3,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime, timezone
|
from datetime import UTC
|
||||||
|
from datetime import datetime
|
||||||
from enum import StrEnum
|
from enum import StrEnum
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
@@ -64,7 +65,7 @@ def build_error_envelope(error: AppError) -> ErrorEnvelope:
|
|||||||
category=error.category.value,
|
category=error.category.value,
|
||||||
message=error.message,
|
message=error.message,
|
||||||
suggestion=error.suggestion,
|
suggestion=error.suggestion,
|
||||||
timestamp=datetime.now(timezone.utc).isoformat(),
|
timestamp=datetime.now(UTC).isoformat(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -83,4 +84,4 @@ def format_error_detail(error: AppError) -> str:
|
|||||||
return (
|
return (
|
||||||
f"[{error.category.value}] {error.message} | "
|
f"[{error.category.value}] {error.message} | "
|
||||||
f"suggestion={error.suggestion} | error_id={error.error_id}"
|
f"suggestion={error.suggestion} | error_id={error.error_id}"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -4,12 +4,16 @@ Three models capture the MVP lifecycle:
|
|||||||
Document -> one-to-many -> Job -> one-to-one -> Transcript
|
Document -> one-to-many -> Job -> one-to-one -> Transcript
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import datetime, timezone
|
from datetime import UTC
|
||||||
|
from datetime import datetime
|
||||||
from enum import StrEnum
|
from enum import StrEnum
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from uuid import UUID, uuid4
|
from uuid import UUID
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
from sqlmodel import Field, Relationship, SQLModel
|
from sqlmodel import Field
|
||||||
|
from sqlmodel import Relationship
|
||||||
|
from sqlmodel import SQLModel
|
||||||
|
|
||||||
|
|
||||||
class JobStatus(StrEnum):
|
class JobStatus(StrEnum):
|
||||||
@@ -26,7 +30,7 @@ class Document(SQLModel, table=True):
|
|||||||
filename: str
|
filename: str
|
||||||
file_path: str
|
file_path: str
|
||||||
uploaded_at: datetime = Field(
|
uploaded_at: datetime = Field(
|
||||||
default_factory=lambda: datetime.now(timezone.utc),
|
default_factory=lambda: datetime.now(UTC),
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- relationships ---
|
# --- relationships ---
|
||||||
@@ -41,10 +45,10 @@ class Job(SQLModel, table=True):
|
|||||||
status: JobStatus = Field(default=JobStatus.QUEUED)
|
status: JobStatus = Field(default=JobStatus.QUEUED)
|
||||||
retry_count: int = Field(default=0, ge=0)
|
retry_count: int = Field(default=0, ge=0)
|
||||||
created_at: datetime = Field(
|
created_at: datetime = Field(
|
||||||
default_factory=lambda: datetime.now(timezone.utc),
|
default_factory=lambda: datetime.now(UTC),
|
||||||
)
|
)
|
||||||
updated_at: datetime = Field(
|
updated_at: datetime = Field(
|
||||||
default_factory=lambda: datetime.now(timezone.utc),
|
default_factory=lambda: datetime.now(UTC),
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- relationships ---
|
# --- relationships ---
|
||||||
@@ -60,7 +64,7 @@ class Transcript(SQLModel, table=True):
|
|||||||
text: str | None = None
|
text: str | None = None
|
||||||
error_detail: str | None = None
|
error_detail: str | None = None
|
||||||
created_at: datetime = Field(
|
created_at: datetime = Field(
|
||||||
default_factory=lambda: datetime.now(timezone.utc),
|
default_factory=lambda: datetime.now(UTC),
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- relationships ---
|
# --- relationships ---
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
"""Provider exports and factory for transcription adapters."""
|
"""Provider exports and factory for transcription adapters."""
|
||||||
|
|
||||||
from transcription.config import Provider, Settings, get_settings
|
from transcription.config import Provider
|
||||||
from transcription.providers.base import (
|
from transcription.config import Settings
|
||||||
ProviderAuthError,
|
from transcription.config import get_settings
|
||||||
ProviderError,
|
from transcription.providers.base import ProviderAuthError
|
||||||
ProviderResponseError,
|
from transcription.providers.base import ProviderError
|
||||||
TranscriptionProvider,
|
from transcription.providers.base import ProviderResponseError
|
||||||
TranscriptionResult,
|
from transcription.providers.base import TranscriptionProvider
|
||||||
)
|
from transcription.providers.base import TranscriptionResult
|
||||||
from transcription.providers.openrouter import OpenRouterTranscriptionProvider
|
from transcription.providers.openrouter import OpenRouterTranscriptionProvider
|
||||||
|
|
||||||
|
|
||||||
@@ -21,11 +21,11 @@ def get_transcription_provider(*, settings: Settings | None = None) -> Transcrip
|
|||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
"OpenRouterTranscriptionProvider",
|
||||||
"ProviderAuthError",
|
"ProviderAuthError",
|
||||||
"ProviderError",
|
"ProviderError",
|
||||||
"ProviderResponseError",
|
"ProviderResponseError",
|
||||||
"TranscriptionProvider",
|
"TranscriptionProvider",
|
||||||
"TranscriptionResult",
|
"TranscriptionResult",
|
||||||
"OpenRouterTranscriptionProvider",
|
|
||||||
"get_transcription_provider",
|
"get_transcription_provider",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -9,13 +9,12 @@ from typing import Any
|
|||||||
|
|
||||||
from openrouter import OpenRouter
|
from openrouter import OpenRouter
|
||||||
|
|
||||||
from transcription.config import Settings, get_settings
|
from transcription.config import Settings
|
||||||
from transcription.providers.base import (
|
from transcription.config import get_settings
|
||||||
ProviderAuthError,
|
from transcription.providers.base import ProviderAuthError
|
||||||
ProviderError,
|
from transcription.providers.base import ProviderError
|
||||||
ProviderResponseError,
|
from transcription.providers.base import ProviderResponseError
|
||||||
TranscriptionResult,
|
from transcription.providers.base import TranscriptionResult
|
||||||
)
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -55,7 +54,7 @@ class OpenRouterTranscriptionProvider:
|
|||||||
http_referer=request.http_referer,
|
http_referer=request.http_referer,
|
||||||
x_open_router_title=request.x_open_router_title,
|
x_open_router_title=request.x_open_router_title,
|
||||||
)
|
)
|
||||||
except Exception as exc: # noqa: BLE001
|
except Exception as exc:
|
||||||
message = str(exc).lower()
|
message = str(exc).lower()
|
||||||
if "401" in message or "auth" in message or "api key" in message:
|
if "401" in message or "auth" in message or "api key" in message:
|
||||||
raise ProviderAuthError("OpenRouter authentication failed") from exc
|
raise ProviderAuthError("OpenRouter authentication failed") from exc
|
||||||
@@ -111,10 +110,7 @@ class OpenRouterTranscriptionProvider:
|
|||||||
parts: list[str] = []
|
parts: list[str] = []
|
||||||
for item in content:
|
for item in content:
|
||||||
text_part = None
|
text_part = None
|
||||||
if isinstance(item, dict):
|
text_part = item.get("text") if isinstance(item, dict) else self._get_optional_attr(item, "text")
|
||||||
text_part = item.get("text")
|
|
||||||
else:
|
|
||||||
text_part = self._get_optional_attr(item, "text")
|
|
||||||
|
|
||||||
if isinstance(text_part, str) and text_part.strip():
|
if isinstance(text_part, str) and text_part.strip():
|
||||||
parts.append(text_part.strip())
|
parts.append(text_part.strip())
|
||||||
|
|||||||
@@ -1,30 +1,25 @@
|
|||||||
"""Service layer exports."""
|
"""Service layer exports."""
|
||||||
|
|
||||||
from transcription.services.transcription import (
|
from transcription.services.transcription import DEFAULT_PROMPT_FILE
|
||||||
DEFAULT_PROMPT_FILE,
|
from transcription.services.transcription import PromptLoadError
|
||||||
PromptLoadError,
|
from transcription.services.transcription import TranscriptionError
|
||||||
TranscriptionError,
|
from transcription.services.transcription import load_image_payload
|
||||||
load_image_payload,
|
from transcription.services.transcription import load_prompt_text
|
||||||
load_prompt_text,
|
from transcription.services.transcription import transcribe_document_image
|
||||||
transcribe_document_image,
|
from transcription.services.upload import SUPPORTED_UPLOAD_EXTENSIONS
|
||||||
)
|
from transcription.services.upload import UploadError
|
||||||
from transcription.services.upload import (
|
from transcription.services.upload import UploadJobResult
|
||||||
SUPPORTED_UPLOAD_EXTENSIONS,
|
from transcription.services.upload import create_upload_job
|
||||||
UploadError,
|
|
||||||
UploadJobResult,
|
|
||||||
create_upload_job,
|
|
||||||
)
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"DEFAULT_PROMPT_FILE",
|
"DEFAULT_PROMPT_FILE",
|
||||||
|
"SUPPORTED_UPLOAD_EXTENSIONS",
|
||||||
"PromptLoadError",
|
"PromptLoadError",
|
||||||
"TranscriptionError",
|
"TranscriptionError",
|
||||||
"load_image_payload",
|
|
||||||
"load_prompt_text",
|
|
||||||
"transcribe_document_image",
|
|
||||||
"SUPPORTED_UPLOAD_EXTENSIONS",
|
|
||||||
"UploadError",
|
"UploadError",
|
||||||
"UploadJobResult",
|
"UploadJobResult",
|
||||||
"create_upload_job",
|
"create_upload_job",
|
||||||
|
"load_image_payload",
|
||||||
|
"load_prompt_text",
|
||||||
|
"transcribe_document_image",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -6,16 +6,16 @@ import logging
|
|||||||
import mimetypes
|
import mimetypes
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from transcription.config import Settings, get_settings
|
from transcription.config import Settings
|
||||||
from transcription.errors import AppError, ErrorCategory
|
from transcription.config import get_settings
|
||||||
from transcription.providers import (
|
from transcription.errors import AppError
|
||||||
ProviderAuthError,
|
from transcription.errors import ErrorCategory
|
||||||
ProviderError,
|
from transcription.providers import ProviderAuthError
|
||||||
ProviderResponseError,
|
from transcription.providers import ProviderError
|
||||||
TranscriptionProvider,
|
from transcription.providers import ProviderResponseError
|
||||||
TranscriptionResult,
|
from transcription.providers import TranscriptionProvider
|
||||||
get_transcription_provider,
|
from transcription.providers import TranscriptionResult
|
||||||
)
|
from transcription.providers import get_transcription_provider
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|||||||
@@ -5,14 +5,19 @@ from __future__ import annotations
|
|||||||
import logging
|
import logging
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from uuid import UUID, uuid4
|
from uuid import UUID
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
from sqlmodel import Session
|
from sqlmodel import Session
|
||||||
|
|
||||||
from transcription.config import Settings, get_settings
|
from transcription.config import Settings
|
||||||
|
from transcription.config import get_settings
|
||||||
from transcription.db import get_session
|
from transcription.db import get_session
|
||||||
from transcription.errors import AppError, ErrorCategory
|
from transcription.errors import AppError
|
||||||
from transcription.models import Document, Job, JobStatus
|
from transcription.errors import ErrorCategory
|
||||||
|
from transcription.models import Document
|
||||||
|
from transcription.models import Job
|
||||||
|
from transcription.models import JobStatus
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -69,7 +74,7 @@ def create_upload_job(
|
|||||||
original_filename=filename,
|
original_filename=filename,
|
||||||
stored_path=stored_path,
|
stored_path=stored_path,
|
||||||
)
|
)
|
||||||
except Exception as exc: # noqa: BLE001
|
except Exception as exc:
|
||||||
_best_effort_delete(stored_path)
|
_best_effort_delete(stored_path)
|
||||||
raise UploadError(
|
raise UploadError(
|
||||||
"Failed to create upload database records",
|
"Failed to create upload database records",
|
||||||
@@ -141,4 +146,4 @@ def _best_effort_delete(path: Path) -> None:
|
|||||||
if path.exists():
|
if path.exists():
|
||||||
path.unlink()
|
path.unlink()
|
||||||
except OSError:
|
except OSError:
|
||||||
logger.warning("Failed to clean up upload file after DB error: %s", path)
|
logger.warning("Failed to clean up upload file after DB error: %s", path)
|
||||||
|
|||||||
@@ -12,5 +12,3 @@ def register_pages(app: FastAPI) -> None:
|
|||||||
register_upload_page()
|
register_upload_page()
|
||||||
register_jobs_page()
|
register_jobs_page()
|
||||||
ui.run_with(app, mount_path="/ui", show_welcome_message=False)
|
ui.run_with(app, mount_path="/ui", show_welcome_message=False)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from nicegui import ui
|
from nicegui import ui
|
||||||
|
|
||||||
from transcription.errors import AppError, ErrorCategory, classify_unexpected_error
|
from transcription.errors import AppError
|
||||||
|
from transcription.errors import ErrorCategory
|
||||||
|
from transcription.errors import classify_unexpected_error
|
||||||
|
|
||||||
|
|
||||||
def to_app_error(exc: Exception, *, operation: str) -> AppError:
|
def to_app_error(exc: Exception, *, operation: str) -> AppError:
|
||||||
@@ -37,4 +39,4 @@ def summarize_error(exc: Exception, *, operation: str) -> str:
|
|||||||
error = to_app_error(exc, operation=operation)
|
error = to_app_error(exc, operation=operation)
|
||||||
if error.category == ErrorCategory.INTERNAL_UNEXPECTED:
|
if error.category == ErrorCategory.INTERNAL_UNEXPECTED:
|
||||||
return f"Unexpected error (ref: {error.error_id})"
|
return f"Unexpected error (ref: {error.error_id})"
|
||||||
return f"{error.message} (ref: {error.error_id})"
|
return f"{error.message} (ref: {error.error_id})"
|
||||||
|
|||||||
@@ -9,8 +9,11 @@ from nicegui import ui
|
|||||||
from sqlmodel import select
|
from sqlmodel import select
|
||||||
|
|
||||||
from transcription.db import get_session
|
from transcription.db import get_session
|
||||||
from transcription.models import Document, Job, Transcript
|
from transcription.models import Document
|
||||||
from transcription.ui.error_presenter import show_error, summarize_error
|
from transcription.models import Job
|
||||||
|
from transcription.models import Transcript
|
||||||
|
from transcription.ui.error_presenter import show_error
|
||||||
|
from transcription.ui.error_presenter import summarize_error
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
|
|||||||
@@ -7,8 +7,11 @@ from dataclasses import dataclass
|
|||||||
from nicegui import ui
|
from nicegui import ui
|
||||||
from nicegui.events import UploadEventArguments
|
from nicegui.events import UploadEventArguments
|
||||||
|
|
||||||
from transcription.services.upload import UploadError, UploadJobResult, create_upload_job
|
from transcription.services.upload import UploadError
|
||||||
from transcription.ui.error_presenter import show_error, summarize_error
|
from transcription.services.upload import UploadJobResult
|
||||||
|
from transcription.services.upload import create_upload_job
|
||||||
|
from transcription.ui.error_presenter import show_error
|
||||||
|
from transcription.ui.error_presenter import summarize_error
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|||||||
+19
-10
@@ -4,17 +4,26 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timezone
|
from datetime import UTC
|
||||||
|
from datetime import datetime
|
||||||
from threading import Event
|
from threading import Event
|
||||||
|
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
from sqlalchemy.engine import Engine
|
from sqlalchemy.engine import Engine
|
||||||
from sqlmodel import Session, select
|
from sqlmodel import Session
|
||||||
|
from sqlmodel import select
|
||||||
|
|
||||||
from transcription.config import Settings, get_settings
|
from transcription.config import Settings
|
||||||
|
from transcription.config import get_settings
|
||||||
from transcription.db import get_session
|
from transcription.db import get_session
|
||||||
from transcription.errors import AppError, ErrorCategory, classify_unexpected_error, format_error_detail
|
from transcription.errors import AppError
|
||||||
from transcription.models import Document, Job, JobStatus, Transcript
|
from transcription.errors import ErrorCategory
|
||||||
|
from transcription.errors import classify_unexpected_error
|
||||||
|
from transcription.errors import format_error_detail
|
||||||
|
from transcription.models import Document
|
||||||
|
from transcription.models import Job
|
||||||
|
from transcription.models import JobStatus
|
||||||
|
from transcription.models import Transcript
|
||||||
from transcription.services.transcription import transcribe_document_image
|
from transcription.services.transcription import transcribe_document_image
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -43,7 +52,7 @@ def _process_next_queued_job(*, session: Session) -> bool:
|
|||||||
|
|
||||||
logger.info("Picked queued job operation=worker.pick job_id=%s", job.id)
|
logger.info("Picked queued job operation=worker.pick job_id=%s", job.id)
|
||||||
job.status = JobStatus.PROCESSING
|
job.status = JobStatus.PROCESSING
|
||||||
job.updated_at = datetime.now(timezone.utc)
|
job.updated_at = datetime.now(UTC)
|
||||||
session.add(job)
|
session.add(job)
|
||||||
session.commit()
|
session.commit()
|
||||||
session.refresh(job)
|
session.refresh(job)
|
||||||
@@ -68,7 +77,7 @@ def _process_next_queued_job(*, session: Session) -> bool:
|
|||||||
result = transcribe_document_image(document.file_path)
|
result = transcribe_document_image(document.file_path)
|
||||||
_upsert_transcript(session=session, job_id=job.id, text=result.text, error_detail=None)
|
_upsert_transcript(session=session, job_id=job.id, text=result.text, error_detail=None)
|
||||||
job.status = JobStatus.TRANSCRIBED
|
job.status = JobStatus.TRANSCRIBED
|
||||||
job.updated_at = datetime.now(timezone.utc)
|
job.updated_at = datetime.now(UTC)
|
||||||
session.add(job)
|
session.add(job)
|
||||||
session.commit()
|
session.commit()
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -77,7 +86,7 @@ def _process_next_queued_job(*, session: Session) -> bool:
|
|||||||
document.id,
|
document.id,
|
||||||
result.provider,
|
result.provider,
|
||||||
)
|
)
|
||||||
except Exception as exc: # noqa: BLE001
|
except Exception as exc:
|
||||||
error = exc if isinstance(exc, AppError) else classify_unexpected_error(exc, operation="worker.process_job")
|
error = exc if isinstance(exc, AppError) else classify_unexpected_error(exc, operation="worker.process_job")
|
||||||
settings = _get_worker_settings()
|
settings = _get_worker_settings()
|
||||||
if _should_retry(job=job, error=error, settings=settings):
|
if _should_retry(job=job, error=error, settings=settings):
|
||||||
@@ -131,7 +140,7 @@ def _requeue_for_retry(*, session: Session, job: Job, error: AppError, settings:
|
|||||||
_upsert_transcript(session=session, job_id=job.id, text=None, error_detail=format_error_detail(error))
|
_upsert_transcript(session=session, job_id=job.id, text=None, error_detail=format_error_detail(error))
|
||||||
job.retry_count += 1
|
job.retry_count += 1
|
||||||
job.status = JobStatus.QUEUED
|
job.status = JobStatus.QUEUED
|
||||||
job.updated_at = datetime.now(timezone.utc)
|
job.updated_at = datetime.now(UTC)
|
||||||
session.add(job)
|
session.add(job)
|
||||||
session.commit()
|
session.commit()
|
||||||
if settings.worker_retry_backoff_seconds > 0:
|
if settings.worker_retry_backoff_seconds > 0:
|
||||||
@@ -141,7 +150,7 @@ def _requeue_for_retry(*, session: Session, job: Job, error: AppError, settings:
|
|||||||
def _finalize_failed_job(*, session: Session, job: Job, error: AppError) -> None:
|
def _finalize_failed_job(*, session: Session, job: Job, error: AppError) -> None:
|
||||||
_upsert_transcript(session=session, job_id=job.id, text=None, error_detail=format_error_detail(error))
|
_upsert_transcript(session=session, job_id=job.id, text=None, error_detail=format_error_detail(error))
|
||||||
job.status = JobStatus.FAILED
|
job.status = JobStatus.FAILED
|
||||||
job.updated_at = datetime.now(timezone.utc)
|
job.updated_at = datetime.now(UTC)
|
||||||
session.add(job)
|
session.add(job)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user