generated from john/python-template
Merge remote-tracking branch 'origin/doc_update' into session-engine
This commit is contained in:
John Lancaster
@@ -1,122 +0,0 @@
|
||||
# Database Schema (V2 Architecture)
|
||||
|
||||
This document describes the PostgreSQL relational schema for the transcription platform. It incorporates multi-image batch orchestration via `asyncio`, page-level execution tracking, many-to-many author/recipient attribution, and JSONB document storage for AI vision outputs.
|
||||
|
||||
All primary and foreign keys are PostgreSQL native UUIDs (`gen_random_uuid()`).
|
||||
|
||||
## Entity Relationship Diagram
|
||||
|
||||
```mermaid
|
||||
erDiagram
|
||||
PERSON {
|
||||
UUID id PK
|
||||
TEXT full_name
|
||||
TEXT display_name
|
||||
TEXT maiden_name
|
||||
DATE birth_date
|
||||
TEXT birth_date_raw
|
||||
TEXT birth_place
|
||||
DATE death_date
|
||||
TEXT death_date_raw
|
||||
TEXT death_place
|
||||
TEXT biography
|
||||
TEXT portrait_path
|
||||
JSONB metadata
|
||||
TIMESTAMPTZ created_at
|
||||
TIMESTAMPTZ updated_at
|
||||
}
|
||||
|
||||
DOCUMENT {
|
||||
UUID id PK
|
||||
TEXT name
|
||||
TEXT document_type
|
||||
DATE document_date
|
||||
TEXT document_date_raw
|
||||
TEXT location_created
|
||||
TEXT notes
|
||||
TEXT archive_identifier
|
||||
TIMESTAMPTZ created_at
|
||||
TIMESTAMPTZ updated_at
|
||||
}
|
||||
|
||||
DOCUMENT_PERSON {
|
||||
UUID id PK
|
||||
UUID document_id FK
|
||||
UUID person_id FK
|
||||
VARCHAR role "author | recipient"
|
||||
TIMESTAMPTZ created_at
|
||||
}
|
||||
|
||||
JOB {
|
||||
UUID id PK
|
||||
UUID document_id FK
|
||||
VARCHAR status "queued | processing | completed | partial_success | failed"
|
||||
INTEGER retry_count
|
||||
TEXT provider
|
||||
TEXT model
|
||||
TEXT prompt_name
|
||||
TIMESTAMPTZ date_created
|
||||
TIMESTAMPTZ date_updated
|
||||
}
|
||||
|
||||
SOURCE {
|
||||
UUID id PK
|
||||
UUID document_id FK
|
||||
INTEGER page_number
|
||||
TEXT upload_name
|
||||
TEXT filename
|
||||
TEXT file_path
|
||||
TEXT raw_transcription
|
||||
TEXT revised_text
|
||||
TIMESTAMPTZ date_uploaded
|
||||
TIMESTAMPTZ date_revised
|
||||
}
|
||||
|
||||
JOB_SOURCE {
|
||||
UUID id PK
|
||||
UUID job_id FK
|
||||
UUID source_id FK
|
||||
VARCHAR status "pending | transcribed | failed"
|
||||
TEXT raw_transcription
|
||||
JSONB ai_metadata
|
||||
JSONB raw_api_response
|
||||
TEXT error_detail
|
||||
TIMESTAMPTZ executed_at
|
||||
}
|
||||
|
||||
DOCUMENT ||--o{ DOCUMENT_PERSON : "has_people"
|
||||
PERSON ||--o{ DOCUMENT_PERSON : "participates_in"
|
||||
DOCUMENT ||--o{ JOB : "has_jobs"
|
||||
DOCUMENT ||--o{ SOURCE : "contains_pages"
|
||||
JOB ||--o{ JOB_SOURCE : "executes"
|
||||
SOURCE ||--o{ JOB_SOURCE : "processed_in"
|
||||
```
|
||||
|
||||
## Domain Invariants & Rules
|
||||
|
||||
### Page-Level Execution & AI Outputs
|
||||
|
||||
* Execution Granularity: Every single image execution by an AI model produces a dedicated record in job_source.
|
||||
* Point-in-Time Auditability: job_source.raw_api_response stores the unparsed REST response envelope for that specific image page call. job_source.ai_metadata stores spatial bounding boxes, token usage, and layout details for that specific image page call.
|
||||
* Active Output Caching: Upon successful completion of an image call, source.raw_transcription is updated with the latest output string from job_source.raw_transcription for fast UI rendering.
|
||||
|
||||
### Page Ordering & Revisions
|
||||
|
||||
* Sequential Integrity: source.page_number dictates page ordering within a document. Reads assembling full documents must query ORDER BY source.document_id, source.page_number ASC.
|
||||
* Inlined Human Corrections: User edits occur at the page level inside source.revised_text. source.raw_transcription remains immutable. If source.revised_text is non-null, application frontends must render source.revised_text.
|
||||
|
||||
### Async Job Lifecycle & Failure Isolation
|
||||
|
||||
* Batch Orchestrator: A job represents an overarching execution run across one or more source images belonging to a document.
|
||||
* Isolated Failures: API requests run concurrently (e.g., using asyncio). A failure on page 3 does not invalidate successful transcriptions on page 1 or 2.
|
||||
* Job States:
|
||||
- queued: Created, awaiting worker execution.
|
||||
- processing: Concurrent HTTP tasks actively running.
|
||||
- completed: 100% of linked job_source tasks succeeded (transcribed).
|
||||
- partial_success: At least one job_source succeeded and at least one failed.
|
||||
- failed: All linked job_source tasks failed or a job-level runtime error occurred.
|
||||
|
||||
### Attribution & Person Roles
|
||||
|
||||
* Multi-Person Roles: Documents support zero, one, or many authors and recipients linked via document_person.
|
||||
* Role Uniqueness: (document_id, person_id, role) must be unique to prevent duplicate role tagging.
|
||||
@@ -1,102 +0,0 @@
|
||||
## PostgreSQL DDL Specification
|
||||
|
||||
```sql
|
||||
-- Enable pgcrypto for UUID generation if on PostgreSQL < 13
|
||||
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
|
||||
|
||||
-- 1. PERSON TABLE
|
||||
CREATE TABLE person (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
full_name TEXT NOT NULL,
|
||||
display_name TEXT,
|
||||
maiden_name TEXT,
|
||||
birth_date DATE,
|
||||
birth_date_raw TEXT,
|
||||
birth_place TEXT,
|
||||
death_date DATE,
|
||||
death_date_raw TEXT,
|
||||
death_place TEXT,
|
||||
biography TEXT,
|
||||
portrait_path TEXT,
|
||||
metadata JSONB DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- 2. DOCUMENT TABLE
|
||||
CREATE TABLE document (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name TEXT NOT NULL,
|
||||
document_type TEXT,
|
||||
document_date DATE,
|
||||
document_date_raw TEXT,
|
||||
location_created TEXT,
|
||||
notes TEXT,
|
||||
archive_identifier TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- 3. DOCUMENT_PERSON (Junction Table for Multi-Author / Multi-Recipient)
|
||||
CREATE TABLE document_person (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
document_id UUID NOT NULL REFERENCES document(id) ON DELETE CASCADE,
|
||||
person_id UUID NOT NULL REFERENCES person(id) ON DELETE CASCADE,
|
||||
role VARCHAR(20) NOT NULL, -- 'author' or 'recipient'
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT unique_document_person_role UNIQUE (document_id, person_id, role)
|
||||
);
|
||||
|
||||
-- 4. JOB TABLE (Batch-level orchestrator)
|
||||
CREATE TABLE job (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
document_id UUID NOT NULL REFERENCES document(id) ON DELETE CASCADE,
|
||||
status VARCHAR(50) NOT NULL DEFAULT 'queued', -- 'queued', 'processing', 'completed', 'partial_success', 'failed'
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
provider TEXT NOT NULL, -- e.g., 'openai', 'anthropic'
|
||||
model TEXT NOT NULL, -- e.g., 'gpt-4o', 'claude-3-5-sonnet'
|
||||
prompt_name TEXT,
|
||||
date_created TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
date_updated TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- 5. SOURCE TABLE (Physical image files & active state)
|
||||
CREATE TABLE source (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
document_id UUID NOT NULL REFERENCES document(id) ON DELETE CASCADE,
|
||||
page_number INTEGER NOT NULL DEFAULT 1,
|
||||
upload_name TEXT NOT NULL,
|
||||
filename TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
raw_transcription TEXT, -- Cached active AI text output
|
||||
revised_text TEXT, -- Active human edited text
|
||||
date_uploaded TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
date_revised TIMESTAMPTZ
|
||||
);
|
||||
|
||||
-- 6. JOB_SOURCE (Junction Table: Per-Image Execution Record)
|
||||
CREATE TABLE job_source (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
job_id UUID NOT NULL REFERENCES job(id) ON DELETE CASCADE,
|
||||
source_id UUID NOT NULL REFERENCES source(id) ON DELETE CASCADE,
|
||||
status VARCHAR(50) NOT NULL DEFAULT 'pending', -- 'pending', 'transcribed', 'failed'
|
||||
raw_transcription TEXT, -- Point-in-time raw AI text output
|
||||
ai_metadata JSONB, -- Page-level bounding boxes, tokens, confidence
|
||||
raw_api_response JSONB, -- Complete REST response envelope
|
||||
error_detail TEXT,
|
||||
executed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT unique_job_source UNIQUE (job_id, source_id)
|
||||
);
|
||||
|
||||
-- INDEXES FOR FAST LOOKUPS & QUERY PERFORMANCE
|
||||
CREATE INDEX idx_person_full_name ON person(full_name);
|
||||
CREATE INDEX idx_document_date ON document(document_date);
|
||||
CREATE INDEX idx_document_person_doc ON document_person(document_id);
|
||||
CREATE INDEX idx_document_person_per ON document_person(person_id);
|
||||
CREATE INDEX idx_source_document ON source(document_id);
|
||||
CREATE INDEX idx_source_page_order ON source(document_id, page_number);
|
||||
CREATE INDEX idx_job_document ON job(document_id);
|
||||
CREATE INDEX idx_job_source_job ON job_source(job_id);
|
||||
CREATE INDEX idx_job_source_source ON job_source(source_id);
|
||||
CREATE INDEX idx_job_source_ai_metadata ON job_source USING GIN (ai_metadata);
|
||||
```
|
||||
Reference in New Issue
Block a user