Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions backend/capabilities.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
"""Authoritative conversion-route capabilities shared by API consumers."""

from config import (
MAX_AUDIO_FILE_SIZE,
MAX_FILE_SIZE,
MAX_IMAGE_FILE_SIZE,
MAX_VIDEO_FILE_SIZE,
)

BATCH_MAX_ITEMS = 10
BATCH_MAX_AGGREGATE_SIZE = 200 * 1024 * 1024
BATCH_RETENTION_SECONDS = 7 * 24 * 60 * 60
BATCH_ITEM_LEASE_SECONDS = 15 * 60
BATCH_MAX_ATTEMPTS = 2
Comment thread
JustAGhosT marked this conversation as resolved.

ROUTE_CAPABILITIES = {
"document": {
"extensions": [".tex"],
"targets": ["pdf"],
"max_file_size": MAX_FILE_SIZE,
"batch_enabled": True,
"retention": "retained",
},
"image": {
"extensions": [".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tiff", ".gif"],
"targets": ["webp", "jpeg", "png", "gif", "tiff", "bmp", "svg"],
"max_file_size": MAX_IMAGE_FILE_SIZE,
"batch_enabled": True,
"retention": "retained",
},
"text": {
"extensions": [".md", ".markdown", ".html", ".htm", ".txt", ".docx"],
"targets": ["md", "html", "txt", "docx"],
"max_file_size": MAX_FILE_SIZE,
"batch_enabled": True,
"retention": "retained",
},
"audio": {
"extensions": [".ogg", ".opus", ".mp3", ".wav", ".m4a", ".aac", ".flac"],
"targets": ["mp3", "wav", "ogg", "m4a", "aac", "flac"],
"max_file_size": MAX_AUDIO_FILE_SIZE,
"batch_enabled": True,
"retention": "retained",
},
"transcript": {
"extensions": [".ogg", ".opus", ".mp3", ".wav", ".m4a", ".aac", ".flac"],
"targets": ["text"],
"max_file_size": MAX_AUDIO_FILE_SIZE,
"batch_enabled": False,
"retention": "session_only",
},
"video": {
"extensions": [".mp4", ".mov", ".mkv", ".webm", ".avi", ".m4v"],
"targets": ["mp4", "webm", "mov"],
"max_file_size": MAX_VIDEO_FILE_SIZE,
"batch_enabled": True,
"retention": "retained",
},
}


def public_capabilities() -> dict:
return {
"routes": ROUTE_CAPABILITIES,
"batch": {
"max_items": BATCH_MAX_ITEMS,
"max_aggregate_size": BATCH_MAX_AGGREGATE_SIZE,
"metadata_retention_seconds": BATCH_RETENTION_SECONDS,
"item_lease_seconds": BATCH_ITEM_LEASE_SECONDS,
"max_attempts": BATCH_MAX_ATTEMPTS,
"execution": "client_coordinated",
},
}
16 changes: 16 additions & 0 deletions backend/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,11 @@ async def _create_indexes(cls):

Indexes are created automatically on startup.
"""
# Batch creation relies on this unique index to make concurrent
# Idempotency-Key requests atomic. Do not start without it.
await cls.db.batches.create_index(
[("user_id", 1), ("idempotency_hash", 1)], unique=True
)
try:
# Indexes for conversions collection
conversions = cls.db.conversions
Expand Down Expand Up @@ -149,6 +154,17 @@ async def _create_indexes(cls):
await documents.create_index("timestamp")
await documents.create_index([("uploaded_by", 1), ("timestamp", -1)])

batches = cls.db.batches
await batches.create_index("id", unique=True)
await batches.create_index([("user_id", 1), ("created_at", -1)])
await batches.create_index("expires_at", expireAfterSeconds=0)

batch_items = cls.db.batch_items
await batch_items.create_index("id", unique=True)
await batch_items.create_index([("batch_id", 1), ("position", 1)])
await batch_items.create_index([("user_id", 1), ("batch_id", 1)])
await batch_items.create_index("expires_at", expireAfterSeconds=0)

logger.info("Database indexes created successfully")
except Exception as e:
logger.warning(f"Failed to create indexes (may already exist): {e}")
Expand Down
16 changes: 15 additions & 1 deletion backend/models.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,27 @@
import re
import uuid
from datetime import UTC, datetime
from typing import Dict, List, Literal, Optional
from typing import Any, Dict, List, Literal, Optional

from pydantic import BaseModel, Field, field_validator

from capabilities import BATCH_MAX_ITEMS

# Request/Response Models


class BatchCreateItem(BaseModel):
filename: str = Field(min_length=1, max_length=255)
size: int = Field(gt=0)
sha256: str = Field(pattern=r"^[0-9a-f]{64}$")


class BatchCreateRequest(BaseModel):
route: Literal["document", "image", "text", "audio", "video"]
settings: Dict[str, Any] = Field(default_factory=dict)
items: List[BatchCreateItem] = Field(min_length=2, max_length=BATCH_MAX_ITEMS)


class ConversionRequest(BaseModel):
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
filename: str
Expand Down
90 changes: 41 additions & 49 deletions backend/routers/batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
- Support for different conversion types in single batch
"""

from fastapi import APIRouter, UploadFile, File, HTTPException, BackgroundTasks
from auth import get_current_user
from fastapi import APIRouter, UploadFile, File, HTTPException, BackgroundTasks, Depends
from typing import List
import uuid
import logging
Expand All @@ -29,11 +30,12 @@
async def batch_convert_latex(
files: List[UploadFile] = File(...),
auto_fix: bool = False,
background_tasks: BackgroundTasks = None
background_tasks: BackgroundTasks = None,
user=Depends(get_current_user),
):
"""
Convert multiple LaTeX files to PDF in batch.

TODO: Production implementation:
- Process files asynchronously using job queue
- Return job ID for status polling
Expand All @@ -43,73 +45,69 @@ async def batch_convert_latex(
if len(files) > 50: # Limit batch size
raise HTTPException(
status_code=400,
detail="Batch size limited to 50 files. Please split into smaller batches."
detail="Batch size limited to 50 files. Please split into smaller batches.",
)

batch_id = str(uuid.uuid4())
results = []
errors = []

for file in files:
try:
content = await file.read()
file_size = len(content)

# Validate file
is_valid, error_message = FileValidator.validate_latex_file(
file.filename, file_size, MAX_FILE_SIZE
)
if not is_valid:
errors.append({
"filename": file.filename,
"error": error_message
})
errors.append({"filename": file.filename, "error": error_message})
continue

# Decode content
try:
file_content = content.decode('utf-8')
file_content = content.decode("utf-8")
except UnicodeDecodeError:
try:
file_content = content.decode('latin-1')
file_content = content.decode("latin-1")
except UnicodeDecodeError:
errors.append({
"filename": file.filename,
"error": "Unable to decode file"
})
errors.append(
{"filename": file.filename, "error": "Unable to decode file"}
)
continue

# Process conversion
filename = file.filename.rsplit('.', 1)[0]
result = await LatexService.process_latex_file(file_content, filename, auto_fix)
filename = file.filename.rsplit(".", 1)[0]
result = await LatexService.process_latex_file(
file_content, filename, auto_fix, user_id=user.id
)
results.append(result.model_dump())

except Exception as e:
logger.error(f"Error processing {file.filename}: {e}", exc_info=True)
errors.append({
"filename": file.filename,
"error": str(e)
})

errors.append({"filename": file.filename, "error": str(e)})

return {
"batch_id": batch_id,
"total_files": len(files),
"successful": len(results),
"failed": len(errors),
"results": results,
"errors": errors
"errors": errors,
}


@router.post("/convert-audio")
async def batch_convert_audio(
files: List[UploadFile] = File(...),
target_format: str = 'mp3',
bitrate: str = '192k'
target_format: str = "mp3",
bitrate: str = "192k",
user=Depends(get_current_user),
):
"""
Convert multiple audio files in batch.

TODO: Production implementation:
- Process files asynchronously using job queue
- Return job ID for status polling
Expand All @@ -118,51 +116,45 @@ async def batch_convert_audio(
if len(files) > 20: # Limit batch size for audio (larger files)
raise HTTPException(
status_code=400,
detail="Batch size limited to 20 files. Please split into smaller batches."
detail="Batch size limited to 20 files. Please split into smaller batches.",
)

batch_id = str(uuid.uuid4())
results = []
errors = []

for file in files:
try:
content = await file.read()
file_size = len(content)

# Validate file
is_valid, error_message = FileValidator.validate_audio_file(
file.filename, file_size, MAX_AUDIO_FILE_SIZE
)
if not is_valid:
errors.append({
"filename": file.filename,
"error": error_message
})
errors.append({"filename": file.filename, "error": error_message})
continue

# Process conversion
result = await AudioService.process_audio_file(
content,
file.filename,
target_format=target_format,
bitrate=bitrate
bitrate=bitrate,
user_id=user.id,
)
results.append(result.model_dump())

except Exception as e:
logger.error(f"Error processing {file.filename}: {e}", exc_info=True)
errors.append({
"filename": file.filename,
"error": str(e)
})

errors.append({"filename": file.filename, "error": str(e)})

return {
"batch_id": batch_id,
"total_files": len(files),
"successful": len(results),
"failed": len(errors),
"results": results,
"errors": errors
"errors": errors,
}

Loading