diff --git a/.gitignore b/.gitignore index 63b270e..3b6ffbf 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,23 @@ # Local agent/tool worktrees and leftover nested checkout (not part of the package) .codex/ .serena/ -/xtox/ +# Ignore the historical nested checkout, but retain the small Python +# compatibility namespace used by the published xtotext distribution. +/xtox/* +!/xtox/__init__.py +!/xtox/core/ +/xtox/core/* +!/xtox/core/__init__.py +!/xtox/workflows/ +/xtox/workflows/* +!/xtox/workflows/__init__.py +!/xtox/utils/ +/xtox/utils/* +!/xtox/utils/__init__.py +!/xtox/cli/ +/xtox/cli/* +!/xtox/cli/__init__.py +!/xtox/cli/main.py # IDE and editors .idea/ diff --git a/CLAUDE.md b/CLAUDE.md index 586d4b1..c4fb821 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,15 +4,18 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project -**xtotext (xtox)** — AI-ready document conversion system. Transforms any document (PDF, DOCX, LaTeX, Markdown, etc.) into AI-optimized text formats for LLM consumption. Includes document storage and permission-based access. +**Mill** — alpha document and media conversion workspace. The public npm +package is `@celladore/mill`; the Python distribution `xtotext` and import +namespace `xtox` remain compatibility APIs. ## Tech Stack -- **Language**: Python -- **CLI**: Click-based CLI tool -- **API**: FastAPI REST API (`api/`) -- **Serverless**: Azure Functions (`azure-functions/`) -- **Storage**: Azure Data Lake Storage Gen2 +- **Languages**: Python and JavaScript +- **CLI**: Node launcher (`mill`) plus the compatible Python `xtotext` executable +- **API**: FastAPI REST API (`backend/`) +- **Frontend**: React (`frontend/`) +- **Legacy serverless lane**: Azure Functions (`azure-functions/`) +- **Storage**: Azure Blob Storage and Cosmos DB's MongoDB API - **Build**: Makefile ## Key Commands @@ -31,10 +34,11 @@ make md-to-pdf # Convert markdown to PDF ## Architecture -- `cli/` — Click CLI for document conversion -- `api/` — FastAPI REST API -- `azure-functions/` — Serverless conversion endpoints -- `backend/` — Core conversion logic +- `mill-cli/` and `bin/mill.js` — active Node CLI and npm package +- `backend/` — active FastAPI REST API and conversion services +- `core/` — local deterministic Python conversion engine +- `cli/` and `api/` — legacy Python compatibility entry points +- `azure-functions/` — legacy serverless lane; not the canonical deployed API ## AgentKit Forge diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 0afd24e..02843f0 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -1,6 +1,7 @@ -# xtotext Implementation Details +# Mill Python Compatibility Implementation -This document describes the implementation of the xtotext package, focusing on the main Markdown -> LaTeX -> PDF pipeline. +This document describes Mill's compatible `xtotext` Python distribution and +`xtox` import namespace, focusing on the Markdown -> LaTeX -> PDF pipeline. ## Core Components @@ -98,4 +99,4 @@ Potential future enhancements include: - Advanced LaTeX customization - PDF metadata management - Custom LaTeX templates -- Integration with AI services for content enhancement \ No newline at end of file +- Integration with AI services for content enhancement diff --git a/README.md b/README.md index 1ea9fde..e0e4d57 100644 --- a/README.md +++ b/README.md @@ -1,343 +1,86 @@ -# xtotext - AI-Ready Document Conversion System +# Mill -A powerful document conversion system that transforms any document into AI-friendly text formats, designed for seamless integration with Large Language Models and AI workflows, with document storage and management capabilities. +Mill is Celladore's alpha conversion workspace for documents, images, audio, and +video. Deterministic local tools handle supported document and image formats; +the deployed API provides authenticated media conversion and transcription. -## Features +- Web: +- API documentation: +- Repository: -- **Universal Document Conversion**: Transform documents (PDF, DOCX, LaTeX, Markdown, etc.) into AI-optimized text formats -- **AI-Ready Output**: Structured text output specifically formatted for LLM consumption and analysis -- **Document Storage**: Securely store and manage documents in Azure Data Lake Storage Gen2 -- **Permission-Based Access**: Fine-grained user permissions for document access and conversion -- **LaTeX to PDF Conversion**: Convert LaTeX documents to PDF format with error handling -- **Repository Integration**: Batch convert entire project documentation into single AI-friendly files -- **Smart Context Preservation**: Maintain document structure and relationships for better AI understanding -- **API-First Design**: RESTful API designed for AI tool integration and automation workflows -- **Proper Package Structure**: Follows Python packaging best practices -- **Type Hints**: Full type annotation support -- **CLI Interface**: Easy-to-use command-line tool -- **Modular Design**: Separate core, utils, and workflow modules +HTTP availability, dependency readiness, deployment, and an authenticated user +conversion are separate evidence. */api/health* is process liveness; +*/api/ready* checks the configured database dependency. */api/status* is a +persisted status-check collection and is not a health endpoint. -## Project Structure +## Mill CLI alpha -``` -xtox/ -├── xtox/ # Main package -│ ├── __init__.py # Package initialization -│ ├── core/ # Core conversion functionality -│ │ ├── __init__.py -│ │ ├── document_converter.py # Main converter class -│ │ ├── markdown_to_latex.py # Markdown to LaTeX conversion -│ │ └── latex_to_pdf.py # LaTeX to PDF conversion -│ ├── utils/ # Utility functions -│ │ ├── __init__.py -│ │ └── image_handler.py -│ ├── workflows/ # High-level workflows -│ │ ├── __init__.py -│ │ └── md_to_pdf.py -│ ├── cli/ # Command-line interface -│ │ ├── __init__.py -│ │ └── main.py -│ ├── api/ # API routes -│ ├── backend/ # Backend services -│ ├── azure-functions/ # Azure Functions -│ └── frontend/ # React frontend -├── tests/ # Test files -├── infra/ # Infrastructure code -├── setup.py # Package setup -├── pyproject.toml # Modern Python project config -├── requirements.txt # Dependencies -├── Makefile # Development tasks -└── README.md # This file -``` +The temporary npm identity is *@celladore/mill*. The bare *mill* package belongs +to somebody else, so *npx mill* is not supported and must not be documented as a +Mill command. -## Architecture +Registry publication is pending npm authentication and Celladore scope-control +verification. Until that gate passes, the commands below describe the approved +alpha surface but will not resolve from the public registry. -This project consists of: + npx @celladore/mill init + npx @celladore/mill inspect "voice notes/input.ogg" + npx @celladore/mill convert "voice notes/input.ogg" --format mp3 -1. **Backend**: Azure Functions for serverless document processing, storage, retrieval, and conversion -2. **Frontend**: React application with AI-focused conversion interfaces -3. **Storage**: Azure Data Lake Storage Gen2 for document and converted text storage -4. **Database**: MongoDB for metadata, conversion history, and AI context mapping -5. **AI Conversion Engine**: Specialized pipeline for creating LLM-optimized text outputs -6. **Repository Processor**: Batch conversion system for entire codebases and documentation sets + npm install --global @celladore/mill + mill doctor -## Prerequisites +Audio conversion requires an operator-provided *MYSTIRA_ACCESS_TOKEN*. Web login +does not currently hand a token to the CLI through a supported user flow. The +CLI therefore fails closed when this external token boundary is absent; it does +not fabricate a native OAuth client, scrape the browser session, or store a +credential. -- Azure subscription (for cloud deployment) -- Azure CLI or PowerShell Az module -- Node.js 14+ -- MongoDB -- Python 3.9+ -- AI conversion dependencies (transformers, tiktoken, etc.) +Commands: -## Installation +- *mill init* writes a non-secret *.millrc.json* in the current directory. +- *mill login* reports whether the external token boundary is satisfied and + points to the web login. It is not native CLI OIDC acceptance. +- *mill inspect INPUT* reports the selected local/API boundary before work. +- *mill convert INPUT* delegates documents/images to *xtotext* and supported + audio to the authenticated Mill API, then retrieves the output. +- *mill doctor* checks Node, configuration, identity input, and the compatible + local Python executable. *--api* checks */docs* availability and explicitly + does not claim readiness. -### Development Installation +The package requires Node 20 or newer. Paths are passed as argument arrays, not +shell strings, so spaces are preserved and no interactive terminal is required. -```bash -# Clone the repository -git clone -cd xtox +## Python compatibility -# Install in development mode with all dependencies -make setup +The Python distribution remains *xtotext*, its import remains *xtox*, and its +legacy executable remains *xtotext*. Those are compatibility APIs, not current +product branding, and this alpha does not perform a breaking namespace migration. -# Or manually: -pip install -e ".[dev,azure,api]" -``` + python -m pip install -e ".[dev,azure,api]" + xtotext --help -### Production Installation +Example: -```bash -pip install -e . -``` + from xtox.core import DocumentConverter -## Usage + converter = DocumentConverter(output_dir="./output") + result = converter.markdown_to_pdf("document.md", refinement_level=2) -### Command Line Interface - -```bash -# Convert Markdown to PDF -xtotext input.md -o output_dir -r 2 - -# Convert LaTeX to PDF -xtotext document.tex -o output_dir - -# Show help -xtotext --help -``` - -### Python API - -```python -from xtox.core import DocumentConverter - -# Initialize converter -converter = DocumentConverter(output_dir="./output") - -# Convert Markdown to PDF -result = converter.markdown_to_pdf( - "document.md", - refinement_level=2 -) - -# Convert LaTeX to PDF -pdf_path = converter.latex_to_pdf("document.tex") -``` - -### Using Workflows - -```python -from xtox.workflows import process_markdown_to_pdf - -result = process_markdown_to_pdf( - "document.md", - output_dir="./output", - refinement_level=1 -) -``` - -## API Endpoints - -### AI-Focused Conversion -- `POST /api/ai/convert` - Convert document for AI consumption -- `POST /api/ai/repository` - Process entire repository for AI -- `GET /api/ai/context/{id}` - Get document with full context -- `POST /api/ai/optimize` - Optimize existing text for specific AI models - -### Document Management -- `POST /api/documents/upload` - Upload a document -- `GET /api/documents` - List documents available to the user -- `GET /api/documents/{id}` - Get document metadata -- `GET /api/documents/{id}/download` - Download document -- `POST /api/documents/{id}/permissions` - Update document permissions -- `DELETE /api/documents/{id}` - Delete a document - -### Document Conversion -- `POST /api/convert` - Convert LaTeX to PDF -- `GET /api/conversion/{id}` - Get conversion result -- `GET /api/download/{id}` - Download converted PDF -- `GET /api/documents/{id}/ai-text` - Get AI-optimized text output -- `POST /api/batch/repository` - Repository-wide batch conversion - -## Usage Examples - -### Single Document for AI -```python -import requests - -# Convert document for AI consumption -with open('technical_doc.pdf', 'rb') as f: - response = requests.post( - 'https://yourfunctionapp.azurewebsites.net/api/ai/convert', - files={'file': f}, - json={ - 'target_model': 'gpt-4', - 'preserve_structure': True, - 'include_metadata': True - }, - headers={'Authorization': 'Bearer your_token'} - ) - ai_ready_text = response.json()['ai_text'] -``` - -### Repository-Wide AI Conversion -```python -# Convert entire project documentation for AI -response = requests.post( - 'https://yourfunctionapp.azurewebsites.net/api/ai/repository', - json={ - 'repository_path': '/path/to/project', - 'include_code': True, - 'include_docs': True, - 'target_model': 'claude-3', - 'output_format': 'contextual' - }, - headers={'Authorization': 'Bearer your_token'} -) -# Get single AI-friendly file representing entire project -project_context = response.json()['consolidated_text'] -``` +The local Python engine currently supports Markdown, HTML, LaTeX, and common +image conversions. Audio is deliberately delegated to the authenticated API; +the Node package does not duplicate FFmpeg or backend conversion logic. ## Development -### Setup Development Environment - -```bash -make dev-install -``` - -### Run Tests - -```bash -make test -make test-cov # with coverage -``` - -### Code Quality - -```bash -make lint # Run linting -make format # Format code -``` - -### Build Package - -```bash -make build -``` - -### Local Setup with Azure Functions Core Tools -1. **Install Azure Functions Core Tools**: - ```bash - npm install -g azure-functions-core-tools@4 - ``` - -2. **Run Functions locally**: - ```bash - cd azure-functions - func start - ``` - -3. **AI Model Integration**: - ```bash - # Install AI optimization tools - pip install tiktoken transformers sentence-transformers - # Configure model-specific tokenizers - python scripts/setup_ai_models.py - ``` - -## AI Integration Features - -### LLM Optimization -```yaml -# config/ai_optimization.yaml -ai_optimization: - token_limits: - gpt-4: 8192 - claude-3: 100000 - gpt-3.5: 4096 - formatting: - preserve_code_blocks: true - add_context_headers: true - include_file_paths: true - maintain_hierarchy: true - chunking: - strategy: "semantic" # or "fixed", "adaptive" - overlap_tokens: 200 - respect_boundaries: true -``` - -### Repository Processing -```yaml -# config/repository.yaml -repository_processing: - include_patterns: - - "*.md" - - "*.rst" - - "*.txt" - - "README*" - - "docs/**" - exclude_patterns: - - "node_modules/**" - - ".git/**" - - "*.log" - - "build/**" - ai_enhancements: - add_file_context: true - preserve_directory_structure: true - include_git_info: false -``` - -## Recent Improvements - -### Security Enhancements -- ✅ JWT secret key management via environment variables and Azure Key Vault -- ✅ Removed mock authentication bypass -- ✅ CORS origin restrictions -- ✅ File path sanitization to prevent path traversal attacks -- ✅ Input validation for all endpoints - -### Performance Optimizations -- ✅ Database connection pooling -- ✅ Rate limiting middleware -- ✅ Database indexes for faster queries -- ✅ Centralized file validation - -### UI/UX Improvements -- ✅ Accessibility components (ARIA labels, keyboard navigation) -- ✅ Design token integration -- ✅ Responsive design support -- ✅ Error handling improvements - -## Contributing - -1. Fork the repository -2. Create a feature branch focused on AI optimization -3. Add tests for AI-specific functionality -4. Ensure compatibility with major LLM providers -5. Submit a pull request - -## Documentation - -- [API Documentation](docs/API.md) - Complete API reference -- [Architecture](docs/ARCHITECTURE.md) - System architecture and design -- [Deployment Guide](docs/DEPLOYMENT.md) - Deployment instructions -- [Contributing](docs/CONTRIBUTING.md) - Contribution guidelines -- [Testing Guide](docs/TESTING.md) - Testing documentation -- [Design System](docs/DESIGN_SYSTEM.md) - Design tokens and components - -## Environment Variables - -See [.env.example](.env.example) for all configuration options. - -**Required for Production:** -- `JWT_SECRET_KEY` - Minimum 32 characters -- `MONGO_URL` - MongoDB connection string -- `ALLOWED_ORIGINS` - Comma-separated frontend URLs -- `ENVIRONMENT` - Set to `production` -- `ALLOW_MOCK_AUTH` - Set to `false` - -## License - -MIT License - see LICENSE file for details \ No newline at end of file + python -m pytest + pnpm --dir frontend install --frozen-lockfile + pnpm --dir frontend test + pnpm --dir frontend build + npm test + npm pack --dry-run + +Production publication, deployment, DNS, secrets, and OIDC registration changes +are separately authorized operations. See +[the Mill identity inventory](docs/mill-identity-inventory.md) before renaming +any remaining *xtox* or *xtotext* identifier. diff --git a/WARP.md b/WARP.md index 227f498..3df73e6 100644 --- a/WARP.md +++ b/WARP.md @@ -4,7 +4,10 @@ This file provides guidance to WARP (warp.dev) when working with code in this re ## Project Overview -xtotext is an AI-Ready Document Conversion System designed to transform documents (PDF, DOCX, LaTeX, Markdown, etc.) into AI-optimized text formats for LLM consumption. The project features document storage, permission-based access, and a multi-tier architecture with Python backend, React frontend, and Azure Functions for serverless processing. +Mill is an alpha document and media conversion workspace with a FastAPI backend +and React frontend. The Python distribution `xtotext`, import namespace `xtox`, +and older path examples in this guide are compatibility surfaces retained for +existing consumers. ## Common Commands diff --git a/__init__.py b/__init__.py index 99974fc..d15721c 100644 --- a/__init__.py +++ b/__init__.py @@ -1,11 +1,11 @@ """ -xtotext - AI-Ready Document Conversion System +Mill - document and media conversion -A powerful document conversion system that transforms documents into AI-friendly formats. +The xtox import and xtotext distribution names remain compatibility APIs. """ __version__ = "1.0.0" -__author__ = "xtotext Team" +__author__ = "Celladore" if __package__: from .core import DocumentConverter, ImageConverter, MultiDocumentProcessor diff --git a/backend/auth.py b/backend/auth.py index 91fa4f3..ea00e75 100644 --- a/backend/auth.py +++ b/backend/auth.py @@ -3,7 +3,7 @@ Validates the caller's Bearer access token against Mystira Identity's JWKS (RS256) — see mystira_auth.py for the resource-server validation logic and -why xtox never holds a client secret. There is no mock/bypass path: if +why Mill never holds a client secret. There is no mock/bypass path: if MYSTIRA_OIDC_ISSUER/MYSTIRA_OIDC_AUDIENCE aren't configured, every protected route fails closed with 503 rather than admitting requests. ALLOW_MOCK_AUTH no longer exists anywhere in this codebase — for tests, override this @@ -49,7 +49,7 @@ async def get_current_user( async def get_transcription_user( authorization: str = Header(default=None), ) -> MystiraPrincipal: - """Authenticate direct XtOX users or explicitly scoped delegated callers. + """Authenticate direct Mill users or explicitly scoped delegated callers. Delegation is limited to the transcription endpoint. Merely adding an audience is insufficient: delegated tokens must also carry the configured @@ -85,7 +85,7 @@ async def get_transcription_user( async def get_render_user( authorization: str = Header(default=None), ) -> MystiraPrincipal: - """Authenticate direct XtOX users or scoped CoilTrace render callers.""" + """Authenticate direct Mill users or scoped CoilTrace render callers.""" delegated = [ audience.strip() for audience in os.environ.get("MYSTIRA_OIDC_RENDER_AUDIENCES", "").split(",") diff --git a/backend/config.py b/backend/config.py index 06ddc8d..672a119 100644 --- a/backend/config.py +++ b/backend/config.py @@ -1,5 +1,5 @@ """ -Configuration module for XToX Converter backend. +Configuration module for the Mill conversion backend. """ import os from pathlib import Path diff --git a/backend/mystira_auth.py b/backend/mystira_auth.py index 437cb67..a80bed7 100644 --- a/backend/mystira_auth.py +++ b/backend/mystira_auth.py @@ -1,9 +1,9 @@ """ Mystira Identity OIDC resource-server token validation. -xtox's API validates Bearer access tokens already issued by Mystira Identity's +Mill's API validates Bearer access tokens already issued by Mystira Identity's OpenIddict authorization server. It does NOT perform the interactive -authorization_code+PKCE login handshake itself; the XtOX frontend does that as +authorization_code+PKCE login handshake itself; the Mill frontend does that as the `celladore-xtox` Public + PKCE client. No client secret is held by the browser or API, unlike house-of-veritas's confidential-RP setup. @@ -241,10 +241,10 @@ def _enforce_delegated_scope( ) -> None: """Require an API-specific scope when a token targets another client. - A direct XtOX-only token keeps the normal first-party path. Any token that + A direct Mill-only token keeps the normal first-party path. Any token that includes a configured delegated client audience is accepted solely when it carries the dedicated scope, including mixed-audience tokens. This prevents - adding a ConvoLens audience from silently granting access to XtOX. + adding a ConvoLens audience from silently granting access to Mill. """ token_audiences = set(_claim_values(payload, "aud")) delegated_match = token_audiences.intersection(delegated_audiences) @@ -253,11 +253,11 @@ def _enforce_delegated_scope( _claim_values(payload, "scope") + _claim_values(payload, "scp") ) if not required_scope or required_scope not in token_scopes: - raise ForbiddenError("Token lacks the required XtOX transcription scope") + raise ForbiddenError("Token lacks the required Mill transcription scope") return if not token_audiences.intersection(direct_audiences): - raise UnauthorizedError("Token audience is not authorized for XtOX") + raise UnauthorizedError("Token audience is not authorized for Mill") def validate_bearer_token( diff --git a/backend/routers/status.py b/backend/routers/status.py index 8ef1ef2..e0651fa 100644 --- a/backend/routers/status.py +++ b/backend/routers/status.py @@ -1,14 +1,38 @@ -from fastapi import APIRouter +import logging from typing import List +from fastapi import APIRouter, HTTPException + from models import StatusCheck, StatusCheckCreate from database import Database router = APIRouter(prefix="/api") +logger = logging.getLogger(__name__) + @router.get("/") async def root(): - return {"message": "XToPDF API - Convert LaTeX to PDF"} + return {"message": "Mill API - document and media conversion"} + + +@router.get("/health") +async def health(): + """Process liveness only; this does not assert dependency readiness.""" + return {"status": "ok", "service": "mill-api", "check": "liveness"} + + +@router.get("/ready") +async def ready(): + """Report readiness only after the configured MongoDB dependency responds.""" + if Database.client is None: + raise HTTPException(status_code=503, detail="Database is not connected") + try: + await Database.client.admin.command("ping") + except Exception as error: + logger.warning("Mill readiness check failed: %s", error) + raise HTTPException(status_code=503, detail="Database is not ready") from error + return {"status": "ready", "service": "mill-api", "check": "readiness"} + @router.post("/status", response_model=StatusCheck) async def create_status_check(input: StatusCheckCreate): diff --git a/backend/routers/webhooks.py b/backend/routers/webhooks.py index e4fc8ec..e537f36 100644 --- a/backend/routers/webhooks.py +++ b/backend/routers/webhooks.py @@ -40,6 +40,7 @@ async def send_webhook_notification(webhook_url: str, payload: dict, secret: Opt try: headers = { "Content-Type": "application/json", + # Compatibility contract: consumers may route/filter this value. "User-Agent": "XToX-Converter/1.0" } diff --git a/backend/server.py b/backend/server.py index 59e3a6d..a3b0e19 100644 --- a/backend/server.py +++ b/backend/server.py @@ -1,6 +1,4 @@ -""" -FastAPI server for XToX Converter backend. -""" +"""FastAPI server for the Mill conversion API.""" import asyncio import contextlib import logging @@ -19,7 +17,10 @@ logger = logging.getLogger(__name__) # Create the main app -app = FastAPI(title="XToPDF API", description="Convert LaTeX to PDF API") +app = FastAPI( + title="Mill API", + description="Authenticated document and media conversion API", +) # Add CORS middleware # TODO: Production hardening - Configure specific allowed origins diff --git a/backend/tests/test_health_routes.py b/backend/tests/test_health_routes.py new file mode 100644 index 0000000..10b3a53 --- /dev/null +++ b/backend/tests/test_health_routes.py @@ -0,0 +1,47 @@ +import asyncio + +import pytest +from fastapi import HTTPException + +from routers import status + + +class _Admin: + def __init__(self, error=None): + self.error = error + + async def command(self, command): + assert command == "ping" + if self.error: + raise self.error + return {"ok": 1} + + +class _Client: + def __init__(self, error=None): + self.admin = _Admin(error) + + +def test_health_is_explicitly_liveness_only(): + result = asyncio.run(status.health()) + assert result == {"status": "ok", "service": "mill-api", "check": "liveness"} + + +def test_ready_pings_database(monkeypatch): + monkeypatch.setattr(status.Database, "client", _Client()) + result = asyncio.run(status.ready()) + assert result == {"status": "ready", "service": "mill-api", "check": "readiness"} + + +def test_ready_fails_closed_without_database(monkeypatch): + monkeypatch.setattr(status.Database, "client", None) + with pytest.raises(HTTPException) as error: + asyncio.run(status.ready()) + assert error.value.status_code == 503 + + +def test_ready_fails_closed_when_ping_fails(monkeypatch): + monkeypatch.setattr(status.Database, "client", _Client(RuntimeError("offline"))) + with pytest.raises(HTTPException) as error: + asyncio.run(status.ready()) + assert error.value.status_code == 503 diff --git a/bin/mill.js b/bin/mill.js new file mode 100644 index 0000000..7bfed14 --- /dev/null +++ b/bin/mill.js @@ -0,0 +1,8 @@ +#!/usr/bin/env node + +import { main } from '../mill-cli/main.mjs'; + +main(process.argv.slice(2)).catch(error => { + console.error(`mill: ${error.message}`); + process.exitCode = error.exitCode || 1; +}); diff --git a/cli/main.py b/cli/main.py index 1d4a52c..8da8323 100644 --- a/cli/main.py +++ b/cli/main.py @@ -27,7 +27,7 @@ def main(): Main entry point for the CLI. """ parser = argparse.ArgumentParser( - description="xtotext - AI-Ready Document Conversion System" + description="Mill local conversion compatibility CLI (xtotext executable)" ) parser.add_argument( "input_files", @@ -238,4 +238,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/core/latex_to_pdf.py b/core/latex_to_pdf.py index d3e7df5..fd37401 100644 --- a/core/latex_to_pdf.py +++ b/core/latex_to_pdf.py @@ -54,7 +54,7 @@ def fix_latex_structure(tex_path, backup=True): # Add missing structure if not has_documentclass: - content = "\\documentclass{article}\\n\\n" + content + content = "\\documentclass{article}\n\n" + content if not has_begin_document: # Add begin{document} after the preamble (after last \usepackage or \documentclass) @@ -63,13 +63,13 @@ def fix_latex_structure(tex_path, backup=True): ) if preamble_end > -1: content = content[:preamble_end] + content[preamble_end:].replace( - "\\n", "\\n\\n\\begin{document}\\n", 1 + "\n", "\n\n\\begin{document}\n", 1 ) else: - content = "\\begin{document}\\n" + content + content = "\\begin{document}\n" + content if not has_end_document: - content += "\\n\\end{document}" + content += "\n\\end{document}" # Write the fixed content with open(tex_path, "w", encoding="utf-8") as file: @@ -86,6 +86,7 @@ def latex_to_pdf(tex_path, auto_fix=False): Convert LaTeX file to PDF using pdflatex. If auto_fix is True, attempts to fix common structure issues. """ + tex_path = os.path.abspath(tex_path) if not os.path.isfile(tex_path): print(f"File not found: {tex_path}") return False @@ -111,10 +112,13 @@ def latex_to_pdf(tex_path, auto_fix=False): return False # Run pdflatex twice for references and cross-references + tex_dir = os.path.dirname(tex_path) + tex_filename = os.path.basename(tex_path) for i in range(2): print(f"Running pdflatex (pass {i+1}/2)...") result = subprocess.run( - ["pdflatex", "-interaction=nonstopmode", tex_path], + ["pdflatex", "-interaction=nonstopmode", tex_filename], + cwd=tex_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8', @@ -152,4 +156,4 @@ def latex_to_pdf(tex_path, auto_fix=False): else: tex_file = sys.argv[1] auto_fix = "--auto-fix" in sys.argv - latex_to_pdf(tex_file, auto_fix) \ No newline at end of file + latex_to_pdf(tex_file, auto_fix) diff --git a/core/markdown_to_latex.py b/core/markdown_to_latex.py index 3d07ea8..98cec86 100644 --- a/core/markdown_to_latex.py +++ b/core/markdown_to_latex.py @@ -81,15 +81,15 @@ def convert_markdown_to_latex(markdown_content: str, output_path: Optional[str] # Headers if re.match(r'^# ', line): - latex_content += f"\\section{{{line[2:]}}}\\n\\n" + latex_content += f"\\section{{{line[2:]}}}\n\n" elif re.match(r'^## ', line): - latex_content += f"\\subsection{{{line[3:]}}}\\n\\n" + latex_content += f"\\subsection{{{line[3:]}}}\n\n" elif re.match(r'^### ', line): - latex_content += f"\\subsubsection{{{line[4:]}}}\\n\\n" + latex_content += f"\\subsubsection{{{line[4:]}}}\n\n" elif re.match(r'^#### ', line): - latex_content += f"\\paragraph{{{line[5:]}}}\\n\\n" + latex_content += f"\\paragraph{{{line[5:]}}}\n\n" elif re.match(r'^##### ', line): - latex_content += f"\\subparagraph{{{line[6:]}}}\\n\\n" + latex_content += f"\\subparagraph{{{line[6:]}}}\n\n" # Images # ![alt text](image_path) @@ -110,20 +110,20 @@ def convert_markdown_to_latex(markdown_content: str, output_path: Optional[str] \\includegraphics[width=0.8\\textwidth]{{{image_path}}} \\caption{{{alt_text}}} \\end{{figure}} -\\n + """ # Code blocks elif line.startswith('```'): if in_code_block: - latex_content += "\\end{lstlisting}\\n\\n" + latex_content += "\\end{lstlisting}\n\n" in_code_block = False else: language = line[3:].strip() if language: - latex_content += f"\\begin{{lstlisting}}[language={language}]\\n" + latex_content += f"\\begin{{lstlisting}}[language={language}]\n" else: - latex_content += "\\begin{lstlisting}\\n" + latex_content += "\\begin{lstlisting}\n" in_code_block = True # Tables @@ -156,18 +156,18 @@ def convert_markdown_to_latex(markdown_content: str, output_path: Optional[str] # Blockquotes elif line.startswith('> '): if not in_blockquote: - latex_content += "\\begin{quote}\\n" + latex_content += "\\begin{quote}\n" in_blockquote = True - latex_content += f"{line[2:]}\\n" + latex_content += f"{line[2:]}\n" elif in_blockquote and not line.startswith('> '): - latex_content += "\\end{quote}\\n\\n" + latex_content += "\\end{quote}\n\n" in_blockquote = False if line.strip(): current_line_idx -= 1 # Process this line again # Regular text (inside or outside code blocks) elif in_code_block: - latex_content += line + "\\n" + latex_content += line + "\n" else: # Ordered lists ordered_list_match = re.match(r'^(\s*)\d+\.\s+(.*)', line) @@ -179,16 +179,16 @@ def convert_markdown_to_latex(markdown_content: str, output_path: Optional[str] if in_ordered_list: # Close previous list if indent level changed for _ in range(list_level + 1): - latex_content += "\\end{enumerate}\\n" + latex_content += "\\end{enumerate}\n" # Start new list with proper nesting for i in range(indent_level + 1): - latex_content += "\\begin{enumerate}\\n" + latex_content += "\\begin{enumerate}\n" in_ordered_list = True list_level = indent_level - latex_content += f"\\item {content}\\n" + latex_content += f"\\item {content}\n" # Unordered lists elif re.match(r'^(\s*)- ', line): @@ -200,52 +200,52 @@ def convert_markdown_to_latex(markdown_content: str, output_path: Optional[str] if in_list: # Close previous list if indent level changed for _ in range(list_level + 1): - latex_content += "\\end{itemize}\\n" + latex_content += "\\end{itemize}\n" # Start new list with proper nesting for i in range(indent_level + 1): - latex_content += "\\begin{itemize}\\n" + latex_content += "\\begin{itemize}\n" in_list = True list_level = indent_level - latex_content += f"\\item {content}\\n" + latex_content += f"\\item {content}\n" # End of lists elif (in_list or in_ordered_list) and line.strip() == "": if in_list: for _ in range(list_level + 1): - latex_content += "\\end{itemize}\\n" + latex_content += "\\end{itemize}\n" in_list = False if in_ordered_list: for _ in range(list_level + 1): - latex_content += "\\end{enumerate}\\n" + latex_content += "\\end{enumerate}\n" in_ordered_list = False list_level = 0 - latex_content += "\\n" + latex_content += "\n" # Bold and italic text elif line.strip(): # Process inline formatting processed_line = process_inline_formatting(line) - latex_content += processed_line + "\\n\\n" + latex_content += processed_line + "\n\n" # Close any open environments if in_list: for _ in range(list_level + 1): - latex_content += "\\end{itemize}\\n" + latex_content += "\\end{itemize}\n" if in_ordered_list: for _ in range(list_level + 1): - latex_content += "\\end{enumerate}\\n" + latex_content += "\\end{enumerate}\n" if in_blockquote: - latex_content += "\\end{quote}\\n" + latex_content += "\\end{quote}\n" if in_code_block: - latex_content += "\\end{lstlisting}\\n" + latex_content += "\\end{lstlisting}\n" if in_table: latex_content += format_table(table_data) @@ -314,4 +314,4 @@ def format_table(table_data: List[List[str]]) -> str: latex_table += ' & '.join(process_inline_formatting(cell) for cell in row) + ' \\\\ \\hline\n' latex_table += "\\end{tabular}\n\\end{table}\n\n" - return latex_table \ No newline at end of file + return latex_table diff --git a/docs/API.md b/docs/API.md index 4517f55..dc23037 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,4 +1,4 @@ -# XToX Converter API Documentation +# Mill API Documentation ## Base URL diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 40e3291..3e6fc3f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,7 +2,7 @@ ## System Overview -XToX Converter is a multi-format document and media conversion platform built with a microservices architecture. +Mill is a multi-format document and media conversion platform built with a microservices architecture. ## Architecture Diagram @@ -49,7 +49,7 @@ XToX Converter is a multi-format document and media conversion platform built wi ### Frontend (React) -- **Location:** `xtox/frontend/` +- **Location:** `frontend/` - **Technology:** React 19, Tailwind CSS, Axios - **Responsibilities:** - User interface for file upload @@ -59,7 +59,7 @@ XToX Converter is a multi-format document and media conversion platform built wi ### Backend API (FastAPI) -- **Location:** `xtox/backend/` +- **Location:** `backend/` - **Technology:** FastAPI, Uvicorn, Motor (MongoDB) - **Responsibilities:** - REST API endpoints @@ -70,7 +70,7 @@ XToX Converter is a multi-format document and media conversion platform built wi ### Core Converters -- **Location:** `xtox/core/` +- **Location:** `core/` (published under the compatible `xtox.core` import) - **Components:** - `AudioConverter`: Handles audio format conversion - `ImageConverter`: Handles image format conversion @@ -78,7 +78,7 @@ XToX Converter is a multi-format document and media conversion platform built wi ### Services Layer -- **Location:** `xtox/backend/services.py` +- **Location:** `backend/services/` - **Components:** - `LatexService`: LaTeX to PDF conversion logic - `AudioService`: Audio conversion orchestration @@ -156,5 +156,7 @@ XToX Converter is a multi-format document and media conversion platform built wi - **Logging:** Structured logging with Python logging - **Metrics:** Application performance metrics - **Error Tracking:** Integration-ready for Sentry -- **Health Checks:** `/api/status` endpoint +- **Liveness:** `/api/health` confirms the API process is serving. +- **Dependency readiness:** `/api/ready` confirms the configured database responds. +- **Persisted status records:** `/api/status` is CRUD data, not a deployment probe. diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index cb3768f..50f77ce 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -3,7 +3,7 @@ ## Getting Started 1. Fork the repository -2. Clone your fork: `git clone https://github.com/your-username/xtox.git` +2. Clone your fork: `git clone https://github.com/your-username/mill.git` 3. Create a branch: `git checkout -b feature/your-feature-name` 4. Make your changes 5. Run tests: `make test` diff --git a/docs/DESIGN_SYSTEM.md b/docs/DESIGN_SYSTEM.md index c25813c..3f13f28 100644 --- a/docs/DESIGN_SYSTEM.md +++ b/docs/DESIGN_SYSTEM.md @@ -2,11 +2,11 @@ ## Overview -XToX Converter uses a comprehensive design system based on Tailwind CSS with custom design tokens. +Mill uses a comprehensive design system based on Tailwind CSS with custom design tokens. ## Design Tokens -Design tokens are defined in `xtox/frontend/src/xtotext Design Tokens.json` and integrated into Tailwind configuration. +Design tokens are defined in `frontend/src/xtotext Design Tokens.json` and integrated into Tailwind configuration. ### Colors @@ -122,14 +122,14 @@ Dark theme tokens are defined but not yet fully implemented. To enable: ## Component Library -See `xtox/frontend/src/components/` for reusable components: +See `frontend/src/components/` for reusable components: - `AccessibleFileUpload` - `AccessibleAlert` - `ProgressBar` ## Resources -- Design Tokens: `xtox/frontend/src/xtotext Design Tokens.json` -- Tailwind Config: `xtox/frontend/tailwind.config.js` -- SCSS Config: `xtox/frontend/src/xtotext SCSS Configuration.scss` +- Design Tokens: `frontend/src/xtotext Design Tokens.json` +- Tailwind Config: `frontend/tailwind.config.js` +- SCSS Config: `frontend/src/xtotext SCSS Configuration.scss` diff --git a/docs/mill-identity-inventory.md b/docs/mill-identity-inventory.md new file mode 100644 index 0000000..649856c --- /dev/null +++ b/docs/mill-identity-inventory.md @@ -0,0 +1,31 @@ +# Mill identity inventory + +Mill is the product name. This inventory prevents a branding pass from breaking +compatibility, infrastructure, identity, persisted records, or audit history. + +| Surface | Classification | Alpha treatment | +| --- | --- | --- | +| README, API title/root response, current product copy | Stale user-facing branding | Present as Mill. | +| npm package and executable | New public surface | *@celladore/mill*; executable *mill*. Never claim the unrelated bare package or *npx mill*. | +| Python distribution *xtotext*, import *xtox*, executable *xtotext* | Compatibility API | Preserve. The Node CLI delegates supported local conversion to this executable. | +| *celladore-xtox* OIDC client and token audience | Load-bearing external identity | Preserve until a separately authorized registration migration exists. | +| *ghcr.io/celladore/xtox-api*, Terraform state key, module/resource names and Azure resource names | Load-bearing cloud identity | Preserve. Renaming source labels does not rename deployed/stateful objects. | +| API routes and serialized fields | External/persisted contract | Preserve. Additive */api/health* liveness and */api/ready* dependency-readiness routes are explicit. | +| MongoDB collection names and stored conversion records | Persisted data contract | Preserve; no migration in this release slice. | +| old migration guides, incident comments, task IDs and historical examples | Historical record | Keep identifiers when needed for provenance; do not present them as the current product name. | +| CoilTrace *mill.render* scope and delegated audience behavior | Compatibility API | Preserve and verify through existing contract tests; do not widen access. | +| legacy Azure Functions source and older deployment/migration guides | Historical/load-bearing boundary | Do not promote as the canonical FastAPI deployment and do not mass-rename without a separately scoped retirement or state migration. | +| *XTOX_MCP_MAX_IMAGE_BYTES* | Compatibility environment variable | Preserve while presenting the MCP server itself as Mill. | +| MCP protocol server name *xtox-images* and webhook User-Agent *XToX-Converter/1.0* | External compatibility contract | Preserve until consumer traces support a versioned migration; human-facing MCP registration examples may use the *mill-images* client alias. | + +Public values used by the npm CLI live in [product.json](../product.json). That +single manifest makes the temporary scoped-package choice easy to reverse without +scattering package names, URLs, and compatibility identifiers through the code. + +If a later explicit decision acquires the bare npm name, publish the same runtime +under that name while retaining *@celladore/mill* as a compatibility package for +a documented migration window. Keep the *mill* executable stable, and do not tie +that npm transfer to the independent *xtotext* distribution or *xtox* import. +Likewise, a future product rebrand should update *product.json* and presentation +copy without rewriting durable API fields, stored records, identity audiences, or +load-bearing cloud names. diff --git a/frontend-example.ts b/frontend-example.ts index d57af10..da42087 100644 --- a/frontend-example.ts +++ b/frontend-example.ts @@ -2,9 +2,9 @@ const API_BASE_URL = process.env.REACT_APP_API_URL || 'http://localhost:7071/api'; /** - * Client for interacting with the XToPDF API + * Client for interacting with the Mill API */ -class XToPdfClient { +class MillClient { /** * Convert a LaTeX file to PDF * @param file - The LaTeX file to convert @@ -54,4 +54,4 @@ class XToPdfClient { } } -export default new XToPdfClient(); \ No newline at end of file +export default new MillClient(); diff --git a/frontend/src/auth/mystiraOidcConfig.js b/frontend/src/auth/mystiraOidcConfig.js index 93d3de8..2dec756 100644 --- a/frontend/src/auth/mystiraOidcConfig.js +++ b/frontend/src/auth/mystiraOidcConfig.js @@ -20,9 +20,9 @@ export function getMystiraOidcSettings() { // automaticSilentRenew iframe, i.e. it reloads the full SPA in that // iframe just to refresh a token. Point it at a minimal dedicated page // instead (see src/silentRenew.js). - silent_redirect_uri: - env.VITE_MYSTIRA_OIDC_SILENT_REDIRECT_URI || `${origin}/silent-renew.html`, + silent_redirect_uri: env.VITE_MYSTIRA_OIDC_SILENT_REDIRECT_URI || `${origin}/silent-renew.html`, response_type: 'code', + disablePKCE: false, scope: configuredScopes(), loadUserInfo: false, automaticSilentRenew: true, diff --git a/frontend/src/auth/mystiraOidcConfig.test.js b/frontend/src/auth/mystiraOidcConfig.test.js index e960229..eaf2525 100644 --- a/frontend/src/auth/mystiraOidcConfig.test.js +++ b/frontend/src/auth/mystiraOidcConfig.test.js @@ -33,6 +33,7 @@ describe('Mystira OIDC configuration', () => { redirect_uri: window.location.origin, post_logout_redirect_uri: window.location.origin, response_type: 'code', + disablePKCE: false, scope: 'openid profile email offline_access', loadUserInfo: false, automaticSilentRenew: true, diff --git a/frontend/src/auth/mystiraOidcInstance.test.js b/frontend/src/auth/mystiraOidcInstance.test.js new file mode 100644 index 0000000..1ec8b39 --- /dev/null +++ b/frontend/src/auth/mystiraOidcInstance.test.js @@ -0,0 +1,88 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const oidc = vi.hoisted(() => ({ + configured: true, + instances: [], +})); + +vi.mock('./mystiraOidcConfig', () => ({ + getMystiraOidcSettings: () => ({ authority: 'https://identity.mystira.app/' }), + isMystiraOidcConfigured: () => oidc.configured, +})); + +vi.mock('oidc-client-ts', () => ({ + UserManager: class UserManager { + constructor() { + this.events = { + addUserLoaded: vi.fn(listener => { + this.onUserLoaded = listener; + }), + addUserUnloaded: vi.fn(listener => { + this.onUserUnloaded = listener; + }), + }; + this.getUser = vi.fn().mockResolvedValue(null); + this.signinRedirectCallback = vi.fn().mockResolvedValue(null); + this.signinSilent = vi.fn().mockResolvedValue(null); + this.signinRedirect = vi.fn().mockResolvedValue(null); + this.signoutRedirect = vi.fn().mockResolvedValue(null); + this.removeUser = vi.fn().mockResolvedValue(null); + oidc.instances.push(this); + } + }, +})); + +async function loadInstance() { + vi.resetModules(); + return import('./mystiraOidcInstance'); +} + +describe('Mystira OIDC lifecycle', () => { + beforeEach(() => { + oidc.configured = true; + oidc.instances.length = 0; + window.history.replaceState({}, '', '/'); + }); + + it('fails closed without identity configuration', async () => { + oidc.configured = false; + const auth = await loadInstance(); + + await auth.initializeMystiraOidc(); + + expect(oidc.instances).toHaveLength(0); + await expect(auth.getMystiraAccessToken()).resolves.toBe(''); + await expect(auth.loginWithMystira()).resolves.toBeUndefined(); + await expect(auth.logoutFromMystira()).resolves.toBeUndefined(); + }); + + it('handles the authorization callback and restores a valid session', async () => { + window.history.replaceState({}, '', '/?code=code-value&state=state-value'); + const replaceState = vi.spyOn(window.history, 'replaceState'); + const auth = await loadInstance(); + const user = { expired: false, access_token: 'access-token' }; + + const initialization = auth.initializeMystiraOidc(); + const manager = oidc.instances[0]; + manager.getUser.mockResolvedValue(user); + await initialization; + + expect(manager.signinRedirectCallback).toHaveBeenCalledOnce(); + expect(replaceState).toHaveBeenCalledWith({}, document.title, '/'); + expect(auth.getActiveMystiraUser()).toBe(user); + await expect(auth.getMystiraAccessToken()).resolves.toBe('access-token'); + }); + + it('starts redirect login and logout through the configured manager', async () => { + const auth = await loadInstance(); + await auth.initializeMystiraOidc(); + const manager = oidc.instances[0]; + + await auth.loginWithMystira(); + await auth.logoutFromMystira(); + + expect(manager.signinRedirect).toHaveBeenCalledOnce(); + expect(manager.signoutRedirect).toHaveBeenCalledOnce(); + expect(auth.getActiveMystiraUser()).toBeNull(); + }); +}); diff --git a/frontend/src/components/MarketingPage.jsx b/frontend/src/components/MarketingPage.jsx index 0145b22..4ac4234 100644 --- a/frontend/src/components/MarketingPage.jsx +++ b/frontend/src/components/MarketingPage.jsx @@ -3,16 +3,15 @@ import { useEffect, useRef, useState } from 'react'; const PREVIEW_MODES = [ { id: 'markdown', label: 'Markdown → PDF', category: 'Document', key: '1' }, { id: 'audio', label: 'Voice → Transcript', category: 'Voice & Audio', key: '2' }, - { id: 'ai', label: 'Docs → AI Context', category: 'LLM Ready', key: '3' }, - { id: 'latex', label: 'LaTeX → Typeset', category: 'Technical', key: '4' }, - { id: 'image', label: 'JPEG → WebP', category: 'Images', key: '5' }, + { id: 'latex', label: 'LaTeX → Typeset', category: 'Technical', key: '3' }, + { id: 'image', label: 'JPEG → WebP', category: 'Images', key: '4' }, ]; const SUPPORTED_FORMAT_GROUPS = [ { id: 'document-text', label: 'Document / Text', - formats: ['Markdown', 'HTML', 'Plain text', 'DOCX', 'LaTeX', 'PDF', 'AI-ready text'], + formats: ['Markdown', 'HTML', 'Plain text', 'DOCX', 'LaTeX'], }, { id: 'image', @@ -28,11 +27,6 @@ const SUPPORTED_FORMAT_GROUPS = [ { id: 'video', label: 'Video', formats: ['MP4', 'WebM', 'MOV'], isNew: true }, ]; -const UPCOMING_FORMAT_GROUPS = [ - { id: 'story-source', label: 'Story source', formats: ['Mystira Story YAML'] }, - { id: 'model-3d', label: '3D interchange', formats: ['GLB', 'GLTF', 'OBJ'] }, -]; - const CODE_SNIPPETS = { curl: `# 1. Compile LaTeX document to publication PDF CONVERSION_ID=$(curl -s -X POST "https://api.mill.celladoresystems.com/api/convert?auto_fix=true" \\ @@ -61,7 +55,7 @@ IMAGE_ID=$(curl -s -X POST "https://api.mill.celladoresystems.com/api/convert-im curl -H "Authorization: Bearer $MYSTIRA_ACCESS_TOKEN" \\ "https://api.mill.celladoresystems.com/api/download-image/$IMAGE_ID" \\ --output banner.webp`, - python: `from core import DocumentConverter + python: `from xtox.core import DocumentConverter # Initialize converter with local output directory converter = DocumentConverter(output_dir="./dist") @@ -119,31 +113,31 @@ const { id: imageId } = await imageResponse.json();`, const FORMAT_ROUTES = { markdown: { name: 'Markdown (.md)', - targets: ['PDF Publication', 'AI-Ready Context', 'Plain Text'], + targets: ['PDF Publication', 'HTML', 'DOCX'], engine: 'Typography & Layout Engine', - latency: '< 320ms', + latency: 'Toolchain dependent', badges: ['Automated PDF Layout', 'Header Hierarchy', 'Table Styling'], }, latex: { name: 'LaTeX Source (.tex)', - targets: ['Typeset PDF', 'AI-Ready Text'], + targets: ['Typeset PDF'], engine: 'TeX Live Compiler + Syntax Auto-Fix', - latency: '< 650ms', + latency: 'Toolchain dependent', badges: ['Math Formula Rendering', 'Syntax Error Recovery', 'Vector Graphics'], }, ogg: { name: 'WhatsApp / Voice (.ogg/.opus)', targets: ['Formatted Transcript (.txt)', 'MP3 Audio (320k)', 'WAV Lossless'], engine: 'Foundry Whisper & FFmpeg Pipeline', - latency: '< 1.2s', - badges: ['Ephemeral Memory Processing', 'Speech Diarization', '48kHz Resampling'], + latency: 'Provider dependent', + badges: ['Ephemeral by Default', 'Language Detection', 'Scoped Delegation'], }, audio: { name: 'Standard Audio (.mp3/.wav/.flac)', targets: ['MP3 Delivery (192k/320k)', 'OGG Opus (Streaming)', 'Formatted Transcript'], engine: 'High-Fidelity Audio Transcoder', - latency: '< 800ms', - badges: ['Bitrate Shaping', 'Dynamic Range Control', 'Sample Rate Normalization'], + latency: 'File dependent', + badges: ['Bitrate Selection', 'Format Conversion', 'Sample Rate Selection'], }, image: { name: 'Images (.jpg/.png/.webp/...)', @@ -155,7 +149,7 @@ const FORMAT_ROUTES = { 'SVG (Deterministic Vector)', ], engine: 'Pillow Transcoder & Vectorizer', - latency: '< 250ms', + latency: 'File dependent', badges: ['EXIF Auto-Orientation', 'Quality & Target-Size Presets', 'Aspect-Preserving Resize'], }, video: { @@ -171,7 +165,7 @@ const FAQ_ITEMS = [ { question: 'What input and output formats does Mill support?', answer: - 'Mill supports Markdown (.md), LaTeX (.tex), PDF, and rich text documents for publishing and AI extraction. Audio supports OGG, Opus, WAV, MP3, M4A, AAC, and FLAC; images support JPEG, PNG, WebP, BMP, TIFF, GIF, and deterministic raster-to-SVG output; and deterministic local video transcoding supports MP4, WebM, and MOV outputs from common video sources.', + 'Mill supports Markdown, HTML, plain text, DOCX, and LaTeX document routes. Audio supports OGG, Opus, WAV, MP3, M4A, AAC, and FLAC; images support JPEG, PNG, WebP, BMP, TIFF, GIF, and deterministic raster-to-SVG output; and deterministic local video transcoding supports MP4, WebM, and MOV outputs from common video sources.', }, { question: 'What are the maximum file upload limits?', @@ -191,12 +185,12 @@ const FAQ_ITEMS = [ { question: 'How does automated syntax repair (auto-fix) work?', answer: - 'When enabled, Mill analyzes document structure and automatically resolves common formatting syntax errors—such as missing documentclass headers, unclosed math blocks, broken markdown fences, and encoding artifacts—ensuring reliable compilation on the first pass.', + 'When enabled, Mill attempts bounded repairs for common LaTeX structural errors before compilation. Results depend on the source document and local TeX toolchain, and unsuccessful repairs are returned as explicit errors.', }, { question: 'How is workspace access secured with Mystira Identity?', answer: - 'Mill is natively integrated with Mystira Identity OIDC. All conversion tools and API endpoints require cryptographically verified access tokens with strict scope isolation, guaranteeing enterprise-grade identity boundaries.', + 'Mill uses Mystira Identity Authorization Code with PKCE for the private workspace. Conversion routes require validated bearer tokens, while public API documentation and health routes remain intentionally unauthenticated.', }, ]; @@ -252,9 +246,8 @@ export function MarketingPage({ if (e.key === '1') setActivePreview('markdown'); if (e.key === '2') setActivePreview('audio'); - if (e.key === '3') setActivePreview('ai'); - if (e.key === '4') setActivePreview('latex'); - if (e.key === '5') setActivePreview('image'); + if (e.key === '3') setActivePreview('latex'); + if (e.key === '4') setActivePreview('image'); }; window.addEventListener('keydown', handleKeyDown); @@ -516,64 +509,11 @@ export function MarketingPage({
AUDIO & VOICE - Lossless transcode & private transcription + Format transcode & private transcription EPHEMERAL
- {/* AI Context / LLM Ingestion Preview */} - - {/* LaTeX to PDF Preview (Technical) */}
- {/* Performance & Trust Metrics Banner */} -
+ {/* Verifiable release and privacy properties */} +
- < 400ms - Median Rendering Latency + Live + Frontend & API
@@ -705,15 +645,6 @@ export function MarketingPage({
))}
-
- {UPCOMING_FORMAT_GROUPS.map(group => ( -
-

{group.label}

-

{group.formats.join(' · ')}

- [Coming soon] -
- ))} -
{/* Capabilities Grid Section */} @@ -728,9 +659,8 @@ export function MarketingPage({

Engineered for clean, faithful transformations.

- Mill converts complex sources into clean, actionable formats without data bloat, leaky - storage, or lost meaning. Live capabilities are separated from the managed pipelines - that are next on the roadmap. + Mill routes supported sources through explicit local or authenticated conversion + boundaries and returns concrete output artifacts.

@@ -795,12 +725,12 @@ export function MarketingPage({ - LLM Ready + Text
-

AI & LLM-Ready Ingestion

+

Deterministic Text Reshaping

- Extract clean, structured Markdown from multi-page PDFs and docs with automated - token compression and semantic context tagging for AI workflows. + Convert Markdown, HTML, plain text, and DOCX through bounded, deterministic + transformations with downloadable results.

@@ -813,8 +743,8 @@ export function MarketingPage({

Precision LaTeX Typesetting

- Compile scientific papers, math formulas, and TeX documents with automated syntax - repair and instant PDF generation. + Compile scientific papers, math formulas, and TeX documents with optional bounded + syntax repair and explicit compilation errors.

@@ -832,34 +762,6 @@ export function MarketingPage({ handling, and input-to-output size history.

- -
-
- - [Coming soon] -
-

Mystira Story YAML → Images / Video

-

- Turn a governed Mystira Story definition into a managed sequence of image and video - artifacts, with provenance, bounded retries, and observable pipeline status. -

-
- -
-
- - [Coming soon] -
-

Image → 3D Model Pipeline

-

- Build production-ready 3D assets through a managed multi-step workflow for geometry, - cleanup, materials, validation, and export. -

-
@@ -896,7 +798,7 @@ export function MarketingPage({ ENGINE / PIPELINE

{routeData.engine}

- {selectedRoute === 'video' ? 'Processing bound' : 'Avg. Speed'}:{' '} + {selectedRoute === 'video' ? 'Processing bound' : 'Execution'}:{' '} {routeData.latency}
@@ -929,14 +831,20 @@ export function MarketingPage({

API-First Architecture

-

Integrate conversion into your pipelines in seconds.

+

Integrate conversion through explicit boundaries.

- Every feature in the workspace is backed by our authenticated REST API and - Python/TypeScript SDKs. + Workspace conversions use the authenticated REST API. Local document conversion is + available through the compatible Python package. The scoped npm CLI is approved for + this alpha and becomes installable after registry publication.

+

+ Scoped command after publication: npx @celladore/mill --help. The unrelated + bare command npx mill is not supported. +

+
@@ -953,8 +861,8 @@ export function MarketingPage({ {lang === 'curl' ? 'cURL' : lang === 'python' - ? 'Python SDK' - : 'TypeScript / Node'} + ? 'Python package' + : 'TypeScript / REST'} ))}
@@ -1007,16 +915,16 @@ export function MarketingPage({ 02

Select & Optimize

- Choose your target format, enable automated syntax repair, token compression, or - audio bitrate tuning. + Choose your target format and, where supported, bounded syntax repair, image + quality, or audio bitrate controls.

03 -

Instant Export

+

Retrieve Output

- Download your publication PDF, transcoded audio, or copy clean text with complete - privacy guarantees. + Download the resulting PDF or media artifact, or copy generated text within your + authenticated workspace.

diff --git a/frontend/src/components/MarketingPage.test.jsx b/frontend/src/components/MarketingPage.test.jsx index 493ede1..5e6d009 100644 --- a/frontend/src/components/MarketingPage.test.jsx +++ b/frontend/src/components/MarketingPage.test.jsx @@ -97,7 +97,7 @@ describe('MarketingPage', () => { expect(markup).toContain('Universal Document Publishing'); expect(markup).toContain('High-Fidelity Audio Reshaping'); expect(markup).toContain('Ephemeral Voice Transcription'); - expect(markup).toContain('AI & LLM-Ready Ingestion'); + expect(markup).toContain('Deterministic Text Reshaping'); expect(markup).toContain('Precision LaTeX Typesetting'); // Default preview tab is Markdown to PDF @@ -111,22 +111,16 @@ describe('MarketingPage', () => { expect(markup).toContain('[Image]'); expect(markup).toContain('[Audio / Speech]'); expect(markup).toContain('Plain text'); - expect(markup).toContain('AI-ready text'); - expect(markup).toContain('Mystira Story YAML'); expect(markup).toContain('SVG'); expect(markup).toContain('[Video]'); expect(markup).toContain('[New]'); expect(markup).toContain('MP4'); expect(markup).toContain('WebM'); expect(markup).toContain('MOV'); - expect(markup).toContain('GLB · GLTF · OBJ'); - expect(markup.match(/\[Coming soon\]/g)).toHaveLength(4); - expect(markup).toContain( - 'class="upcoming-format-groups" role="group" aria-label="Coming soon formats"' - ); - expect(markup).toContain('Mystira Story YAML → Images / Video'); - expect(markup).toContain('Image → 3D Model Pipeline'); - expect(markup.match(/capability-card is-upcoming/g)).toHaveLength(2); + expect(markup).not.toContain('Mystira Story YAML'); + expect(markup).not.toContain('GLB · GLTF · OBJ'); + expect(markup).not.toContain('Docs → AI Context'); + expect(markup).not.toContain('RAG-optimized chunking'); }); it('renders trust metrics, developer code snippets, route inspector, and FAQ accordion', () => { @@ -135,14 +129,16 @@ describe('MarketingPage', () => { ); // Trust metrics band - expect(markup).toContain('Median Rendering Latency'); - expect(markup).toContain('Ephemeral Voice Processing'); - expect(markup).toContain('Mystira Authenticated Sessions'); + expect(markup).toContain('Frontend & API'); + expect(markup).toContain('Ephemeral Transcription'); + expect(markup).toContain('Current Release Track'); // Developer section & code tabs expect(markup).toContain('API-First Architecture'); - expect(markup).toContain('Python SDK'); - expect(markup).toContain('TypeScript / Node'); + expect(markup).toContain('Python package'); + expect(markup).toContain('TypeScript / REST'); + expect(markup).toContain('npx @celladore/mill --help'); + expect(markup).toContain('npx mill'); expect(markup).toContain('api.mill.celladoresystems.com'); // Interactive Route Matrix @@ -228,7 +224,7 @@ describe('MarketingPage', () => { expect(container.querySelector('.route-engine-name').textContent).toContain( 'TeX Live Compiler + Syntax Auto-Fix' ); - expect(container.querySelector('.route-metric-pill').textContent).toContain('Avg. Speed:'); + expect(container.querySelector('.route-metric-pill').textContent).toContain('Execution:'); const videoRouteBtn = Array.from(container.querySelectorAll('.route-btn')).find(btn => btn.textContent.includes('Video (.mp4/.mov/.mkv/.webm/...)') @@ -277,7 +273,9 @@ describe('MarketingPage', () => { const faqAnswer = container.querySelector('#faq-answer-0'); expect(faqAnswer).not.toBeNull(); - expect(faqAnswer.textContent).toContain('Markdown (.md), LaTeX (.tex)'); + expect(faqAnswer.textContent).toContain( + 'Markdown, HTML, plain text, DOCX, and LaTeX document routes', + ); await act(async () => { firstFaqBtn.click(); diff --git a/mcp_server/README.md b/mcp_server/README.md index 31886da..89521e5 100644 --- a/mcp_server/README.md +++ b/mcp_server/README.md @@ -1,6 +1,6 @@ -# xtox image conversion — MCP server +# Mill image conversion — MCP server -A local stdio [MCP](https://modelcontextprotocol.io) server that exposes xtox's +A local stdio [MCP](https://modelcontextprotocol.io) server that exposes Mill's Pillow-backed image conversion (`core/image_converter.py`) as tools any MCP client can call — Claude Code, Claude Desktop, or another agent. @@ -31,7 +31,7 @@ pip install -r mcp_server/requirements.txt ## Register with Claude Code ```bash -claude mcp add xtox-images -- python /absolute/path/to/xtox/mcp_server/server.py +claude mcp add mill-images -- python /absolute/path/to/mill/mcp_server/server.py ``` Or add directly to `.mcp.json`: @@ -39,16 +39,16 @@ Or add directly to `.mcp.json`: ```json { "mcpServers": { - "xtox-images": { + "mill-images": { "command": "python", - "args": ["/absolute/path/to/xtox/mcp_server/server.py"] + "args": ["/absolute/path/to/mill/mcp_server/server.py"] } } } ``` Use the interpreter that has `mcp_server/requirements.txt` installed (a venv -path, e.g. `/absolute/path/to/xtox/.venv/bin/python`, if you're not using the +path, e.g. `/absolute/path/to/mill/.venv/bin/python`, if you're not using the system Python). ## Notes diff --git a/mcp_server/server.py b/mcp_server/server.py index 694c1de..43f2537 100644 --- a/mcp_server/server.py +++ b/mcp_server/server.py @@ -1,5 +1,5 @@ """ -MCP (Model Context Protocol) stdio server exposing xtox's image conversion +MCP (Model Context Protocol) stdio server exposing Mill's image conversion as callable tools. Runs entirely in-process against core/image_converter.py — it does not call @@ -13,7 +13,7 @@ python mcp_server/server.py Register with Claude Code: - claude mcp add xtox-images -- python /absolute/path/to/mcp_server/server.py + claude mcp add mill-images -- python /absolute/path/to/mcp_server/server.py See mcp_server/README.md for the full registration snippet (including the .mcp.json form) and the list of tools this server exposes. @@ -48,11 +48,13 @@ logger = logging.getLogger("xtox-mcp") mcp = MCPServer( + # Protocol identity retained for existing MCP consumers. Client-side + # registration aliases and all human-facing copy use Mill. name="xtox-images", version="1.0.0", instructions=( "Convert and inspect image files (JPEG, PNG, WebP, BMP, TIFF, GIF) " - "using xtox's Pillow-backed converter. Prefer convert_image when " + "using Mill's Pillow-backed converter. Prefer convert_image when " "both the client and server share a filesystem (e.g. Claude Code); " "use convert_image_base64 when the client can only send/receive " "inline image bytes (e.g. a chat client with no shared disk)." diff --git a/mill-cli/api.mjs b/mill-cli/api.mjs new file mode 100644 index 0000000..575b169 --- /dev/null +++ b/mill-cli/api.mjs @@ -0,0 +1,94 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +import { defaultAudioOutput } from './files.mjs'; + +async function responseError(response) { + let detail = `${response.status} ${response.statusText}`; + try { + const body = await response.json(); + detail = body.detail || body.message || detail; + } catch { + // The status line is sufficient when the body is not JSON. + } + const error = new Error(`Mill API request failed: ${detail}`); + error.exitCode = response.status === 401 || response.status === 403 ? 2 : 1; + return error; +} + +export async function convertAudio({ + input, + output, + targetFormat = 'mp3', + bitrate = '192k', + sampleRate, + apiUrl, + token, + force = false, + fetchImpl = fetch, +}) { + if (!token) { + const error = new Error( + 'Audio conversion requires an operator-provided Mystira access token in MYSTIRA_ACCESS_TOKEN. Web login does not currently hand a token to this CLI; the alpha CLI does not implement a separate OAuth client or store credentials.', + ); + error.exitCode = 2; + throw error; + } + + const inputPath = path.resolve(input); + const outputPath = path.resolve(output || defaultAudioOutput(input, targetFormat)); + if (path.relative(inputPath, outputPath) === '') { + const error = new Error('Output path must differ from the input path'); + error.exitCode = 2; + throw error; + } + if (!force) { + try { + await readFile(outputPath); + const error = new Error(`${outputPath} already exists; use --force to replace it`); + error.exitCode = 2; + throw error; + } catch (error) { + if (error.code !== 'ENOENT') throw error; + } + } + + const source = await readFile(inputPath); + const form = new FormData(); + form.append('file', new Blob([source]), path.basename(inputPath)); + const query = new URLSearchParams({ target_format: targetFormat, bitrate }); + if (sampleRate) query.set('sample_rate', String(sampleRate)); + const headers = { Authorization: `Bearer ${token}`, 'X-Request-ID': crypto.randomUUID() }; + const conversion = await fetchImpl(`${apiUrl}/api/convert-audio?${query}`, { + method: 'POST', + headers, + body: form, + }); + if (!conversion.ok) throw await responseError(conversion); + const result = await conversion.json(); + if (!result.id || result.success === false) { + throw new Error(`Mill API did not complete the conversion: ${(result.errors || []).join('; ') || 'missing conversion id'}`); + } + + const download = await fetchImpl(`${apiUrl}/api/download-audio/${encodeURIComponent(result.id)}`, { headers }); + if (!download.ok) throw await responseError(download); + await writeFile(outputPath, Buffer.from(await download.arrayBuffer())); + return { ...result, output: outputPath }; +} + +export async function probeApiDocs(apiUrl, fetchImpl = fetch) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5000); + try { + const response = await fetchImpl(`${apiUrl}/docs`, { signal: controller.signal }); + return { + available: response.ok, + status: response.status, + note: 'This checks API documentation availability, not application readiness.', + }; + } catch (error) { + return { available: false, error: error.message, note: 'No readiness claim is made.' }; + } finally { + clearTimeout(timeout); + } +} diff --git a/mill-cli/args.mjs b/mill-cli/args.mjs new file mode 100644 index 0000000..568762e --- /dev/null +++ b/mill-cli/args.mjs @@ -0,0 +1,29 @@ +export function parseArgs(argv) { + const positionals = []; + const options = {}; + const booleanOptions = new Set(['force', 'json', 'help', 'version', 'api', 'verbose']); + + for (let index = 0; index < argv.length; index += 1) { + const value = argv[index]; + if (value === '--') { + positionals.push(...argv.slice(index + 1)); + break; + } + if (!value.startsWith('--')) { + positionals.push(value); + continue; + } + const equals = value.indexOf('='); + const key = value.slice(2, equals === -1 ? undefined : equals); + if (booleanOptions.has(key)) { + options[key] = equals === -1 ? true : value.slice(equals + 1) !== 'false'; + continue; + } + const optionValue = equals === -1 ? argv[++index] : value.slice(equals + 1); + if (optionValue === undefined || optionValue.startsWith('--')) { + throw new Error(`--${key} requires a value`); + } + options[key] = optionValue; + } + return { positionals, options }; +} diff --git a/mill-cli/config.mjs b/mill-cli/config.mjs new file mode 100644 index 0000000..b7eb902 --- /dev/null +++ b/mill-cli/config.mjs @@ -0,0 +1,75 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +import product from '../product.json' with { type: 'json' }; + +export const CONFIG_FILENAME = '.millrc.json'; + +export function configPath(cwd = process.cwd()) { + return path.join(cwd, CONFIG_FILENAME); +} + +export async function readConfig(cwd = process.cwd()) { + const filename = configPath(cwd); + try { + const value = JSON.parse(await readFile(filename, 'utf8')); + return { + path: filename, + exists: true, + value: { + schemaVersion: 1, + apiUrl: product.apiUrl, + pythonExecutable: product.pythonExecutable, + ...value, + }, + }; + } catch (error) { + if (error.code === 'ENOENT') { + return { + path: filename, + exists: false, + value: { + schemaVersion: 1, + apiUrl: product.apiUrl, + pythonExecutable: product.pythonExecutable, + }, + }; + } + if (error instanceof SyntaxError) { + throw new Error(`${filename} is not valid JSON`); + } + throw error; + } +} + +export async function writeConfig(cwd, value, { force = false } = {}) { + const current = await readConfig(cwd); + if (current.exists && !force) { + const error = new Error(`${current.path} already exists; use --force to replace it`); + error.exitCode = 2; + throw error; + } + const next = { + schemaVersion: 1, + apiUrl: value.apiUrl || product.apiUrl, + pythonExecutable: product.pythonExecutable, + }; + await writeFile(current.path, `${JSON.stringify(next, null, 2)}\n`, { + encoding: 'utf8', + flag: force ? 'w' : 'wx', + }); + return { path: current.path, value: next }; +} + +export function normalizeApiUrl(value) { + let url; + try { + url = new URL(value); + } catch { + throw new Error(`Invalid API URL: ${value}`); + } + if (!['http:', 'https:'].includes(url.protocol)) { + throw new Error('API URL must use http or https'); + } + return url.toString().replace(/\/$/, ''); +} diff --git a/mill-cli/files.mjs b/mill-cli/files.mjs new file mode 100644 index 0000000..a9d77b4 --- /dev/null +++ b/mill-cli/files.mjs @@ -0,0 +1,32 @@ +import { access, stat } from 'node:fs/promises'; +import path from 'node:path'; + +export const AUDIO_EXTENSIONS = new Set(['.ogg', '.opus', '.wav', '.mp3', '.m4a', '.aac', '.flac']); +export const LOCAL_EXTENSIONS = new Set([ + '.md', '.markdown', '.html', '.htm', '.tex', '.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.gif', '.webp', +]); + +export async function inspectFile(filename) { + const absolutePath = path.resolve(filename); + await access(absolutePath); + const details = await stat(absolutePath); + if (!details.isFile()) { + throw new Error(`Not a file: ${absolutePath}`); + } + const extension = path.extname(absolutePath).toLowerCase(); + let execution = 'unsupported'; + if (AUDIO_EXTENSIONS.has(extension)) execution = 'authenticated-api'; + if (LOCAL_EXTENSIONS.has(extension)) execution = 'local-python'; + return { + path: absolutePath, + name: path.basename(absolutePath), + extension, + sizeBytes: details.size, + execution, + }; +} + +export function defaultAudioOutput(input, targetFormat) { + const parsed = path.parse(path.resolve(input)); + return path.join(parsed.dir, `${parsed.name}.${targetFormat}`); +} diff --git a/mill-cli/main.mjs b/mill-cli/main.mjs new file mode 100644 index 0000000..6d43807 --- /dev/null +++ b/mill-cli/main.mjs @@ -0,0 +1,221 @@ +import path from 'node:path'; + +import product from '../product.json' with { type: 'json' }; +import { convertAudio, probeApiDocs } from './api.mjs'; +import { parseArgs } from './args.mjs'; +import { normalizeApiUrl, readConfig, writeConfig } from './config.mjs'; +import { defaultAudioOutput, inspectFile } from './files.mjs'; +import { commandAvailable, run } from './process.mjs'; + +const HELP = `Mill ${product.version} (${product.status}) + +Usage: + mill init [--api-url URL] [--force] [--json] + mill login [--json] + mill convert INPUT [--format FORMAT] [--output FILE] [--output-dir DIR] [--force] [--json] + mill inspect INPUT [--json] + mill doctor [--api] [--json] + mill --help + mill --version + +Examples: + npx @celladore/mill init + npx @celladore/mill convert "voice notes/input.ogg" --format mp3 + npm install --global @celladore/mill + +The Python distribution/import names xtotext and xtox remain compatibility APIs. +Audio uses Mill's authenticated API and requires MYSTIRA_ACCESS_TOKEN.`; + +const COMMAND_OPTIONS = { + init: new Set(['api-url', 'force', 'json']), + login: new Set(['json']), + convert: new Set([ + 'format', + 'output', + 'output-dir', + 'force', + 'json', + 'bitrate', + 'sample-rate', + 'api-url', + 'verbose', + ]), + inspect: new Set(['json']), + doctor: new Set(['api', 'json']), +}; + +function validateInvocation(command, positionals, options) { + const allowed = COMMAND_OPTIONS[command]; + if (!allowed) { + const error = new Error(`Unknown command: ${command}. Run "mill --help".`); + error.exitCode = 2; + throw error; + } + const unknown = Object.keys(options).find(option => !allowed.has(option)); + if (unknown) { + const error = new Error(`Unknown option for ${command}: --${unknown}. Run "mill ${command} --help".`); + error.exitCode = 2; + throw error; + } + const expected = command === 'convert' || command === 'inspect' ? 1 : 0; + if (positionals.length !== expected) { + const noun = expected === 1 ? 'exactly one input file' : 'no positional arguments'; + const error = new Error(`${command} expects ${noun}`); + error.exitCode = 2; + throw error; + } +} + +function print(value, json = false) { + console.log(json ? JSON.stringify(value, null, 2) : value); +} + +async function initCommand(options) { + const apiUrl = normalizeApiUrl(options['api-url'] || product.apiUrl); + const result = await writeConfig(process.cwd(), { apiUrl }, { force: options.force }); + print(options.json ? result : `Created ${result.path}\nAPI: ${result.value.apiUrl}`, options.json); +} + +async function loginCommand(options, environment) { + const tokenPresent = Boolean(environment[product.tokenEnvironmentVariable]); + const result = { + authenticatedInputPresent: tokenPresent, + tokenSource: product.tokenEnvironmentVariable, + loginUrl: product.siteUrl, + storesCredentials: false, + note: tokenPresent + ? 'A Mystira access token is available to this process. The API validates it on each request.' + : 'No operator-provided token is available. Web login does not currently hand a token to this CLI. Native CLI OAuth is not part of this alpha package; no credential is fabricated or stored.', + }; + print(options.json ? result : `${result.note}\nLogin: ${result.loginUrl}`, options.json); + if (!tokenPresent) process.exitCode = 2; +} + +async function inspectCommand(input, options) { + if (!input) throw new Error('inspect requires an input file'); + const file = await inspectFile(input); + const result = { + ...file, + requirement: + file.execution === 'authenticated-api' + ? product.tokenEnvironmentVariable + : file.execution === 'local-python' + ? product.pythonExecutable + : 'No alpha conversion route is defined for this format.', + }; + print(options.json ? result : `${result.path}\nMode: ${result.execution}\nRequirement: ${result.requirement}`, options.json); + if (file.execution === 'unsupported') process.exitCode = 2; +} + +export function resolveAuthenticatedOutput(input, options, targetFormat) { + if (options.output && options['output-dir']) { + const error = new Error('Use either --output or --output-dir, not both'); + error.exitCode = 2; + throw error; + } + if (options.output) return path.resolve(options.output); + if (!options['output-dir']) return undefined; + return path.join( + path.resolve(options['output-dir']), + path.basename(defaultAudioOutput(input, targetFormat)), + ); +} + +async function convertCommand(input, options, environment) { + if (!input) throw new Error('convert requires an input file'); + const file = await inspectFile(input); + const config = await readConfig(); + if (file.execution === 'authenticated-api') { + const targetFormat = options.format || 'mp3'; + const result = await convertAudio({ + input: file.path, + output: resolveAuthenticatedOutput(file.path, options, targetFormat), + targetFormat, + bitrate: options.bitrate || '192k', + sampleRate: options['sample-rate'], + apiUrl: normalizeApiUrl(options['api-url'] || config.value.apiUrl), + token: environment[product.tokenEnvironmentVariable], + force: options.force, + }); + print(options.json ? result : `Converted ${file.name}\nOutput: ${result.output}`, options.json); + return; + } + if (file.execution !== 'local-python') { + const error = new Error(`No alpha conversion route is defined for ${file.extension || 'files without an extension'}`); + error.exitCode = 2; + throw error; + } + if (options.json) { + const error = new Error( + '--json is not available for the local xtotext compatibility engine; omit --json or use mill inspect --json first', + ); + error.exitCode = 2; + throw error; + } + if (options.output) { + throw new Error('Local Python conversion accepts --output-dir, not --output'); + } + const args = [file.path]; + if (options['output-dir']) args.push('--output', path.resolve(options['output-dir'])); + if (options.format) args.push('--format', options.format); + if (options.verbose) args.push('--verbose'); + const code = await run(config.value.pythonExecutable, args); + if (code !== 0) { + const error = new Error(`${config.value.pythonExecutable} exited with code ${code}`); + error.exitCode = code; + throw error; + } +} + +async function doctorCommand(options, environment) { + const config = await readConfig(); + const [nodeMajor, nodeMinor] = process.versions.node.split('.').map(Number); + const result = { + product: `${product.name} ${product.version} (${product.status})`, + node: { + version: process.versions.node, + supported: nodeMajor > 20 || (nodeMajor === 20 && nodeMinor >= 10), + }, + config: { path: config.path, exists: config.exists, apiUrl: config.value.apiUrl }, + auth: { + environmentVariable: product.tokenEnvironmentVariable, + present: Boolean(environment[product.tokenEnvironmentVariable]), + storedByMill: false, + }, + localPython: { + executable: config.value.pythonExecutable, + available: await commandAvailable(config.value.pythonExecutable), + compatibilityDistribution: product.pythonDistribution, + compatibilityImport: product.pythonImport, + }, + }; + if (options.api) result.apiDocs = await probeApiDocs(normalizeApiUrl(config.value.apiUrl)); + print( + options.json + ? result + : [ + result.product, + `Node: ${result.node.version} (${result.node.supported ? 'supported' : 'unsupported'})`, + `Config: ${result.config.exists ? result.config.path : 'defaults (run mill init)'}`, + `Mystira token: ${result.auth.present ? 'present' : 'absent'}`, + `Local ${result.localPython.executable}: ${result.localPython.available ? 'available' : 'not found'}`, + ...(result.apiDocs + ? [`API docs: ${result.apiDocs.available ? `available (${result.apiDocs.status})` : 'unavailable'}`, result.apiDocs.note] + : []), + ].join('\n'), + options.json, + ); +} + +export async function main(argv, { environment = process.env } = {}) { + const { positionals, options } = parseArgs(argv); + const command = positionals.shift(); + if (options.version || command === 'version') return print(product.version); + if (options.help || !command || command === 'help') return print(HELP); + validateInvocation(command, positionals, options); + if (command === 'init') return initCommand(options); + if (command === 'login') return loginCommand(options, environment); + if (command === 'inspect') return inspectCommand(positionals[0], options); + if (command === 'convert') return convertCommand(positionals[0], options, environment); + if (command === 'doctor') return doctorCommand(options, environment); +} diff --git a/mill-cli/process.mjs b/mill-cli/process.mjs new file mode 100644 index 0000000..a1d862f --- /dev/null +++ b/mill-cli/process.mjs @@ -0,0 +1,34 @@ +import { spawn } from 'node:child_process'; + +export function run(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { shell: false, stdio: 'inherit', ...options }); + child.once('error', error => { + if (error.code === 'ENOENT') { + const wrapped = new Error( + `Could not find ${command}. Install the compatible Python distribution with "python -m pip install xtotext".`, + ); + wrapped.exitCode = 2; + reject(wrapped); + return; + } + reject(error); + }); + child.once('exit', (code, signal) => { + if (signal) reject(new Error(`${command} stopped by signal ${signal}`)); + else resolve(code ?? 1); + }); + }); +} + +export async function commandAvailable(command) { + return new Promise(resolve => { + const child = spawn(command, ['--help'], { + shell: false, + stdio: 'ignore', + windowsHide: true, + }); + child.once('error', () => resolve(false)); + child.once('exit', code => resolve(code === 0)); + }); +} diff --git a/mill-cli/test/cli.test.mjs b/mill-cli/test/cli.test.mjs new file mode 100644 index 0000000..198f536 --- /dev/null +++ b/mill-cli/test/cli.test.mjs @@ -0,0 +1,190 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; + +import product from '../../product.json' with { type: 'json' }; +import packageManifest from '../../package.json' with { type: 'json' }; +import { convertAudio } from '../api.mjs'; +import { parseArgs } from '../args.mjs'; +import { inspectFile } from '../files.mjs'; +import { resolveAuthenticatedOutput } from '../main.mjs'; +import { commandAvailable } from '../process.mjs'; + +test('public package metadata stays synchronized', () => { + assert.equal(packageManifest.name, product.npmPackage); + assert.equal(packageManifest.version, product.version); + assert.deepEqual(packageManifest.bin, { mill: 'bin/mill.js' }); + assert.equal(packageManifest.engines.node, '>=20.10.0'); +}); + +test('argument parser preserves paths with spaces as one value', () => { + const parsed = parseArgs(['convert', 'voice notes/input.ogg', '--output', 'converted notes/result.mp3']); + assert.deepEqual(parsed.positionals, ['convert', 'voice notes/input.ogg']); + assert.equal(parsed.options.output, 'converted notes/result.mp3'); +}); + +test('init is non-interactive and writes non-secret configuration', async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), 'mill init ')); + const result = spawnSync(process.execPath, [path.resolve('bin/mill.js'), 'init', '--json'], { + cwd: directory, + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stderr); + const config = JSON.parse(await readFile(path.join(directory, '.millrc.json'), 'utf8')); + assert.equal(config.apiUrl, product.apiUrl); + assert.equal(config.pythonExecutable, product.pythonExecutable); + assert.doesNotMatch(JSON.stringify(config), /token|secret/i); +}); + +test('login reports the external token boundary without storing credentials', () => { + const environment = { ...process.env }; + delete environment.MYSTIRA_ACCESS_TOKEN; + const result = spawnSync(process.execPath, [path.resolve('bin/mill.js'), 'login', '--json'], { + env: environment, + encoding: 'utf8', + }); + assert.equal(result.status, 2); + const report = JSON.parse(result.stdout); + assert.equal(report.authenticatedInputPresent, false); + assert.equal(report.storesCredentials, false); + assert.match(report.note, /does not currently hand a token/); +}); + +test('doctor is useful in a clean non-TTY process', () => { + const result = spawnSync(process.execPath, [path.resolve('bin/mill.js'), 'doctor', '--json'], { + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stderr); + const report = JSON.parse(result.stdout); + assert.equal(report.node.supported, true); + assert.equal(report.auth.storedByMill, false); + assert.equal(report.localPython.compatibilityImport, 'xtox'); +}); + +test('executable diagnostics probe the command directly', async () => { + assert.equal(await commandAvailable(process.execPath), true); + assert.equal(await commandAvailable('mill-command-that-does-not-exist'), false); +}); + +test('unknown options fail actionably instead of being ignored', () => { + const result = spawnSync( + process.execPath, + [path.resolve('bin/mill.js'), 'convert', 'input.md', '--formta', 'pdf'], + { encoding: 'utf8' }, + ); + assert.equal(result.status, 2); + assert.match(result.stderr, /Unknown option for convert: --formta/); +}); + +test('extra positional arguments are rejected', () => { + const result = spawnSync( + process.execPath, + [path.resolve('bin/mill.js'), 'inspect', 'first.md', 'second.md'], + { encoding: 'utf8' }, + ); + assert.equal(result.status, 2); + assert.match(result.stderr, /inspect expects exactly one input file/); +}); + +test('local conversion rejects unsupported JSON output explicitly', async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), 'mill local json ')); + const input = path.join(directory, 'input file.md'); + await writeFile(input, '# fixture'); + const result = spawnSync( + process.execPath, + [path.resolve('bin/mill.js'), 'convert', input, '--format', 'html', '--json'], + { encoding: 'utf8' }, + ); + assert.equal(result.status, 2); + assert.match(result.stderr, /--json is not available for the local xtotext compatibility engine/); +}); + +test('inspect routes audio to the authenticated API boundary', async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), 'mill inspect ')); + const input = path.join(directory, 'voice note.ogg'); + await writeFile(input, 'fixture'); + const result = await inspectFile(input); + assert.equal(result.execution, 'authenticated-api'); + assert.equal(result.path, input); +}); + +test('audio conversion fails closed without Mystira identity', async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), 'mill auth ')); + const input = path.join(directory, 'voice note.ogg'); + await writeFile(input, 'fixture'); + await assert.rejects( + convertAudio({ input, apiUrl: product.apiUrl, token: '' }), + /operator-provided Mystira access token in MYSTIRA_ACCESS_TOKEN/, + ); +}); + +test('audio conversion never overwrites its input, even with force', async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), 'mill same path ')); + const input = path.join(directory, 'voice note.ogg'); + await writeFile(input, 'fixture'); + await assert.rejects( + convertAudio({ + input, + output: input, + apiUrl: product.apiUrl, + token: 'test-token', + force: true, + }), + /Output path must differ from the input path/, + ); + assert.equal(await readFile(input, 'utf8'), 'fixture'); +}); + +test('authenticated output-dir preserves the derived filename and spaces', () => { + const output = resolveAuthenticatedOutput( + path.join('source notes', 'voice note.ogg'), + { 'output-dir': path.join('converted notes', 'release output') }, + 'mp3', + ); + assert.equal( + output, + path.resolve('converted notes', 'release output', 'voice note.mp3'), + ); + assert.throws( + () => + resolveAuthenticatedOutput( + 'voice.ogg', + { output: 'voice.mp3', 'output-dir': 'converted' }, + 'mp3', + ), + /Use either --output or --output-dir, not both/, + ); +}); + +test('audio conversion uses the API contract and retrieves the output', async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), 'mill api ')); + const input = path.join(directory, 'voice note.ogg'); + const output = path.join(directory, 'result file.mp3'); + await writeFile(input, 'audio'); + const calls = []; + const fetchImpl = async (url, options) => { + calls.push({ url, options }); + if (calls.length === 1) { + return new Response(JSON.stringify({ id: 'conversion-1', filename: 'voice note.ogg', success: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response('converted-audio', { status: 200 }); + }; + const result = await convertAudio({ + input, + output, + apiUrl: product.apiUrl, + token: 'test-token', + fetchImpl, + }); + assert.match(calls[0].url, /\/api\/convert-audio\?target_format=mp3&bitrate=192k$/); + assert.equal(calls[0].options.headers.Authorization, 'Bearer test-token'); + assert.match(calls[1].url, /\/api\/download-audio\/conversion-1$/); + assert.equal(await readFile(output, 'utf8'), 'converted-audio'); + assert.equal(result.output, output); +}); diff --git a/package.json b/package.json new file mode 100644 index 0000000..f3a6571 --- /dev/null +++ b/package.json @@ -0,0 +1,32 @@ +{ + "name": "@celladore/mill", + "version": "0.1.0", + "description": "Mill alpha command-line interface for local and authenticated conversions", + "type": "module", + "bin": { + "mill": "bin/mill.js" + }, + "files": [ + "bin/", + "mill-cli/*.mjs", + "product.json", + "README.md" + ], + "scripts": { + "test": "node --test mill-cli/test/*.test.mjs", + "pack:check": "npm pack --dry-run" + }, + "engines": { + "node": ">=20.10.0" + }, + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/celladore/mill.git" + }, + "homepage": "https://mill.celladoresystems.com", + "bugs": "https://github.com/celladore/mill/issues", + "license": "UNLICENSED" +} diff --git a/product.json b/product.json new file mode 100644 index 0000000..efecaae --- /dev/null +++ b/product.json @@ -0,0 +1,14 @@ +{ + "name": "Mill", + "description": "Convert documents and media through local deterministic tools or Mill's authenticated API.", + "status": "alpha", + "version": "0.1.0", + "npmPackage": "@celladore/mill", + "siteUrl": "https://mill.celladoresystems.com", + "apiUrl": "https://api.mill.celladoresystems.com", + "repositoryUrl": "https://github.com/celladore/mill", + "tokenEnvironmentVariable": "MYSTIRA_ACCESS_TOKEN", + "pythonDistribution": "xtotext", + "pythonImport": "xtox", + "pythonExecutable": "xtotext" +} diff --git a/setup.py b/setup.py index 436a977..36c84c7 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,4 @@ -""" -Setup script for xtotext package. -""" +"""Setup for Mill's compatible xtotext Python distribution.""" from setuptools import setup, find_packages @@ -10,15 +8,14 @@ setup( name="xtotext", version="1.0.0", - author="xtotext Team", - description="AI-Ready Document Conversion System", + author="Celladore", + description="Mill document conversion compatibility package", long_description=long_description, long_description_content_type="text/markdown", packages=find_packages(), classifiers=[ - "Development Status :: 4 - Beta", + "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", @@ -28,7 +25,6 @@ ], python_requires=">=3.8", install_requires=[ - "pathlib", "Pillow>=8.0.0", "python-docx>=0.8.11", "beautifulsoup4>=4.9.0", @@ -57,4 +53,4 @@ "xtotext=xtox.cli.main:main", ], }, -) \ No newline at end of file +) diff --git a/tests/test_md_to_pdf.py b/tests/test_md_to_pdf.py index a87fa25..b36126e 100644 --- a/tests/test_md_to_pdf.py +++ b/tests/test_md_to_pdf.py @@ -59,6 +59,8 @@ def hello_world(): assert "\\subsection{Section 1}" in latex_content assert "\\begin{itemize}" in latex_content assert "\\begin{lstlisting}[language=python]" in latex_content + assert "\\section{Test Document}\n\n" in latex_content + assert "\\begin{itemize}\n\\item Item 1\n" in latex_content @pytest.mark.skipif( @@ -87,4 +89,4 @@ def test_full_conversion_workflow(tmp_path): # Check that the output files were created assert os.path.exists(result["latex_path"]) - assert os.path.exists(result["pdf_path"]) \ No newline at end of file + assert os.path.exists(result["pdf_path"]) diff --git a/xtox/__init__.py b/xtox/__init__.py new file mode 100644 index 0000000..4cf591c --- /dev/null +++ b/xtox/__init__.py @@ -0,0 +1,13 @@ +"""Compatibility namespace for the historical xtox Python import.""" + +from .core import DocumentConverter, ImageConverter, MultiDocumentProcessor +from .workflows import process_markdown_to_docx, process_markdown_to_pdf + +__version__ = "1.0.0" +__all__ = [ + "DocumentConverter", + "ImageConverter", + "MultiDocumentProcessor", + "process_markdown_to_docx", + "process_markdown_to_pdf", +] diff --git a/xtox/cli/__init__.py b/xtox/cli/__init__.py new file mode 100644 index 0000000..9ebf5fc --- /dev/null +++ b/xtox/cli/__init__.py @@ -0,0 +1 @@ +"""Compatibility CLI package retained for the xtotext entry point.""" diff --git a/xtox/cli/main.py b/xtox/cli/main.py new file mode 100644 index 0000000..c42654e --- /dev/null +++ b/xtox/cli/main.py @@ -0,0 +1,9 @@ +"""Compatibility bridge for xtox.cli.main.""" + +from cli.main import main + +__all__ = ["main"] + + +if __name__ == "__main__": + main() diff --git a/xtox/core/__init__.py b/xtox/core/__init__.py new file mode 100644 index 0000000..5284d83 --- /dev/null +++ b/xtox/core/__init__.py @@ -0,0 +1,29 @@ +"""Compatibility bridge from xtox.core to Mill's existing core package.""" + +from pathlib import Path + +__path__ = [str(Path(__file__).resolve().parents[2] / "core")] + +from .document_converter import DocumentConverter # noqa: E402 +from .html_to_markdown import convert_html_to_markdown # noqa: E402 +from .image_converter import ImageConverter # noqa: E402 +from .interactive_processor import InteractiveProcessor # noqa: E402 +from .latex_to_pdf import check_latex_structure, fix_latex_structure, latex_to_pdf # noqa: E402 +from .markdown_to_docx import convert_markdown_to_docx # noqa: E402 +from .markdown_to_html import convert_markdown_to_html # noqa: E402 +from .markdown_to_latex import convert_markdown_to_latex # noqa: E402 +from .multi_document_processor import MultiDocumentProcessor # noqa: E402 + +__all__ = [ + "DocumentConverter", + "ImageConverter", + "InteractiveProcessor", + "MultiDocumentProcessor", + "check_latex_structure", + "convert_html_to_markdown", + "convert_markdown_to_docx", + "convert_markdown_to_html", + "convert_markdown_to_latex", + "fix_latex_structure", + "latex_to_pdf", +] diff --git a/xtox/utils/__init__.py b/xtox/utils/__init__.py new file mode 100644 index 0000000..8f3bff6 --- /dev/null +++ b/xtox/utils/__init__.py @@ -0,0 +1,5 @@ +"""Compatibility bridge from xtox.utils to the existing utilities.""" + +from pathlib import Path + +__path__ = [str(Path(__file__).resolve().parents[2] / "utils")] diff --git a/xtox/workflows/__init__.py b/xtox/workflows/__init__.py new file mode 100644 index 0000000..58ae437 --- /dev/null +++ b/xtox/workflows/__init__.py @@ -0,0 +1,10 @@ +"""Compatibility bridge from xtox.workflows to workflows.""" + +from pathlib import Path + +__path__ = [str(Path(__file__).resolve().parents[2] / "workflows")] + +from .md_to_docx import process_markdown_to_docx # noqa: E402 +from .md_to_pdf import process_markdown_to_pdf # noqa: E402 + +__all__ = ["process_markdown_to_docx", "process_markdown_to_pdf"]