77 bin/mail/import --since 2026-01-01 only messages after a date
88 bin/mail/import --limit 50 cap messages per run
99 bin/mail/import --no-attachments body only, skip attachment conversion
10- bin/mail/import --ocr OCR scanned PDFs/images via docling
10+ bin/mail/import --ocr OCR images ( PDFs OCR when textless)
1111 bin/mail/import --dry-run list messages without writing anything
1212
1313Writes one directory per message: var/mail/{folder}/{message_id}/
@@ -16,10 +16,11 @@ Writes one directory per message: var/mail/{folder}/{message_id}/
1616 attachments/*.md converted attachment content
1717
1818Indexing is a separate step (`bin/brain/index.go --rebuild`): conversion can
19- crash in native docling and must not leave the brain DB mid-transaction.
19+ crash and must not leave the brain DB mid-transaction.
2020
21- Requires ONLYOFFICE_URL/USER/PASS in .env (or env). Idempotent: a message
22- already present (message.md exists) is skipped unless --force.
21+ Requires ONLYOFFICE_URL/USER/PASS in .env (or env) except `--from-raw`.
22+ Idempotent: a message already present (message.md exists) is skipped unless
23+ --force.
2324"""
2425from __future__ import annotations
2526
@@ -41,10 +42,10 @@ from mailconv import ( # noqa: E402
4142 IMAGE_SUFFIXES ,
4243 LEGACY_OFFICE_SUFFIXES ,
4344 TEXT_SUFFIXES ,
45+ convert_pdf ,
4446 html_to_markdown ,
45- is_convertible ,
4647 normalize_markdown ,
47- subject_to_filename ,
48+ ocr_image ,
4849 zip_extract_safe ,
4950)
5051
@@ -147,77 +148,16 @@ def convert_file_to_md(path: Path, ocr: bool) -> str | None:
147148 except Exception as e :
148149 return f"\n <!-- conversion failed: { e } \n "
149150 if suffix == ".pdf" :
150- return _convert_pdf (path , ocr )
151+ return convert_pdf (path , ocr )
151152 if suffix in IMAGE_SUFFIXES and ocr :
152- return _convert_pdf (path , ocr )
153+ return ocr_image (path ) or " \n <!-- ocr unavailable --> \n "
153154 if suffix in LEGACY_OFFICE_SUFFIXES :
154155 return _convert_legacy (path )
155156 if suffix in ARCHIVE_SUFFIXES :
156157 return None # handled by caller (unpack + recurse)
157158 return None
158159
159160
160- def _convert_pdf (path : Path , ocr : bool ) -> str :
161- """Convert one PDF to markdown.
162-
163- Fast path: poppler's pdftotext (-layout) extracts exact text from
164- born-digital PDFs in ~15ms vs docling's 1-3s. Only textless PDFs (scanned
165- pages, layout-heavy) fall back to docling, which runs isolated in a
166- subprocess because its native onnx/RT-DETR has segfaulted the main process.
167- """
168- text = _pdf_fast_text (path )
169- if ocr or text is None or not text .strip ():
170- return _convert_pdf_docling (path , ocr )
171- return normalize_markdown (text )
172-
173-
174- def _pdf_fast_text (path : Path ) -> str | None :
175- """pdftotext -layout; None when poppler is unavailable (or the PDF has no text layer)."""
176- try :
177- proc = subprocess .run (
178- ["pdftotext" , "-layout" , str (path ), "-" ],
179- capture_output = True , timeout = 60 )
180- except (OSError , subprocess .TimeoutExpired ):
181- return None
182- if proc .returncode != 0 :
183- return None
184- return proc .stdout .decode ("utf-8" , errors = "replace" )
185-
186-
187- def _convert_pdf_docling (path : Path , ocr : bool ) -> str :
188- try :
189- proc = subprocess .run (
190- [sys .executable , os .path .abspath (__file__ ), "--pdf-worker" , str (path ),
191- "--ocr" if ocr else "--no-ocr" ],
192- capture_output = True , text = True , timeout = 600 )
193- except subprocess .TimeoutExpired :
194- return "\n <!-- pdf conversion timed out -->\n "
195- if proc .returncode != 0 :
196- tail = proc .stderr .strip ().splitlines ()[- 3 :]
197- return f"\n <!-- pdf conversion failed: { proc .returncode } { ' | ' .join (tail )} \n "
198- return proc .stdout
199-
200-
201- def _pdf_worker (path : Path , ocr : bool ) -> None :
202- """docling worker entry: prints converted markdown on stdout, exits non-zero on error."""
203- try :
204- from docling .document_converter import DocumentConverter , PdfFormatOption
205- from docling .datamodel .pipeline_options import PdfPipelineOptions
206- opts = PdfPipelineOptions ()
207- opts .do_ocr = bool (ocr )
208- opts .do_table_structure = True
209- conv = DocumentConverter (format_options = {"pdf" : PdfFormatOption (pipeline_options = opts )})
210- res = conv .convert (str (path ))
211- sys .stdout .write (normalize_markdown (res .document .export_to_markdown ()))
212- sys .exit (0 )
213- except Exception as e :
214- # errors/stacktraces to stderr; the caller only reports a one-liner
215- print (f"pdf-worker: { e } , file = sys .stderr )
216- import traceback
217- traceback .print_exc (file = sys .stderr )
218- sys .exit (1 )
219-
220-
221161def _convert_legacy (path : Path ) -> str :
222162 """Legacy .doc/.xls/.ppt -> md via pandoc (installed) or a stub."""
223163 try :
@@ -356,19 +296,12 @@ def main(argv: list[str]) -> int:
356296 p .add_argument ("--from-raw" , default = "" ,
357297 help = "convert Go-synced dirs (var/mail/<folder>/<id>/message.json) to markdown" )
358298 p .add_argument ("--no-attachments" , action = "store_true" , help = "skip attachment download+convert" )
359- p .add_argument ("--ocr" , action = "store_true" , help = "OCR scanned PDFs/images via docling " )
299+ p .add_argument ("--ocr" , action = "store_true" , help = "OCR images ( PDFs OCR when textless) " )
360300 p .add_argument ("--force" , action = "store_true" , help = "re-import even if message.md exists" )
361301 p .add_argument ("--dry-run" , action = "store_true" , help = "list messages, write nothing" )
362302 p .add_argument ("--json" , action = "store_true" )
363- p .add_argument ("--pdf-worker" , default = "" , help = argparse .SUPPRESS )
364- p .add_argument ("--no-ocr" , action = "store_true" , help = argparse .SUPPRESS )
365303 a = p .parse_args (argv )
366304
367- if a .pdf_worker :
368- _pdf_worker (Path (a .pdf_worker ), ocr = not a .no_ocr )
369- return 0
370-
371- conf = load_env ()
372305 fid = folder_id (a .folder )
373306 out_root = ROOT / "var" / "mail"
374307 summary : list [dict ] = []
@@ -394,6 +327,7 @@ def main(argv: list[str]) -> int:
394327 target_dir = msg_dir .parent ))
395328 summary .append (entry )
396329 else :
330+ conf = load_env ()
397331 OOCLIENT = OOClient (conf )
398332 if a .id :
399333 messages = [{"id" : i } for i in a .id ]
0 commit comments