diff --git a/CHANGELOG.md b/CHANGELOG.md index 014bff3d..18c32d15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ version. A separate release pull request bumps the manifests with version and date. Earlier releases are described in their [GitHub releases](https://github.com/firecrawl/pdf-inspector/releases). +## [Unreleased] + +### Added + +- `PdfOptions::best_effort` (`best_effort=True` in the Python `process_pdf` + and `process_pdf_bytes`) returns whatever native text can be read, even + where the library is not confident in it. A Scanned or ImageBased + document, whose text layer may still hold real or invisible text, + otherwise returned no Markdown, as did a Mixed or TextBased one whose + text was judged garbled or undecodable. With the option set, extraction + runs anyway, and a TextBased page whose text reads as garbage is kept + too. Detection still sets the OCR flags (`pdf_type`, `pages_needing_ocr`, + `ocr_reasons_by_page`), so a caller can tell the text may be unreliable. + The new `PdfProcessResult::text_unreliable` (also on the Python result) + is `true` when the Markdown is text the default would have withheld. Off + by default. + ## [1.25.2] - 2026-09-28 Changes since 1.25.1. diff --git a/pdf_inspector.pyi b/pdf_inspector.pyi index 550af20a..19d64d69 100644 --- a/pdf_inspector.pyi +++ b/pdf_inspector.pyi @@ -40,6 +40,9 @@ class PdfResult: pages_with_tables: list[int] pages_with_columns: list[int] has_encoding_issues: bool + text_unreliable: bool + """True when ``markdown`` is text that ``best_effort=True`` kept and the + default would have withheld. Always False without ``best_effort``.""" cmap_gaps: list["FontCMapGaps"] """Fonts whose ToUnicode CMap (or, for a font without one, the embedded program's cmap table) lacked an entry for a code the document shows @@ -304,12 +307,22 @@ class PagesExtractionResult: is_complex: bool """True if any page has tables or multi-column layout.""" -def process_pdf(path: str, pages: Optional[list[int]] = None) -> PdfResult: - """Process a PDF: detect type, extract text, convert to Markdown.""" +def process_pdf( + path: str, pages: Optional[list[int]] = None, best_effort: bool = False +) -> PdfResult: + """Process a PDF: detect type, extract text, convert to Markdown. + + ``best_effort`` returns whatever native text can be read even where the + library is not confident in it (scanned or image-based document, garbled + text, undecodable fonts) instead of none; ``pdf_type`` and + ``pages_needing_ocr`` still report the flag, and ``text_unreliable`` is + True when the markdown is text kept this way.""" ... -def process_pdf_bytes(data: bytes, pages: Optional[list[int]] = None) -> PdfResult: - """Process a PDF from bytes in memory.""" +def process_pdf_bytes( + data: bytes, pages: Optional[list[int]] = None, best_effort: bool = False +) -> PdfResult: + """Process a PDF from bytes in memory. See ``process_pdf`` for ``best_effort``.""" ... def process_pdf_with_ocr( diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs index 8fb01fd5..0822be5e 100644 --- a/src/bin/pdf2md.rs +++ b/src/bin/pdf2md.rs @@ -408,6 +408,7 @@ mod tests { confidence: 1.0, layout: pdf_inspector::LayoutComplexity::default(), has_encoding_issues: false, + text_unreliable: false, cmap_gaps: Vec::new(), }; assert_eq!( diff --git a/src/lib.rs b/src/lib.rs index 35037922..c325ea10 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -225,6 +225,11 @@ pub struct PdfProcessResult { /// `true` when broken font encodings are detected (garbled text, /// replacement characters). Clients should fall back to OCR. pub has_encoding_issues: bool, + /// `true` when `markdown` is text that [`PdfOptions::best_effort`] kept + /// and the default would have withheld: the document was classified + /// Scanned or ImageBased, or its text was judged garbled or its fonts + /// undecodable. Always `false` without `best_effort`. + pub text_unreliable: bool, /// The fonts whose ToUnicode CMap — or, for a font without one, the /// embedded program's cmap table — lacked an entry for a code the /// document shows through it, with the counts of codes shown, read from @@ -262,6 +267,11 @@ pub struct PdfOptions { /// Password for decrypting an encrypted PDF. `None` falls back to the /// empty password (owner-only encryption). pub password: Option, + /// Return whatever native text can be read, even where the library is + /// not confident in it (default: `false`). The flags (`pdf_type`, + /// `pages_needing_ocr`, `ocr_reasons_by_page`) are reported as usual, and + /// [`PdfProcessResult::text_unreliable`] marks text kept by this option. + pub best_effort: bool, } // Manual `Debug` so the password is never leaked through debug logging or a @@ -274,6 +284,7 @@ impl std::fmt::Debug for PdfOptions { .field("markdown", &self.markdown) .field("page_filter", &self.page_filter) .field("password", &self.password.as_ref().map(|_| "[REDACTED]")) + .field("best_effort", &self.best_effort) .finish() } } @@ -286,6 +297,7 @@ impl Default for PdfOptions { markdown: MarkdownOptions::default(), page_filter: None, password: None, + best_effort: false, } } } @@ -328,6 +340,21 @@ impl PdfOptions { self } + /// Return whatever native text can be read, even where the library is + /// not confident in it. + /// + /// By default a Scanned or ImageBased document yields no markdown, and + /// the markdown of a Mixed or TextBased document is dropped when it is + /// garbled or its fonts are undecodable. With this set, extraction runs + /// anyway (including a scanned page's invisible text layer) and whatever + /// it reads is returned. Detection still sets `pdf_type` and the OCR + /// page flags, and [`PdfProcessResult::text_unreliable`] says when the + /// markdown is text the default would have withheld. + pub fn best_effort(mut self, best_effort: bool) -> Self { + self.best_effort = best_effort; + self + } + /// Set the password used to decrypt an encrypted PDF. pub fn password(mut self, password: impl Into) -> Self { self.password = Some(password.into()); @@ -4615,12 +4642,14 @@ fn process_document( confidence, layout: LayoutComplexity::default(), has_encoding_issues: false, + text_unreliable: false, cmap_gaps: Vec::new(), }); } - // Scanned / ImageBased → nothing to extract - if matches!(pdf_type, PdfType::Scanned | PdfType::ImageBased) { + // Scanned / ImageBased → nothing to extract, unless extraction is forced + let scanned_type = matches!(pdf_type, PdfType::Scanned | PdfType::ImageBased); + if scanned_type && !options.best_effort { return Ok(PdfProcessResult { pdf_type, markdown: None, @@ -4639,10 +4668,18 @@ fn process_document( confidence, layout: LayoutComplexity::default(), has_encoding_issues: false, + text_unreliable: false, cmap_gaps: Vec::new(), }); } + // Mixed PDFs, and Scanned/ImageBased ones whose extraction is forced, are + // backed by images: their text layer may be absent or invisible. + let image_backed = pdf_type == PdfType::Mixed || scanned_type; + // A Scanned or ImageBased document reaches extraction only under + // `best_effort`, so whatever it yields is text the default withholds. + let mut text_unreliable = scanned_type; + // Step 2 — Extraction (reuses the already-loaded document) let extracted = { let font_cmaps = FontCMaps::from_doc(&doc); @@ -4658,7 +4695,7 @@ fn process_document( // For Mixed/template PDFs: if normal extraction produces garbage text // (mostly non-alphanumeric), retry with invisible (Tr=3) text included. // This unlocks OCR text layers behind scanned images. - if pdf_type == PdfType::Mixed { + if image_backed { if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _, _, _, _)| e) { let sample: String = items .iter() @@ -4694,7 +4731,7 @@ fn process_document( }; // For Mixed PDFs, extraction failure is non-fatal - let extracted = if pdf_type == PdfType::Mixed { + let extracted = if image_backed { extracted.ok() } else { Some(extracted?) @@ -4773,8 +4810,9 @@ fn process_document( (items, rects, lines) } else { log::debug!( - "suppressing garbage text from OCR-flagged pages: {:?}", - garbage_pages + "garbage text on OCR-flagged pages {:?} (kept: {})", + garbage_pages, + options.best_effort ); for page in &garbage_pages { add_ocr_reason( @@ -4783,17 +4821,25 @@ fn process_document( OCR_REASON_SUSPECTED_GARBLED_TEXT, ); } + // `best_effort` keeps the text of these pages; the + // reason above still says it is suspect. + text_unreliable |= options.best_effort; + let dropped: std::collections::HashSet = if options.best_effort { + std::collections::HashSet::new() + } else { + garbage_pages + }; let items: Vec<_> = items .into_iter() - .filter(|i| !garbage_pages.contains(&i.page)) + .filter(|i| !dropped.contains(&i.page)) .collect(); let rects: Vec<_> = rects .into_iter() - .filter(|r| !garbage_pages.contains(&r.page)) + .filter(|r| !dropped.contains(&r.page)) .collect(); let lines: Vec<_> = lines .into_iter() - .filter(|l| !garbage_pages.contains(&l.page)) + .filter(|l| !dropped.contains(&l.page)) .collect(); (items, rects, lines) } @@ -4893,7 +4939,12 @@ fn process_document( // layer comes from a bad OCR pass, and callers should use proper OCR. let (pdf_type, markdown, confidence) = if pdf_type == PdfType::Mixed && markdown.as_ref().is_some_and(|m| is_garbage_text(m)) { - (PdfType::Scanned, None, 0.95) + text_unreliable |= options.best_effort; + ( + PdfType::Scanned, + if options.best_effort { markdown } else { None }, + 0.95, + ) } else { (pdf_type, markdown, confidence) }; @@ -4905,7 +4956,12 @@ fn process_document( && markdown.as_ref().is_some_and(|m| is_garbage_text(m)) { log::debug!("TextBased PDF has garbage text — flagging all pages for OCR"); - (None, true, true) + text_unreliable |= options.best_effort; + ( + if options.best_effort { markdown } else { None }, + true, + true, + ) } else { (markdown, has_encoding_issues, false) }; @@ -4961,7 +5017,8 @@ fn process_document( } } - let markdown = if all_gid { + text_unreliable |= all_gid && options.best_effort; + let markdown = if all_gid && !options.best_effort { log::debug!( "all {} pages have gid-encoded fonts — suppressing markdown output", page_count @@ -4971,6 +5028,10 @@ fn process_document( markdown }; + // Only text that is actually returned can be unreliable. + let text_unreliable = + text_unreliable && markdown.as_ref().is_some_and(|m| !m.trim().is_empty()); + Ok(PdfProcessResult { pdf_type, markdown, @@ -4996,6 +5057,7 @@ fn process_document( confidence, layout, has_encoding_issues, + text_unreliable, cmap_gaps, }) } diff --git a/src/python.rs b/src/python.rs index b52033fd..568b149a 100644 --- a/src/python.rs +++ b/src/python.rs @@ -18,7 +18,8 @@ pub struct PyPdfResult { /// The detected PDF type: "text_based", "scanned", "image_based", or "mixed". #[pyo3(get)] pub pdf_type: String, - /// Markdown output (None if detect-only or scanned PDF). + /// Markdown output (None if detect-only, or if the text was withheld as + /// scanned or unreliable and best_effort was not set). #[pyo3(get)] pub markdown: Option, /// Total number of pages. @@ -78,6 +79,11 @@ pub struct PyPdfResult { /// Whether encoding issues were detected. #[pyo3(get)] pub has_encoding_issues: bool, + /// True when `markdown` is text that `best_effort=True` kept and the + /// default would have withheld (scanned or image-based document, garbled + /// text, undecodable fonts). Always False without `best_effort`. + #[pyo3(get)] + pub text_unreliable: bool, /// Fonts whose ToUnicode CMap — or, for a font without one, the embedded /// program's cmap table — lacked an entry for a code the document shows /// through it, with the counts of codes shown, read from their neighbours @@ -597,6 +603,7 @@ fn to_py_result(r: crate::PdfProcessResult) -> PyPdfResult { pages_with_tables: r.layout.pages_with_tables, pages_with_columns: r.layout.pages_with_columns, has_encoding_issues: r.has_encoding_issues, + text_unreliable: r.text_unreliable, cmap_gaps: to_py_font_cmap_gaps(r.cmap_gaps), } } @@ -848,10 +855,14 @@ fn convert_region_results(results: Vec) -> Vec>) -> PyResult { - let mut opts = crate::PdfOptions::new(); +#[pyo3(signature = (path, pages=None, best_effort=false))] +fn process_pdf(path: &str, pages: Option>, best_effort: bool) -> PyResult { + let mut opts = crate::PdfOptions::new().best_effort(best_effort); if let Some(p) = pages { opts = opts.pages(p); } @@ -861,9 +872,13 @@ fn process_pdf(path: &str, pages: Option>) -> PyResult { /// Process a PDF from bytes in memory. #[pyfunction] -#[pyo3(signature = (data, pages=None))] -fn process_pdf_bytes(data: &[u8], pages: Option>) -> PyResult { - let mut opts = crate::PdfOptions::new(); +#[pyo3(signature = (data, pages=None, best_effort=false))] +fn process_pdf_bytes( + data: &[u8], + pages: Option>, + best_effort: bool, +) -> PyResult { + let mut opts = crate::PdfOptions::new().best_effort(best_effort); if let Some(p) = pages { opts = opts.pages(p); } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index c12e6861..992f9d23 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -2141,6 +2141,7 @@ fn test_pages_needing_ocr_field_accessible() { confidence: 1.0, layout: pdf_inspector::LayoutComplexity::default(), has_encoding_issues: false, + text_unreliable: false, cmap_gaps: Vec::new(), }; assert_eq!(process_result.pages_needing_ocr, vec![1, 3]); @@ -3196,6 +3197,28 @@ fn test_inline_image_raster_covers_the_page() { assert_eq!(processed.pdf_type, PdfType::Scanned); } +/// `best_effort` returns the text layer behind a scan that the default +/// withholds, leaves the OCR flags as they were, and marks the text unreliable. +#[test] +fn test_best_effort_keeps_scanned_text() { + let buf = make_pdf_with_glyph_layer(&[SCAN_WITH_INVISIBLE_LAYER]); + + let default = process_pdf_mem(&buf).unwrap(); + assert!(matches!( + default.pdf_type, + PdfType::Scanned | PdfType::ImageBased + )); + assert!(default.markdown.is_none()); + assert!(!default.text_unreliable); + + let forced = process_pdf_mem_with_options(&buf, PdfOptions::new().best_effort(true)).unwrap(); + assert_eq!(forced.pdf_type, default.pdf_type); + assert_eq!(forced.pages_needing_ocr, default.pages_needing_ocr); + assert_eq!(forced.ocr_reasons_by_page, default.ocr_reasons_by_page); + assert!(forced.text_unreliable); + assert!(forced.markdown.is_some_and(|m| !m.trim().is_empty())); +} + /// A scan tiled into two thousand strips — image XObjects or inline /// images — covers the page as one draw does: the hidden layer over it /// is a layer nobody sees.