From 2fa731ab76ade69ab548563d62da463f2762bf8d Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:20:02 -0700 Subject: [PATCH 1/7] feat: add force_extraction option to keep the text of pages flagged for OCR A Scanned or ImageBased document returned no Markdown even when it carries a readable (or invisible) text layer, and a Mixed or TextBased document lost its Markdown when the text was judged garbled or its fonts undecodable. PdfOptions::force_extraction(true), and force_extraction=True in the Python process_pdf and process_pdf_bytes, runs extraction anyway and returns what it reads. pdf_type and the OCR page flags are reported as before, so a caller still knows the text may be unreliable. Off by default. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 14 ++++++++++ pdf_inspector.pyi | 16 ++++++++--- src/lib.rs | 54 +++++++++++++++++++++++++++++++++----- src/python.rs | 23 +++++++++++----- tests/integration_tests.rs | 21 +++++++++++++++ 5 files changed, 111 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 014bff3d..2aa5b4ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ version. A separate release pull request bumps the manifests with version and date. Earlier releases are described in their [GitHub releases](https://github.com/firecrawl/pdf-inspector/releases). +## [Unreleased] + +### Added + +- `PdfOptions::force_extraction` (`force_extraction=True` in the Python + `process_pdf` and `process_pdf_bytes`) keeps the text of pages flagged for + OCR instead of suppressing it. A Scanned or ImageBased document, whose + text layer may still hold real or invisible text, otherwise returned no + Markdown, as did a Mixed or TextBased one whose text was judged garbled or + undecodable. With the option set, extraction runs anyway and returns what + it reads. `pdf_type`, `pages_needing_ocr` and `ocr_reasons_by_page` are + unchanged, so a caller still sees that the text may be unreliable. Off by + default. + ## [1.25.2] - 2026-09-28 Changes since 1.25.1. diff --git a/pdf_inspector.pyi b/pdf_inspector.pyi index 550af20a..8d66928f 100644 --- a/pdf_inspector.pyi +++ b/pdf_inspector.pyi @@ -304,12 +304,20 @@ class PagesExtractionResult: is_complex: bool """True if any page has tables or multi-column layout.""" -def process_pdf(path: str, pages: Optional[list[int]] = None) -> PdfResult: - """Process a PDF: detect type, extract text, convert to Markdown.""" +def process_pdf( + path: str, pages: Optional[list[int]] = None, force_extraction: bool = False +) -> PdfResult: + """Process a PDF: detect type, extract text, convert to Markdown. + + ``force_extraction`` keeps the text of pages flagged for OCR (scanned, + garbled, undecodable fonts) instead of returning none; ``pdf_type`` and + ``pages_needing_ocr`` still report the flag.""" ... -def process_pdf_bytes(data: bytes, pages: Optional[list[int]] = None) -> PdfResult: - """Process a PDF from bytes in memory.""" +def process_pdf_bytes( + data: bytes, pages: Optional[list[int]] = None, force_extraction: bool = False +) -> PdfResult: + """Process a PDF from bytes in memory. See ``process_pdf`` for ``force_extraction``.""" ... def process_pdf_with_ocr( diff --git a/src/lib.rs b/src/lib.rs index 35037922..cc676baa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -262,6 +262,11 @@ pub struct PdfOptions { /// Password for decrypting an encrypted PDF. `None` falls back to the /// empty password (owner-only encryption). pub password: Option, + /// Keep the extracted text of pages flagged for OCR instead of + /// suppressing it (default: `false`). The flags (`pdf_type`, + /// `pages_needing_ocr`, `ocr_reasons_by_page`) are reported as usual, so + /// the caller still knows the text may be incomplete or unreliable. + pub force_extraction: bool, } // Manual `Debug` so the password is never leaked through debug logging or a @@ -274,6 +279,7 @@ impl std::fmt::Debug for PdfOptions { .field("markdown", &self.markdown) .field("page_filter", &self.page_filter) .field("password", &self.password.as_ref().map(|_| "[REDACTED]")) + .field("force_extraction", &self.force_extraction) .finish() } } @@ -286,6 +292,7 @@ impl Default for PdfOptions { markdown: MarkdownOptions::default(), page_filter: None, password: None, + force_extraction: false, } } } @@ -328,6 +335,18 @@ impl PdfOptions { self } + /// Keep the text of pages flagged for OCR rather than returning none. + /// + /// By default a Scanned or ImageBased document yields no markdown, and + /// the markdown of a Mixed or TextBased document is dropped when it is + /// garbled or its fonts are undecodable. With this set, extraction runs + /// anyway (including a scanned page's invisible text layer) and whatever + /// it reads is returned; `pdf_type` and the OCR page flags are unchanged. + pub fn force_extraction(mut self, force: bool) -> Self { + self.force_extraction = force; + self + } + /// Set the password used to decrypt an encrypted PDF. pub fn password(mut self, password: impl Into) -> Self { self.password = Some(password.into()); @@ -4619,8 +4638,9 @@ fn process_document( }); } - // Scanned / ImageBased → nothing to extract - if matches!(pdf_type, PdfType::Scanned | PdfType::ImageBased) { + // Scanned / ImageBased → nothing to extract, unless extraction is forced + let scanned_type = matches!(pdf_type, PdfType::Scanned | PdfType::ImageBased); + if scanned_type && !options.force_extraction { return Ok(PdfProcessResult { pdf_type, markdown: None, @@ -4643,6 +4663,10 @@ fn process_document( }); } + // Mixed PDFs, and Scanned/ImageBased ones whose extraction is forced, are + // backed by images: their text layer may be absent or invisible. + let image_backed = pdf_type == PdfType::Mixed || scanned_type; + // Step 2 — Extraction (reuses the already-loaded document) let extracted = { let font_cmaps = FontCMaps::from_doc(&doc); @@ -4658,7 +4682,7 @@ fn process_document( // For Mixed/template PDFs: if normal extraction produces garbage text // (mostly non-alphanumeric), retry with invisible (Tr=3) text included. // This unlocks OCR text layers behind scanned images. - if pdf_type == PdfType::Mixed { + if image_backed { if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _, _, _, _)| e) { let sample: String = items .iter() @@ -4694,7 +4718,7 @@ fn process_document( }; // For Mixed PDFs, extraction failure is non-fatal - let extracted = if pdf_type == PdfType::Mixed { + let extracted = if image_backed { extracted.ok() } else { Some(extracted?) @@ -4893,7 +4917,15 @@ fn process_document( // layer comes from a bad OCR pass, and callers should use proper OCR. let (pdf_type, markdown, confidence) = if pdf_type == PdfType::Mixed && markdown.as_ref().is_some_and(|m| is_garbage_text(m)) { - (PdfType::Scanned, None, 0.95) + ( + PdfType::Scanned, + if options.force_extraction { + markdown + } else { + None + }, + 0.95, + ) } else { (pdf_type, markdown, confidence) }; @@ -4905,7 +4937,15 @@ fn process_document( && markdown.as_ref().is_some_and(|m| is_garbage_text(m)) { log::debug!("TextBased PDF has garbage text — flagging all pages for OCR"); - (None, true, true) + ( + if options.force_extraction { + markdown + } else { + None + }, + true, + true, + ) } else { (markdown, has_encoding_issues, false) }; @@ -4961,7 +5001,7 @@ fn process_document( } } - let markdown = if all_gid { + let markdown = if all_gid && !options.force_extraction { log::debug!( "all {} pages have gid-encoded fonts — suppressing markdown output", page_count diff --git a/src/python.rs b/src/python.rs index b52033fd..3b10724c 100644 --- a/src/python.rs +++ b/src/python.rs @@ -848,10 +848,17 @@ fn convert_region_results(results: Vec) -> Vec>) -> PyResult { - let mut opts = crate::PdfOptions::new(); +#[pyo3(signature = (path, pages=None, force_extraction=false))] +fn process_pdf( + path: &str, + pages: Option>, + force_extraction: bool, +) -> PyResult { + let mut opts = crate::PdfOptions::new().force_extraction(force_extraction); if let Some(p) = pages { opts = opts.pages(p); } @@ -861,9 +868,13 @@ fn process_pdf(path: &str, pages: Option>) -> PyResult { /// Process a PDF from bytes in memory. #[pyfunction] -#[pyo3(signature = (data, pages=None))] -fn process_pdf_bytes(data: &[u8], pages: Option>) -> PyResult { - let mut opts = crate::PdfOptions::new(); +#[pyo3(signature = (data, pages=None, force_extraction=false))] +fn process_pdf_bytes( + data: &[u8], + pages: Option>, + force_extraction: bool, +) -> PyResult { + let mut opts = crate::PdfOptions::new().force_extraction(force_extraction); if let Some(p) = pages { opts = opts.pages(p); } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index c12e6861..a6bb915a 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -3196,6 +3196,27 @@ fn test_inline_image_raster_covers_the_page() { assert_eq!(processed.pdf_type, PdfType::Scanned); } +/// `force_extraction` keeps the text layer behind a scan that the default +/// withholds, and leaves the OCR flags as they were. +#[test] +fn test_force_extraction_keeps_scanned_text() { + let buf = make_pdf_with_glyph_layer(&[SCAN_WITH_INVISIBLE_LAYER]); + + let default = process_pdf_mem(&buf).unwrap(); + assert!(matches!( + default.pdf_type, + PdfType::Scanned | PdfType::ImageBased + )); + assert!(default.markdown.is_none()); + + let forced = + process_pdf_mem_with_options(&buf, PdfOptions::new().force_extraction(true)).unwrap(); + assert_eq!(forced.pdf_type, default.pdf_type); + assert_eq!(forced.pages_needing_ocr, default.pages_needing_ocr); + assert_eq!(forced.ocr_reasons_by_page, default.ocr_reasons_by_page); + assert!(forced.markdown.is_some_and(|m| !m.trim().is_empty())); +} + /// A scan tiled into two thousand strips — image XObjects or inline /// images — covers the page as one draw does: the hidden layer over it /// is a layer nobody sees. From a9bf79907686ad8a354ee9dc6f7e60da65d54ed6 Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:33:52 -0700 Subject: [PATCH 2/7] refactor: rename force_extraction to best_effort and report text_unreliable The option changes what the library withholds, not what the text is, so it is named for that. The result gains text_unreliable, true when the markdown is text the default would have withheld, so a caller no longer has to infer it from pages_needing_ocr. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 19 ++++++------ pdf_inspector.pyi | 17 +++++++---- src/bin/pdf2md.rs | 1 + src/lib.rs | 59 +++++++++++++++++++++++--------------- src/python.rs | 27 +++++++++-------- tests/integration_tests.rs | 12 ++++---- 6 files changed, 80 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2aa5b4ea..00ef97eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,15 +11,16 @@ version and date. Earlier releases are described in their ### Added -- `PdfOptions::force_extraction` (`force_extraction=True` in the Python - `process_pdf` and `process_pdf_bytes`) keeps the text of pages flagged for - OCR instead of suppressing it. A Scanned or ImageBased document, whose - text layer may still hold real or invisible text, otherwise returned no - Markdown, as did a Mixed or TextBased one whose text was judged garbled or - undecodable. With the option set, extraction runs anyway and returns what - it reads. `pdf_type`, `pages_needing_ocr` and `ocr_reasons_by_page` are - unchanged, so a caller still sees that the text may be unreliable. Off by - default. +- `PdfOptions::best_effort` (`best_effort=True` in the Python `process_pdf` + and `process_pdf_bytes`) returns whatever native text can be read, even + where the library is not confident in it. A Scanned or ImageBased + document, whose text layer may still hold real or invisible text, + otherwise returned no Markdown, as did a Mixed or TextBased one whose + text was judged garbled or undecodable. With the option set, extraction + runs anyway. `pdf_type`, `pages_needing_ocr` and `ocr_reasons_by_page` + are unchanged, and the new `PdfProcessResult::text_unreliable` (also on + the Python result) is `true` when the Markdown is text the default would + have withheld. Off by default. ## [1.25.2] - 2026-09-28 diff --git a/pdf_inspector.pyi b/pdf_inspector.pyi index 8d66928f..19d64d69 100644 --- a/pdf_inspector.pyi +++ b/pdf_inspector.pyi @@ -40,6 +40,9 @@ class PdfResult: pages_with_tables: list[int] pages_with_columns: list[int] has_encoding_issues: bool + text_unreliable: bool + """True when ``markdown`` is text that ``best_effort=True`` kept and the + default would have withheld. Always False without ``best_effort``.""" cmap_gaps: list["FontCMapGaps"] """Fonts whose ToUnicode CMap (or, for a font without one, the embedded program's cmap table) lacked an entry for a code the document shows @@ -305,19 +308,21 @@ class PagesExtractionResult: """True if any page has tables or multi-column layout.""" def process_pdf( - path: str, pages: Optional[list[int]] = None, force_extraction: bool = False + path: str, pages: Optional[list[int]] = None, best_effort: bool = False ) -> PdfResult: """Process a PDF: detect type, extract text, convert to Markdown. - ``force_extraction`` keeps the text of pages flagged for OCR (scanned, - garbled, undecodable fonts) instead of returning none; ``pdf_type`` and - ``pages_needing_ocr`` still report the flag.""" + ``best_effort`` returns whatever native text can be read even where the + library is not confident in it (scanned or image-based document, garbled + text, undecodable fonts) instead of none; ``pdf_type`` and + ``pages_needing_ocr`` still report the flag, and ``text_unreliable`` is + True when the markdown is text kept this way.""" ... def process_pdf_bytes( - data: bytes, pages: Optional[list[int]] = None, force_extraction: bool = False + data: bytes, pages: Optional[list[int]] = None, best_effort: bool = False ) -> PdfResult: - """Process a PDF from bytes in memory. See ``process_pdf`` for ``force_extraction``.""" + """Process a PDF from bytes in memory. See ``process_pdf`` for ``best_effort``.""" ... def process_pdf_with_ocr( diff --git a/src/bin/pdf2md.rs b/src/bin/pdf2md.rs index 8fb01fd5..0822be5e 100644 --- a/src/bin/pdf2md.rs +++ b/src/bin/pdf2md.rs @@ -408,6 +408,7 @@ mod tests { confidence: 1.0, layout: pdf_inspector::LayoutComplexity::default(), has_encoding_issues: false, + text_unreliable: false, cmap_gaps: Vec::new(), }; assert_eq!( diff --git a/src/lib.rs b/src/lib.rs index cc676baa..92b89019 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -225,6 +225,11 @@ pub struct PdfProcessResult { /// `true` when broken font encodings are detected (garbled text, /// replacement characters). Clients should fall back to OCR. pub has_encoding_issues: bool, + /// `true` when `markdown` is text that [`PdfOptions::best_effort`] kept + /// and the default would have withheld: the document was classified + /// Scanned or ImageBased, or its text was judged garbled or its fonts + /// undecodable. Always `false` without `best_effort`. + pub text_unreliable: bool, /// The fonts whose ToUnicode CMap — or, for a font without one, the /// embedded program's cmap table — lacked an entry for a code the /// document shows through it, with the counts of codes shown, read from @@ -262,11 +267,11 @@ pub struct PdfOptions { /// Password for decrypting an encrypted PDF. `None` falls back to the /// empty password (owner-only encryption). pub password: Option, - /// Keep the extracted text of pages flagged for OCR instead of - /// suppressing it (default: `false`). The flags (`pdf_type`, - /// `pages_needing_ocr`, `ocr_reasons_by_page`) are reported as usual, so - /// the caller still knows the text may be incomplete or unreliable. - pub force_extraction: bool, + /// Return whatever native text can be read, even where the library is + /// not confident in it (default: `false`). The flags (`pdf_type`, + /// `pages_needing_ocr`, `ocr_reasons_by_page`) are reported as usual, and + /// [`PdfProcessResult::text_unreliable`] marks text kept by this option. + pub best_effort: bool, } // Manual `Debug` so the password is never leaked through debug logging or a @@ -279,7 +284,7 @@ impl std::fmt::Debug for PdfOptions { .field("markdown", &self.markdown) .field("page_filter", &self.page_filter) .field("password", &self.password.as_ref().map(|_| "[REDACTED]")) - .field("force_extraction", &self.force_extraction) + .field("best_effort", &self.best_effort) .finish() } } @@ -292,7 +297,7 @@ impl Default for PdfOptions { markdown: MarkdownOptions::default(), page_filter: None, password: None, - force_extraction: false, + best_effort: false, } } } @@ -335,15 +340,18 @@ impl PdfOptions { self } - /// Keep the text of pages flagged for OCR rather than returning none. + /// Return whatever native text can be read, even where the library is + /// not confident in it. /// /// By default a Scanned or ImageBased document yields no markdown, and /// the markdown of a Mixed or TextBased document is dropped when it is /// garbled or its fonts are undecodable. With this set, extraction runs /// anyway (including a scanned page's invisible text layer) and whatever - /// it reads is returned; `pdf_type` and the OCR page flags are unchanged. - pub fn force_extraction(mut self, force: bool) -> Self { - self.force_extraction = force; + /// it reads is returned. `pdf_type` and the OCR page flags are unchanged, + /// and [`PdfProcessResult::text_unreliable`] says when the markdown is + /// text the default would have withheld. + pub fn best_effort(mut self, best_effort: bool) -> Self { + self.best_effort = best_effort; self } @@ -4634,13 +4642,14 @@ fn process_document( confidence, layout: LayoutComplexity::default(), has_encoding_issues: false, + text_unreliable: false, cmap_gaps: Vec::new(), }); } // Scanned / ImageBased → nothing to extract, unless extraction is forced let scanned_type = matches!(pdf_type, PdfType::Scanned | PdfType::ImageBased); - if scanned_type && !options.force_extraction { + if scanned_type && !options.best_effort { return Ok(PdfProcessResult { pdf_type, markdown: None, @@ -4659,6 +4668,7 @@ fn process_document( confidence, layout: LayoutComplexity::default(), has_encoding_issues: false, + text_unreliable: false, cmap_gaps: Vec::new(), }); } @@ -4666,6 +4676,9 @@ fn process_document( // Mixed PDFs, and Scanned/ImageBased ones whose extraction is forced, are // backed by images: their text layer may be absent or invisible. let image_backed = pdf_type == PdfType::Mixed || scanned_type; + // A Scanned or ImageBased document reaches extraction only under + // `best_effort`, so whatever it yields is text the default withholds. + let mut text_unreliable = scanned_type; // Step 2 — Extraction (reuses the already-loaded document) let extracted = { @@ -4917,13 +4930,10 @@ fn process_document( // layer comes from a bad OCR pass, and callers should use proper OCR. let (pdf_type, markdown, confidence) = if pdf_type == PdfType::Mixed && markdown.as_ref().is_some_and(|m| is_garbage_text(m)) { + text_unreliable |= options.best_effort; ( PdfType::Scanned, - if options.force_extraction { - markdown - } else { - None - }, + if options.best_effort { markdown } else { None }, 0.95, ) } else { @@ -4937,12 +4947,9 @@ fn process_document( && markdown.as_ref().is_some_and(|m| is_garbage_text(m)) { log::debug!("TextBased PDF has garbage text — flagging all pages for OCR"); + text_unreliable |= options.best_effort; ( - if options.force_extraction { - markdown - } else { - None - }, + if options.best_effort { markdown } else { None }, true, true, ) @@ -5001,7 +5008,8 @@ fn process_document( } } - let markdown = if all_gid && !options.force_extraction { + text_unreliable |= all_gid && options.best_effort; + let markdown = if all_gid && !options.best_effort { log::debug!( "all {} pages have gid-encoded fonts — suppressing markdown output", page_count @@ -5011,6 +5019,10 @@ fn process_document( markdown }; + // Only text that is actually returned can be unreliable. + let text_unreliable = + text_unreliable && markdown.as_ref().is_some_and(|m| !m.trim().is_empty()); + Ok(PdfProcessResult { pdf_type, markdown, @@ -5036,6 +5048,7 @@ fn process_document( confidence, layout, has_encoding_issues, + text_unreliable, cmap_gaps, }) } diff --git a/src/python.rs b/src/python.rs index 3b10724c..9a4e3eca 100644 --- a/src/python.rs +++ b/src/python.rs @@ -78,6 +78,11 @@ pub struct PyPdfResult { /// Whether encoding issues were detected. #[pyo3(get)] pub has_encoding_issues: bool, + /// True when `markdown` is text that `best_effort=True` kept and the + /// default would have withheld (scanned or image-based document, garbled + /// text, undecodable fonts). Always False without `best_effort`. + #[pyo3(get)] + pub text_unreliable: bool, /// Fonts whose ToUnicode CMap — or, for a font without one, the embedded /// program's cmap table — lacked an entry for a code the document shows /// through it, with the counts of codes shown, read from their neighbours @@ -597,6 +602,7 @@ fn to_py_result(r: crate::PdfProcessResult) -> PyPdfResult { pages_with_tables: r.layout.pages_with_tables, pages_with_columns: r.layout.pages_with_columns, has_encoding_issues: r.has_encoding_issues, + text_unreliable: r.text_unreliable, cmap_gaps: to_py_font_cmap_gaps(r.cmap_gaps), } } @@ -849,16 +855,13 @@ fn convert_region_results(results: Vec) -> Vec>, - force_extraction: bool, -) -> PyResult { - let mut opts = crate::PdfOptions::new().force_extraction(force_extraction); +#[pyo3(signature = (path, pages=None, best_effort=false))] +fn process_pdf(path: &str, pages: Option>, best_effort: bool) -> PyResult { + let mut opts = crate::PdfOptions::new().best_effort(best_effort); if let Some(p) = pages { opts = opts.pages(p); } @@ -868,13 +871,13 @@ fn process_pdf( /// Process a PDF from bytes in memory. #[pyfunction] -#[pyo3(signature = (data, pages=None, force_extraction=false))] +#[pyo3(signature = (data, pages=None, best_effort=false))] fn process_pdf_bytes( data: &[u8], pages: Option>, - force_extraction: bool, + best_effort: bool, ) -> PyResult { - let mut opts = crate::PdfOptions::new().force_extraction(force_extraction); + let mut opts = crate::PdfOptions::new().best_effort(best_effort); if let Some(p) = pages { opts = opts.pages(p); } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index a6bb915a..992f9d23 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -2141,6 +2141,7 @@ fn test_pages_needing_ocr_field_accessible() { confidence: 1.0, layout: pdf_inspector::LayoutComplexity::default(), has_encoding_issues: false, + text_unreliable: false, cmap_gaps: Vec::new(), }; assert_eq!(process_result.pages_needing_ocr, vec![1, 3]); @@ -3196,10 +3197,10 @@ fn test_inline_image_raster_covers_the_page() { assert_eq!(processed.pdf_type, PdfType::Scanned); } -/// `force_extraction` keeps the text layer behind a scan that the default -/// withholds, and leaves the OCR flags as they were. +/// `best_effort` returns the text layer behind a scan that the default +/// withholds, leaves the OCR flags as they were, and marks the text unreliable. #[test] -fn test_force_extraction_keeps_scanned_text() { +fn test_best_effort_keeps_scanned_text() { let buf = make_pdf_with_glyph_layer(&[SCAN_WITH_INVISIBLE_LAYER]); let default = process_pdf_mem(&buf).unwrap(); @@ -3208,12 +3209,13 @@ fn test_force_extraction_keeps_scanned_text() { PdfType::Scanned | PdfType::ImageBased )); assert!(default.markdown.is_none()); + assert!(!default.text_unreliable); - let forced = - process_pdf_mem_with_options(&buf, PdfOptions::new().force_extraction(true)).unwrap(); + let forced = process_pdf_mem_with_options(&buf, PdfOptions::new().best_effort(true)).unwrap(); assert_eq!(forced.pdf_type, default.pdf_type); assert_eq!(forced.pages_needing_ocr, default.pages_needing_ocr); assert_eq!(forced.ocr_reasons_by_page, default.ocr_reasons_by_page); + assert!(forced.text_unreliable); assert!(forced.markdown.is_some_and(|m| !m.trim().is_empty())); } From c10f7101e68cc3ccef9200228aa9bc51b17a4701 Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:55:24 -0700 Subject: [PATCH 3/7] fix: keep garbled TextBased pages under best_effort and correct the docs The per-page CID-garbage strip ran before the markdown-stage gates and dropped the text of OCR-flagged TextBased pages even with best_effort set. It now keeps those pages, still adds the garbled-text reason, and marks the result text_unreliable. The changelog no longer claims the OCR reasons are untouched, and the Python markdown docs no longer say a scanned PDF always has none. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 11 +++++++---- src/lib.rs | 19 ++++++++++++++----- src/python.rs | 3 ++- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00ef97eb..9cf2d54c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,10 +17,13 @@ version and date. Earlier releases are described in their document, whose text layer may still hold real or invisible text, otherwise returned no Markdown, as did a Mixed or TextBased one whose text was judged garbled or undecodable. With the option set, extraction - runs anyway. `pdf_type`, `pages_needing_ocr` and `ocr_reasons_by_page` - are unchanged, and the new `PdfProcessResult::text_unreliable` (also on - the Python result) is `true` when the Markdown is text the default would - have withheld. Off by default. + runs anyway, and a TextBased page whose text reads as garbage is kept + too. The OCR flags (`pdf_type`, `pages_needing_ocr`, + `ocr_reasons_by_page`) still report what detection found, and extraction + may add a `suspected_garbled_text` reason for text it reads. The new + `PdfProcessResult::text_unreliable` (also on the Python result) is `true` + when the Markdown is text the default would have withheld. Off by + default. ## [1.25.2] - 2026-09-28 diff --git a/src/lib.rs b/src/lib.rs index 92b89019..5c818910 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4810,8 +4810,9 @@ fn process_document( (items, rects, lines) } else { log::debug!( - "suppressing garbage text from OCR-flagged pages: {:?}", - garbage_pages + "garbage text on OCR-flagged pages {:?} (kept: {})", + garbage_pages, + options.best_effort ); for page in &garbage_pages { add_ocr_reason( @@ -4820,17 +4821,25 @@ fn process_document( OCR_REASON_SUSPECTED_GARBLED_TEXT, ); } + // `best_effort` keeps the text of these pages; the + // reason above still says it is suspect. + text_unreliable |= options.best_effort; + let dropped: std::collections::HashSet = if options.best_effort { + std::collections::HashSet::new() + } else { + garbage_pages + }; let items: Vec<_> = items .into_iter() - .filter(|i| !garbage_pages.contains(&i.page)) + .filter(|i| !dropped.contains(&i.page)) .collect(); let rects: Vec<_> = rects .into_iter() - .filter(|r| !garbage_pages.contains(&r.page)) + .filter(|r| !dropped.contains(&r.page)) .collect(); let lines: Vec<_> = lines .into_iter() - .filter(|l| !garbage_pages.contains(&l.page)) + .filter(|l| !dropped.contains(&l.page)) .collect(); (items, rects, lines) } diff --git a/src/python.rs b/src/python.rs index 9a4e3eca..568b149a 100644 --- a/src/python.rs +++ b/src/python.rs @@ -18,7 +18,8 @@ pub struct PyPdfResult { /// The detected PDF type: "text_based", "scanned", "image_based", or "mixed". #[pyo3(get)] pub pdf_type: String, - /// Markdown output (None if detect-only or scanned PDF). + /// Markdown output (None if detect-only, or if the text was withheld as + /// scanned or unreliable and best_effort was not set). #[pyo3(get)] pub markdown: Option, /// Total number of pages. From 172a0d8f47cea9bbb53245f8d941dd31b845e37d Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:55:33 -0700 Subject: [PATCH 4/7] docs: say that best_effort extraction may add a garbled-text reason Co-Authored-By: Claude Sonnet 5.5 --- src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 5c818910..a18cf123 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -347,9 +347,10 @@ impl PdfOptions { /// the markdown of a Mixed or TextBased document is dropped when it is /// garbled or its fonts are undecodable. With this set, extraction runs /// anyway (including a scanned page's invisible text layer) and whatever - /// it reads is returned. `pdf_type` and the OCR page flags are unchanged, - /// and [`PdfProcessResult::text_unreliable`] says when the markdown is - /// text the default would have withheld. + /// it reads is returned. `pdf_type` and the OCR page flags still report + /// what detection found (extraction may add a garbled-text reason), and + /// [`PdfProcessResult::text_unreliable`] says when the markdown is text + /// the default would have withheld. pub fn best_effort(mut self, best_effort: bool) -> Self { self.best_effort = best_effort; self From 64807ec01788e4b48e56c60d5a980a6990beca5e Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:02:33 -0700 Subject: [PATCH 5/7] docs: do not describe pdf_type as unchanged under best_effort Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 6 ++++-- src/lib.rs | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cf2d54c..d1c8b06e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,10 @@ version and date. Earlier releases are described in their text was judged garbled or undecodable. With the option set, extraction runs anyway, and a TextBased page whose text reads as garbage is kept too. The OCR flags (`pdf_type`, `pages_needing_ocr`, - `ocr_reasons_by_page`) still report what detection found, and extraction - may add a `suspected_garbled_text` reason for text it reads. The new + `ocr_reasons_by_page`) are still reported, so a caller can tell the text + may be unreliable. Extraction may add a `suspected_garbled_text` reason, + and a Mixed document whose text is garbage is still reclassified Scanned. + The new `PdfProcessResult::text_unreliable` (also on the Python result) is `true` when the Markdown is text the default would have withheld. Off by default. diff --git a/src/lib.rs b/src/lib.rs index a18cf123..5170e286 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -347,8 +347,9 @@ impl PdfOptions { /// the markdown of a Mixed or TextBased document is dropped when it is /// garbled or its fonts are undecodable. With this set, extraction runs /// anyway (including a scanned page's invisible text layer) and whatever - /// it reads is returned. `pdf_type` and the OCR page flags still report - /// what detection found (extraction may add a garbled-text reason), and + /// it reads is returned. `pdf_type` and the OCR page flags are still + /// reported (extraction may add a garbled-text reason, and a Mixed + /// document of garbage text is still reclassified Scanned), and /// [`PdfProcessResult::text_unreliable`] says when the markdown is text /// the default would have withheld. pub fn best_effort(mut self, best_effort: bool) -> Self { From f89b5a218dd8f54092573e649256f575be9a7ad5 Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:02:44 -0700 Subject: [PATCH 6/7] docs: rewrap best_effort changelog entry Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1c8b06e..1947e603 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,10 +22,9 @@ version and date. Earlier releases are described in their `ocr_reasons_by_page`) are still reported, so a caller can tell the text may be unreliable. Extraction may add a `suspected_garbled_text` reason, and a Mixed document whose text is garbage is still reclassified Scanned. - The new - `PdfProcessResult::text_unreliable` (also on the Python result) is `true` - when the Markdown is text the default would have withheld. Off by - default. + The new `PdfProcessResult::text_unreliable` (also on the Python result) + is `true` when the Markdown is text the default would have withheld. Off + by default. ## [1.25.2] - 2026-09-28 From e032326320031ad3e4281959bea23f57b055edbf Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:07:42 -0700 Subject: [PATCH 7/7] docs: state only what best_effort always preserves Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 6 ++---- src/lib.rs | 8 +++----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1947e603..18c32d15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,10 +18,8 @@ version and date. Earlier releases are described in their otherwise returned no Markdown, as did a Mixed or TextBased one whose text was judged garbled or undecodable. With the option set, extraction runs anyway, and a TextBased page whose text reads as garbage is kept - too. The OCR flags (`pdf_type`, `pages_needing_ocr`, - `ocr_reasons_by_page`) are still reported, so a caller can tell the text - may be unreliable. Extraction may add a `suspected_garbled_text` reason, - and a Mixed document whose text is garbage is still reclassified Scanned. + too. Detection still sets the OCR flags (`pdf_type`, `pages_needing_ocr`, + `ocr_reasons_by_page`), so a caller can tell the text may be unreliable. The new `PdfProcessResult::text_unreliable` (also on the Python result) is `true` when the Markdown is text the default would have withheld. Off by default. diff --git a/src/lib.rs b/src/lib.rs index 5170e286..c325ea10 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -347,11 +347,9 @@ impl PdfOptions { /// the markdown of a Mixed or TextBased document is dropped when it is /// garbled or its fonts are undecodable. With this set, extraction runs /// anyway (including a scanned page's invisible text layer) and whatever - /// it reads is returned. `pdf_type` and the OCR page flags are still - /// reported (extraction may add a garbled-text reason, and a Mixed - /// document of garbage text is still reclassified Scanned), and - /// [`PdfProcessResult::text_unreliable`] says when the markdown is text - /// the default would have withheld. + /// it reads is returned. Detection still sets `pdf_type` and the OCR + /// page flags, and [`PdfProcessResult::text_unreliable`] says when the + /// markdown is text the default would have withheld. pub fn best_effort(mut self, best_effort: bool) -> Self { self.best_effort = best_effort; self