feat: add best_effort option and text_unreliable flag

CHANGELOG.md

@@ -7,6 +7,20 @@ version. A separate release pull request bumps the manifests with
 version and date. Earlier releases are described in their
 [GitHub releases](https://github.com/firecrawl/pdf-inspector/releases).
 
+## [Unreleased]
+
+### Added
+
+- `PdfOptions::force_extraction` (`force_extraction=True` in the Python
+  `process_pdf` and `process_pdf_bytes`) keeps the text of pages flagged for
+  OCR instead of suppressing it. A Scanned or ImageBased document, whose
+  text layer may still hold real or invisible text, otherwise returned no
+  Markdown, as did a Mixed or TextBased one whose text was judged garbled or
+  undecodable. With the option set, extraction runs anyway and returns what
+  it reads. `pdf_type`, `pages_needing_ocr` and `ocr_reasons_by_page` are
+  unchanged, so a caller still sees that the text may be unreliable. Off by
+  default.
+
 ## [1.25.2] - 2026-09-28
 
 Changes since 1.25.1.

pdf_inspector.pyi

@@ -304,12 +304,20 @@ class PagesExtractionResult:
     is_complex: bool
     """True if any page has tables or multi-column layout."""
 
-def process_pdf(path: str, pages: Optional[list[int]] = None) -> PdfResult:
-    """Process a PDF: detect type, extract text, convert to Markdown."""
+def process_pdf(
+    path: str, pages: Optional[list[int]] = None, force_extraction: bool = False
+) -> PdfResult:
+    """Process a PDF: detect type, extract text, convert to Markdown.
+
+    ``force_extraction`` keeps the text of pages flagged for OCR (scanned,
+    garbled, undecodable fonts) instead of returning none; ``pdf_type`` and
+    ``pages_needing_ocr`` still report the flag."""
     ...
 
-def process_pdf_bytes(data: bytes, pages: Optional[list[int]] = None) -> PdfResult:
-    """Process a PDF from bytes in memory."""
+def process_pdf_bytes(
+    data: bytes, pages: Optional[list[int]] = None, force_extraction: bool = False
+) -> PdfResult:
+    """Process a PDF from bytes in memory. See ``process_pdf`` for ``force_extraction``."""
     ...
 
 def process_pdf_with_ocr(

src/lib.rs

@@ -262,6 +262,11 @@ pub struct PdfOptions {
     /// Password for decrypting an encrypted PDF. `None` falls back to the
     /// empty password (owner-only encryption).
     pub password: Option<String>,
+    /// Keep the extracted text of pages flagged for OCR instead of
+    /// suppressing it (default: `false`). The flags (`pdf_type`,
+    /// `pages_needing_ocr`, `ocr_reasons_by_page`) are reported as usual, so
+    /// the caller still knows the text may be incomplete or unreliable.
+    pub force_extraction: bool,
 }
 
 // Manual `Debug` so the password is never leaked through debug logging or a
@@ -274,6 +279,7 @@ impl std::fmt::Debug for PdfOptions {
             .field("markdown", &self.markdown)
             .field("page_filter", &self.page_filter)
             .field("password", &self.password.as_ref().map(|_| "[REDACTED]"))
+            .field("force_extraction", &self.force_extraction)
             .finish()
     }
 }
@@ -286,6 +292,7 @@ impl Default for PdfOptions {
             markdown: MarkdownOptions::default(),
             page_filter: None,
             password: None,
+            force_extraction: false,
         }
     }
 }
@@ -328,6 +335,18 @@ impl PdfOptions {
         self
     }
 
+    /// Keep the text of pages flagged for OCR rather than returning none.
+    ///
+    /// By default a Scanned or ImageBased document yields no markdown, and
+    /// the markdown of a Mixed or TextBased document is dropped when it is
+    /// garbled or its fonts are undecodable. With this set, extraction runs
+    /// anyway (including a scanned page's invisible text layer) and whatever
+    /// it reads is returned; `pdf_type` and the OCR page flags are unchanged.
+    pub fn force_extraction(mut self, force: bool) -> Self {
+        self.force_extraction = force;
+        self
+    }
+
     /// Set the password used to decrypt an encrypted PDF.
     pub fn password(mut self, password: impl Into<String>) -> Self {
         self.password = Some(password.into());
@@ -4619,8 +4638,9 @@ fn process_document(
         });
     }
 
-    // Scanned / ImageBased → nothing to extract
-    if matches!(pdf_type, PdfType::Scanned | PdfType::ImageBased) {
+    // Scanned / ImageBased → nothing to extract, unless extraction is forced
+    let scanned_type = matches!(pdf_type, PdfType::Scanned | PdfType::ImageBased);
+    if scanned_type && !options.force_extraction {
         return Ok(PdfProcessResult {
             pdf_type,
             markdown: None,
@@ -4643,6 +4663,10 @@ fn process_document(
         });
     }
 
+    // Mixed PDFs, and Scanned/ImageBased ones whose extraction is forced, are
+    // backed by images: their text layer may be absent or invisible.
+    let image_backed = pdf_type == PdfType::Mixed || scanned_type;
+
     // Step 2 — Extraction (reuses the already-loaded document)
     let extracted = {
         let font_cmaps = FontCMaps::from_doc(&doc);
@@ -4658,7 +4682,7 @@ fn process_document(
         // For Mixed/template PDFs: if normal extraction produces garbage text
         // (mostly non-alphanumeric), retry with invisible (Tr=3) text included.
         // This unlocks OCR text layers behind scanned images.
-        if pdf_type == PdfType::Mixed {
+        if image_backed {
             if let Ok((ref items, _, _)) = result.as_ref().map(|(e, _, _, _, _)| e) {
                 let sample: String = items
                     .iter()
@@ -4694,7 +4718,7 @@ fn process_document(
     };
 
     // For Mixed PDFs, extraction failure is non-fatal
-    let extracted = if pdf_type == PdfType::Mixed {
+    let extracted = if image_backed {
         extracted.ok()
     } else {
         Some(extracted?)
@@ -4893,7 +4917,15 @@ fn process_document(
     // layer comes from a bad OCR pass, and callers should use proper OCR.
     let (pdf_type, markdown, confidence) =
         if pdf_type == PdfType::Mixed && markdown.as_ref().is_some_and(|m| is_garbage_text(m)) {
-            (PdfType::Scanned, None, 0.95)
+            (
+                PdfType::Scanned,
+                if options.force_extraction {
+                    markdown
+                } else {
+                    None
+                },
+                0.95,
+            )
         } else {
             (pdf_type, markdown, confidence)
         };
@@ -4905,7 +4937,15 @@ fn process_document(
         && markdown.as_ref().is_some_and(|m| is_garbage_text(m))
     {
         log::debug!("TextBased PDF has garbage text — flagging all pages for OCR");
-        (None, true, true)
+        (
+            if options.force_extraction {
+                markdown
+            } else {
+                None
+            },
+            true,
+            true,
+        )
     } else {
         (markdown, has_encoding_issues, false)
     };
@@ -4961,7 +5001,7 @@ fn process_document(
         }
     }
 
-    let markdown = if all_gid {
+    let markdown = if all_gid && !options.force_extraction {
         log::debug!(
             "all {} pages have gid-encoded fonts — suppressing markdown output",
             page_count

src/python.rs

@@ -848,10 +848,17 @@ fn convert_region_results(results: Vec<crate::PageRegionResult>) -> Vec<PyPageRe
 // ---------------------------------------------------------------------------
 
 /// Process a PDF file: detect type, extract text, and convert to Markdown.
+///
+/// With force_extraction=True the text of pages flagged for OCR is kept
+/// instead of suppressed; pdf_type and pages_needing_ocr are unchanged.
 #[pyfunction]
-#[pyo3(signature = (path, pages=None))]
-fn process_pdf(path: &str, pages: Option<Vec<u32>>) -> PyResult<PyPdfResult> {
-    let mut opts = crate::PdfOptions::new();
+#[pyo3(signature = (path, pages=None, force_extraction=false))]
+fn process_pdf(
+    path: &str,
+    pages: Option<Vec<u32>>,
+    force_extraction: bool,
+) -> PyResult<PyPdfResult> {
+    let mut opts = crate::PdfOptions::new().force_extraction(force_extraction);
     if let Some(p) = pages {
         opts = opts.pages(p);
     }
@@ -861,9 +868,13 @@ fn process_pdf(path: &str, pages: Option<Vec<u32>>) -> PyResult<PyPdfResult> {
 
 /// Process a PDF from bytes in memory.
 #[pyfunction]
-#[pyo3(signature = (data, pages=None))]
-fn process_pdf_bytes(data: &[u8], pages: Option<Vec<u32>>) -> PyResult<PyPdfResult> {
-    let mut opts = crate::PdfOptions::new();
+#[pyo3(signature = (data, pages=None, force_extraction=false))]
+fn process_pdf_bytes(
+    data: &[u8],
+    pages: Option<Vec<u32>>,
+    force_extraction: bool,
+) -> PyResult<PyPdfResult> {
+    let mut opts = crate::PdfOptions::new().force_extraction(force_extraction);
     if let Some(p) = pages {
         opts = opts.pages(p);
     }

tests/integration_tests.rs

@@ -3196,6 +3196,27 @@ fn test_inline_image_raster_covers_the_page() {
     assert_eq!(processed.pdf_type, PdfType::Scanned);
 }
 
+/// `force_extraction` keeps the text layer behind a scan that the default
+/// withholds, and leaves the OCR flags as they were.
+#[test]
+fn test_force_extraction_keeps_scanned_text() {
+    let buf = make_pdf_with_glyph_layer(&[SCAN_WITH_INVISIBLE_LAYER]);
+
+    let default = process_pdf_mem(&buf).unwrap();
+    assert!(matches!(
+        default.pdf_type,
+        PdfType::Scanned | PdfType::ImageBased
+    ));
+    assert!(default.markdown.is_none());
+
+    let forced =
+        process_pdf_mem_with_options(&buf, PdfOptions::new().force_extraction(true)).unwrap();
+    assert_eq!(forced.pdf_type, default.pdf_type);
+    assert_eq!(forced.pages_needing_ocr, default.pages_needing_ocr);
+    assert_eq!(forced.ocr_reasons_by_page, default.ocr_reasons_by_page);
+    assert!(forced.markdown.is_some_and(|m| !m.trim().is_empty()));
+}
+
 /// A scan tiled into two thousand strips — image XObjects or inline
 /// images — covers the page as one draw does: the hidden layer over it
 /// is a layer nobody sees.