// lib/services/pdf_text_indexer.dart // // Import-time document-body text indexing for file-backed PDF notebooks, so the // vault-scan search index (VaultSearchIndex) covers the underlying document — // not just the user's annotations. Three text sources now feed search: // (a) handwriting → OCR'd into the sidecar's `ocrText` (OcrService), // (b) a PDF text layer → its embedded printed text, captured here, // (c) a RASTERIZED PDF → background OCR of the rendered pages, captured here. // // Flow (kicked off after VaultService.createNotebook, fire-and-forget): // 1. IDEMPOTENCY: if the sidecar already carries `pageText`, do nothing. // 2. EXTRACT the embedded text layer per page (pdfrx `loadText`). // 3. DECIDE text-layer vs rasterized: sum the embedded text length across all // pages; if it clears [textLayerThreshold] the PDF has a usable text layer // and we persist that. Otherwise the PDF is rasterized (scanned image, no // text) and we OCR each rendered page. // 4. PERSIST the per-page text (joined by form-feed) into the sidecar's // `pageText` field — writing THROUGH an open SidecarRepository when the // editor already has the doc open (no race), else a transient handle. // // GRACEFUL DEGRADATION / HONESTY: // * The OCR engine is whatever OcrService/OcrEngine resolves to: the bundled // ONNX PP-OCR recognizer if present, else the native Windows OCR // MethodChannel, else NONE. On a platform/CI without any backend, OCR // returns null and a rasterized PDF simply gets no `pageText` — no crash, // and the embedded-text path still works. // * pdfrx render + native OCR can only be exercised on-device. The pdfrx- // backed loaders are injected through [PdfTextIndexer] so unit tests fake // them; the production wiring lives in [PdfrxPageTextSource]. import 'dart:async'; import 'dart:io'; import '../editor/persistence/sidecar_repository.dart'; import '../storage/badnote_sidecar.dart'; import '../storage/sidecar_store.dart'; /// Extracts a PDF's embedded text layer, one entry per page (page order). An /// empty list (or all-empty entries) means "no usable text layer". typedef PdfEmbeddedTextLoader = Future> Function(String pdfPath); /// OCRs the rendered pages of a rasterized PDF, returning one entry per page /// (page order). Entries may be empty where a page yielded nothing. Returns an /// empty list when no OCR backend is available (a clean no-op). typedef PdfPageOcrRunner = Future> Function(String pdfPath); /// Indexes a PDF's document body into its sidecar's `pageText` at import time. /// /// Stateless apart from the two injected text sources; safe to construct per /// import. All disk/native work is awaited internally — call [indexPdf] without /// awaiting (fire-and-forget) from the import handler to keep import snappy. class PdfTextIndexer { PdfTextIndexer({ required PdfEmbeddedTextLoader loadEmbeddedText, required PdfPageOcrRunner ocrPages, this.textLayerThreshold = 16, }) : _loadEmbeddedText = loadEmbeddedText, _ocrPages = ocrPages; final PdfEmbeddedTextLoader _loadEmbeddedText; final PdfPageOcrRunner _ocrPages; /// Minimum total embedded-text length (across all pages, after trimming) for a /// PDF to count as having a usable text layer. Below this it is treated as /// rasterized and routed to OCR. Small on purpose: a scanned PDF typically /// yields zero or a few stray ligature chars, while any real text page clears /// it easily. final int textLayerThreshold; /// The page separator stored inside `pageText` (form feed). The search index /// treats `pageText` as a flat blob, so this is purely cosmetic / future-proof. static const String pageSeparator = '\f'; /// Index the PDF at [pdfPath] (an in-vault copy) into its sidecar's `pageText`. /// /// Idempotent: returns immediately if the sidecar already has non-empty /// `pageText`. Never throws — any failure (unreadable PDF, missing OCR backend) /// degrades to leaving `pageText` unset. Returns the text it persisted (for /// tests), or null when nothing was indexed. Future indexPdf(String pdfPath) async { try { // 1. Idempotency: skip a doc whose body has already been indexed. final existing = await _currentPageText(pdfPath); if (existing != null && existing.trim().isNotEmpty) return null; // 2. Embedded text layer. final embedded = await _loadEmbeddedText(pdfPath); final embeddedLen = embedded.fold( 0, (sum, page) => sum + page.trim().length, ); // 3. Text-layer vs rasterized decision. List pages; if (embeddedLen >= textLayerThreshold) { pages = embedded; } else { // Rasterized (scanned, no text layer) → background OCR. pages = await _ocrPages(pdfPath); } final joined = _joinPages(pages); if (joined.isEmpty) return null; // 4. Persist into the sidecar (through an open repo if the editor holds it). await _persistPageText(pdfPath, joined); return joined; } catch (_) { // Background indexing must never surface an error to the import flow. return null; } } /// Whether the embedded text in [embedded] clears [textLayerThreshold], i.e. /// the PDF has a usable text layer (false → rasterized, needs OCR). Exposed for /// unit-testing the decision in isolation. bool hasUsableTextLayer(List embedded) { final len = embedded.fold(0, (sum, p) => sum + p.trim().length); return len >= textLayerThreshold; } static String _joinPages(List pages) { final nonEmpty = pages.map((p) => p.trim()).where((p) => p.isNotEmpty); return nonEmpty.join(pageSeparator).trim(); } /// Read the sidecar's current `pageText` (for the idempotency check), from the /// open repo if present else from disk. Null when no sidecar exists yet. Future _currentPageText(String pdfPath) async { final open = SidecarRepositoryRegistry.forPath(pdfPath); if (open != null) return open.loadedPageText; final sidecar = await SidecarStore.read( File('$pdfPath$kSidecarSuffix'), ); return sidecar?.pageText; } /// Write [pageText] into the sidecar. Prefer the editor's already-open repo /// (same in-memory sidecar — no race); otherwise merge into the on-disk /// sidecar (creating one if the editor hasn't yet). Future _persistPageText(String pdfPath, String pageText) async { final open = SidecarRepositoryRegistry.forPath(pdfPath); if (open != null) { open.schedulePageTextSave(pageText); await open.flush(); return; } final file = File('$pdfPath$kSidecarSuffix'); final current = await SidecarStore.read(file); final merged = _withPageText(current, pdfPath, pageText); await SidecarStore.writeAtomic(file, merged); } /// Build a sidecar carrying [pageText], preserving every other field of /// [current] (or a fresh minimal sidecar when none exists yet). static BadnoteSidecar _withPageText( BadnoteSidecar? current, String pdfPath, String pageText, ) { if (current == null) { return BadnoteSidecar( sourceFile: _basename(pdfPath), docType: 'pdf', createdAt: DateTime.now().toUtc(), updatedAt: DateTime.now().toUtc(), pageText: pageText, ); } return BadnoteSidecar( version: current.version, sourceFile: current.sourceFile, docType: current.docType, title: current.title, pageCount: current.pageCount, rotation: current.rotation, createdAt: current.createdAt, updatedAt: DateTime.now().toUtc(), strokes: current.strokes, highlights: current.highlights, texts: current.texts, bookmarks: current.bookmarks, scratchLinks: current.scratchLinks, legacyAnnotations: current.legacyAnnotations, ocrText: current.ocrText, pageText: pageText, legacyId: current.legacyId, background: current.background, ); } static String _basename(String path) { final norm = path.replaceAll('\\', '/'); final i = norm.lastIndexOf('/'); return i == -1 ? norm : norm.substring(i + 1); } }