feat(search): index PDF text, OCR scanned PDFs on import
All checks were successful
CI / Windows build (push) Successful in 15m50s
All checks were successful
CI / Windows build (push) Successful in 15m50s
Search now covers handwriting, the PDF text layer, AND scanned (rasterized) PDFs. - PdfTextIndexer runs at import: sums the embedded text layer across pages; if present it stores that as the document body, otherwise the PDF is rasterized and its rendered pages are OCR'd in the background. The result lands in the sidecar `pageText` field (distinct from `ocrText`, the handwriting OCR). Idempotent (skips a sidecar that already has pageText); degrades gracefully with no OCR engine. - pdfrx_page_text_source abstracts text/render so it's testable. - VaultSearchIndex now harvests title + typed text + handwriting OCR + PDF pageText, so search finds notes, typed PDFs and scanned PDFs. analyze clean, 409 tests green.
This commit is contained in:
@@ -8,8 +8,9 @@
|
||||
// * the title / source filename,
|
||||
// * a typed text box (EditorStroke.textContent),
|
||||
// * the persisted handwriting OCR text (sidecar `ocrText`),
|
||||
// * the PDF document-body text captured at import (sidecar `pageText`): both
|
||||
// the embedded text layer AND the background-OCR result for a scanned PDF,
|
||||
// * a CJK substring (this user writes Chinese).
|
||||
// It also documents the known GAP: a PDF's embedded text layer is NOT indexed.
|
||||
|
||||
import 'dart:io';
|
||||
|
||||
@@ -56,6 +57,7 @@ void main() {
|
||||
required String pdfName,
|
||||
List<EditorStroke> page0 = const [],
|
||||
String? ocrText,
|
||||
String? pageText,
|
||||
}) async {
|
||||
final dir = Directory(p.join(vaultDir.path, folder));
|
||||
await dir.create(recursive: true);
|
||||
@@ -66,6 +68,7 @@ void main() {
|
||||
docType: 'pdf',
|
||||
strokes: page0.isEmpty ? null : {0: page0},
|
||||
ocrText: ocrText,
|
||||
pageText: pageText,
|
||||
createdAt: DateTime.now().toUtc(),
|
||||
);
|
||||
await SidecarStore.writeAtomic(
|
||||
@@ -173,11 +176,39 @@ void main() {
|
||||
expect(await index.search('x'), isEmpty);
|
||||
});
|
||||
|
||||
test('KNOWN GAP: a PDF embedded text layer is NOT indexed', () async {
|
||||
// The sidecar carries no annotations; only the PDF body would contain the
|
||||
// word "bodytext". Search does NOT read the PDF text layer (documented
|
||||
// limitation), so this returns nothing.
|
||||
await seedDocNotebook(folder: 'Plain', pdfName: 'plain.pdf');
|
||||
test('finds a PDF by its embedded text layer (sidecar pageText)', () async {
|
||||
// The PDF body text captured at import lives in the sidecar's `pageText`.
|
||||
// No annotations at all — only the document body contains "bodytext".
|
||||
await seedDocNotebook(
|
||||
folder: 'Plain',
|
||||
pdfName: 'plain.pdf',
|
||||
pageText: 'introduction to bodytext and more printed content',
|
||||
);
|
||||
final index = VaultSearchIndex(vault);
|
||||
final hits = await index.search('bodytext');
|
||||
expect(hits, hasLength(1));
|
||||
expect(hits.single.entry.openPath, endsWith('plain.pdf'));
|
||||
});
|
||||
|
||||
test('finds a SCANNED PDF by its background-OCR pageText (CJK)', () async {
|
||||
// A rasterized/scanned PDF has no text layer; the import-time OCR pass
|
||||
// writes the recognized text into the SAME `pageText` field, so search
|
||||
// covers scanned documents — including Chinese substrings.
|
||||
await seedDocNotebook(
|
||||
folder: '扫描讲义',
|
||||
pdfName: 'scanned.pdf',
|
||||
pageText: '微积分第三讲 导数的定义与几何意义',
|
||||
);
|
||||
final index = VaultSearchIndex(vault);
|
||||
expect(await index.search('导数'), hasLength(1));
|
||||
expect((await index.search('几何')).single.entry.openPath,
|
||||
endsWith('scanned.pdf'));
|
||||
});
|
||||
|
||||
test('a PDF with no pageText (un-indexed) is not found by body text', () async {
|
||||
// Back-compat: a PDF imported before the feature (or whose OCR backend was
|
||||
// unavailable) has no `pageText`; only its annotations are searchable.
|
||||
await seedDocNotebook(folder: 'Old', pdfName: 'old.pdf');
|
||||
final index = VaultSearchIndex(vault);
|
||||
expect(await index.search('bodytext'), isEmpty);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user