feat(search): index PDF text, OCR scanned PDFs on import
All checks were successful
CI / Windows build (push) Successful in 15m50s

Search now covers handwriting, the PDF text layer, AND scanned
(rasterized) PDFs.

- PdfTextIndexer runs at import: sums the embedded text layer across
  pages; if present it stores that as the document body, otherwise the
  PDF is rasterized and its rendered pages are OCR'd in the background.
  The result lands in the sidecar `pageText` field (distinct from
  `ocrText`, the handwriting OCR). Idempotent (skips a sidecar that
  already has pageText); degrades gracefully with no OCR engine.
- pdfrx_page_text_source abstracts text/render so it's testable.
- VaultSearchIndex now harvests title + typed text + handwriting OCR +
  PDF pageText, so search finds notes, typed PDFs and scanned PDFs.

analyze clean, 409 tests green.
This commit is contained in:
2026-06-25 00:23:19 +08:00
parent 20add27a30
commit e939759458
10 changed files with 668 additions and 16 deletions

View File

@@ -0,0 +1,195 @@
// test/pdf_text_indexer_test.dart
//
// Unit tests for the import-time PDF document-body indexer. The two native text
// sources (pdfrx embedded-text loader + page-render OCR) are FAKED here — no
// real pdfium render and no real OCR channel run in CI; those are exercised only
// on-device (see PdfrxPageTextSource). These tests pin:
// * the text-layer-vs-rasterized DECISION (empty text layer → OCR fallback),
// * the background flow: embedded text OR OCR text lands in the sidecar's
// `pageText`,
// * IDEMPOTENCY: a sidecar that already has `pageText` is not re-indexed,
// * graceful degradation: no OCR backend (empty OCR result) → no `pageText`,
// no crash.
import 'dart:io';
import 'package:flutter_test/flutter_test.dart';
import 'package:path/path.dart' as p;
import 'package:badnote/editor/persistence/sidecar_repository.dart';
import 'package:badnote/services/pdf_text_indexer.dart';
import 'package:badnote/storage/badnote_sidecar.dart';
import 'package:badnote/storage/sidecar_store.dart';
void main() {
TestWidgetsFlutterBinding.ensureInitialized();
late Directory dir;
late String pdfPath;
setUp(() async {
SidecarRepositoryRegistry.resetForTest();
dir = await Directory.systemTemp.createTemp('pdf_indexer_test');
pdfPath = p.join(dir.path, 'doc.pdf');
await File(pdfPath).writeAsString('%PDF-1.7 fake');
});
tearDown(() async {
SidecarRepositoryRegistry.resetForTest();
if (await dir.exists()) await dir.delete(recursive: true);
});
Future<BadnoteSidecar?> readSidecar() =>
SidecarStore.read(File('$pdfPath$kSidecarSuffix'));
// A text-layer PDF: embedded loader returns real text, OCR should NOT run.
test('a PDF with a usable text layer persists the embedded text', () async {
var ocrCalled = false;
final indexer = PdfTextIndexer(
loadEmbeddedText: (_) async => ['Chapter one introduction', 'page two'],
ocrPages: (_) async {
ocrCalled = true;
return ['should not be used'];
},
);
final result = await indexer.indexPdf(pdfPath);
expect(ocrCalled, isFalse, reason: 'text layer present → no OCR');
expect(result, contains('Chapter one introduction'));
expect(result, contains('page two'));
final sidecar = await readSidecar();
expect(sidecar?.pageText, contains('introduction'));
});
// A rasterized/scanned PDF: empty embedded text → OCR fallback runs.
test('a rasterized PDF (empty text layer) is OCR\'d into pageText', () async {
var ocrCalled = false;
final indexer = PdfTextIndexer(
loadEmbeddedText: (_) async => ['', '', ''], // scanned: no text layer
ocrPages: (_) async {
ocrCalled = true;
return ['扫描出的文字', 'recognized line two'];
},
);
final result = await indexer.indexPdf(pdfPath);
expect(ocrCalled, isTrue, reason: 'no text layer → OCR runs');
expect(result, contains('扫描出的文字'));
final sidecar = await readSidecar();
expect(sidecar?.pageText, contains('扫描出的文字'));
expect(sidecar?.pageText, contains('recognized line two'));
});
test('the text-layer decision: empty/near-empty text → needs OCR', () {
final indexer = PdfTextIndexer(
loadEmbeddedText: (_) async => const [],
ocrPages: (_) async => const [],
textLayerThreshold: 16,
);
// Empty pages → rasterized (needs OCR).
expect(indexer.hasUsableTextLayer(['', '', '']), isFalse);
// A few stray ligature chars below threshold → still rasterized.
expect(indexer.hasUsableTextLayer([' ', 'fi', ' ']), isFalse);
// Real text above threshold → usable text layer.
expect(
indexer.hasUsableTextLayer(['This is real printed body text']),
isTrue,
);
});
test('idempotent: a sidecar that already has pageText is not re-indexed',
() async {
// Pre-seed a sidecar with existing pageText.
await SidecarStore.writeAtomic(
File('$pdfPath$kSidecarSuffix'),
BadnoteSidecar(
sourceFile: 'doc.pdf',
docType: 'pdf',
pageText: 'already indexed body',
),
);
var embeddedCalled = false;
var ocrCalled = false;
final indexer = PdfTextIndexer(
loadEmbeddedText: (_) async {
embeddedCalled = true;
return ['new text'];
},
ocrPages: (_) async {
ocrCalled = true;
return ['new ocr'];
},
);
final result = await indexer.indexPdf(pdfPath);
expect(result, isNull, reason: 'already indexed → no-op');
expect(embeddedCalled, isFalse);
expect(ocrCalled, isFalse);
// Existing text untouched.
final sidecar = await readSidecar();
expect(sidecar?.pageText, 'already indexed body');
});
test('no OCR backend (empty OCR result) leaves pageText unset, no crash',
() async {
final indexer = PdfTextIndexer(
loadEmbeddedText: (_) async => ['', ''], // rasterized
ocrPages: (_) async => const [], // no OCR backend → clean no-op
);
final result = await indexer.indexPdf(pdfPath);
expect(result, isNull);
final sidecar = await readSidecar();
// No sidecar written (nothing to index) — or, if present, no pageText.
expect(sidecar?.pageText, isNull);
});
test('writes THROUGH an open editor repo (no second on-disk race)', () async {
// Open a repo for the same path: the indexer must write through it.
final repo = await SidecarRepository.open(pdfPath, docType: 'pdf');
addTearDown(repo.dispose);
final indexer = PdfTextIndexer(
loadEmbeddedText: (_) async => ['printed body via open editor'],
ocrPages: (_) async => const [],
);
final result = await indexer.indexPdf(pdfPath);
expect(result, contains('printed body'));
// The in-memory sidecar held by the editor now carries the pageText.
expect(repo.loadedPageText, contains('printed body'));
// And it was flushed to disk.
final sidecar = await readSidecar();
expect(sidecar?.pageText, contains('printed body'));
});
test('preserves existing annotations when merging pageText on disk', () async {
// A sidecar with annotations but no pageText (e.g. user annotated before the
// background OCR finished). Indexing must not clobber the annotations.
await SidecarStore.writeAtomic(
File('$pdfPath$kSidecarSuffix'),
BadnoteSidecar(
sourceFile: 'doc.pdf',
docType: 'pdf',
ocrText: 'handwriting note',
),
);
final indexer = PdfTextIndexer(
loadEmbeddedText: (_) async => ['printed document body text here'],
ocrPages: (_) async => const [],
);
await indexer.indexPdf(pdfPath);
final sidecar = await readSidecar();
expect(sidecar?.ocrText, 'handwriting note'); // preserved
expect(sidecar?.pageText, contains('printed document body'));
});
}