feat(f8): CJK-safe search text normalization + matching
Some checks failed
CI / Windows build (push) Has been cancelled

normalizeForIndex (lowercase + collapse whitespace runs incl. hard PDF/OCR
newlines + trim) and matchesNormalized so a query matches across the line breaks
in raw extracted text. Deliberately NO word-tokenization: Chinese has no
inter-word spaces, so substring match over normalized text is correct for both
Latin and CJK (段/word segmentation belongs in the DB FTS tokenizer). Verified
on CJK inputs (你好/笔记应用).

flutter analyze lib/editor clean; 181/181 tests (+9).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-23 03:30:42 +08:00
parent 1aa2865d6f
commit 0cf58fb67a
2 changed files with 81 additions and 0 deletions

View File

@@ -0,0 +1,30 @@
// lib/editor/search/search_text.dart
//
// Pure text normalization + matching for full-text search (F8). PDF text layers
// and OCR output are full of hard line breaks and irregular whitespace, so a
// query like "hello world" won't substring-match raw extracted text that reads
// "hello\nworld". Normalizing both sides (lowercase + collapse every whitespace
// run to a single space + trim) fixes that.
//
// CJK NOTE: this user writes Chinese. We deliberately do NOT word-tokenize —
// Chinese has no inter-word spaces, so a whitespace/punctuation tokenizer would
// mangle it. Substring matching over normalized text is correct for both Latin
// and CJK; word/段 segmentation belongs in the DB FTS tokenizer (trigram /
// unicode61), not here.
/// Matches any run of Unicode whitespace (spaces, tabs, newlines, NBSP, …).
final RegExp _whitespaceRun = RegExp(r'\s+');
/// Normalize [text] for indexing/matching: lowercase, collapse whitespace runs
/// (incl. the hard newlines PDF/OCR insert mid-sentence) to single spaces, trim.
String normalizeForIndex(String text) {
return text.toLowerCase().replaceAll(_whitespaceRun, ' ').trim();
}
/// Whether [source] contains [query] after both are normalized — so a match can
/// span the line breaks present in the raw text. Empty query never matches.
bool matchesNormalized(String source, String query) {
final q = normalizeForIndex(query);
if (q.isEmpty) return false;
return normalizeForIndex(source).contains(q);
}

View File

@@ -0,0 +1,51 @@
// Tests for CJK-safe search text normalization + matching (F8).
import 'package:flutter_test/flutter_test.dart';
import 'package:badnote/editor/search/search_text.dart';
void main() {
group('normalizeForIndex', () {
test('lowercases and collapses whitespace runs to single spaces', () {
expect(normalizeForIndex('Hello World'), 'hello world');
expect(normalizeForIndex('a\t b\n\nc'), 'a b c');
});
test('trims leading/trailing whitespace', () {
expect(normalizeForIndex(' padded '), 'padded');
});
test('collapses hard newlines from PDF/OCR mid-sentence', () {
expect(normalizeForIndex('hello\nworld'), 'hello world');
});
test('leaves CJK intact (no tokenization/mangling)', () {
expect(normalizeForIndex('你好 世界'), '你好 世界');
expect(normalizeForIndex('笔记\n应用'), '笔记 应用');
});
});
group('matchesNormalized', () {
test('matches across a line break in the source', () {
expect(matchesNormalized('hello\nworld', 'hello world'), isTrue);
});
test('is case-insensitive', () {
expect(matchesNormalized('The Quick Fox', 'quick'), isTrue);
});
test('CJK substring match works', () {
expect(matchesNormalized('这是一个笔记应用', '笔记'), isTrue);
expect(matchesNormalized('这是一个笔记\n应用', '笔记 应用'), isTrue);
});
test('empty query never matches', () {
expect(matchesNormalized('anything', ''), isFalse);
expect(matchesNormalized('anything', ' '), isFalse);
});
test('non-match returns false', () {
expect(matchesNormalized('hello world', 'zzz'), isFalse);
});
});
}