akiba/BadNote
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(storage): one-time SQLite to sidecar migration
Some checks failed
CI / Windows build (push) Has been cancelled
Details

Browse Source 
Phase 5. On first launch with a valid vault, migrate legacy SQLite
data into vault sidecars so nothing is lost on upgrade.

- SqliteToSidecarMigrator: documents (+ per-page ink, scratch-links
  + scratchpads, bookmarks) -> notebook folder + <file>.badnote.json;
  notes (+ strokes) -> notebook.badnote.json. Reuses existing JSON.
- Idempotent (skips already-migrated targets); missing source files
  still get their annotations migrated.
- DB relocates to <vault>/.badnote/index.sqlite; the legacy DB is
  renamed to .premigration ONLY after a successful pass, so a failed
  migration leaves data intact and the run-once flag unset.
- main.dart runs it once, gated on vaultMigrationDone.

Golden migrator tests (seeded legacy DB -> sidecars, idempotent
re-run, legacy preserved). analyze clean, tests green.
This commit is contained in:
Akiba So
2026-06-24 23:05:10 +08:00
parent f4f0853eae
commit 4886f1b2df
6 changed files with 1070 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

423
test/sqlite_to_sidecar_migrator_test.dart Normal file
View File

@@ -0,0 +1,423 @@
// test/sqlite_to_sidecar_migrator_test.dart
//
// Golden-style proof of the one-time SQLite→sidecar migration (Phase 5, §B):
// * Seed a LEGACY sqlite DB (the pre-vault `badnote.db` schema) with a
// document (+ ink strokes on two pages, a scratch_link + its scratchpad, a
// bookmark, a legacy annotation blob) and a free-ink note (+ strokes).
// * Run the migrator against a temp vault.
// * Assert the sidecars contain the migrated data (table → field mapping).
// * Assert an idempotent re-run is a no-op (no duplicate folders/sidecars).
// * Assert the legacy DB is preserved (renamed to `*.premigration`).
// * Assert the missing-source case still migrates annotations.
//
// Uses a real ffi sqlite DB on disk (so it exercises the migrator's read path)
// and real temp dirs for the vault.
import 'dart:convert';
import 'dart:io';
import 'package:flutter_test/flutter_test.dart';
import 'package:path/path.dart' as p;
import 'package:shared_preferences/shared_preferences.dart';
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
import 'package:badnote/editor/notebook/ink_stroke_adapter.dart';
import 'package:badnote/services/vault_service.dart';
import 'package:badnote/storage/sidecar_store.dart';
import 'package:badnote/storage/sqlite_to_sidecar_migrator.dart';
const _sep = '/'; // path.join uses platform sep; tests run on POSIX CI.
/// Build the subset of the legacy v8 schema the migrator reads.
Future<Database> _openLegacyDb(String path) async {
sqfliteFfiInit();
return databaseFactoryFfi.openDatabase(
path,
options: OpenDatabaseOptions(
version: 1,
onCreate: (db, _) async {
await db.execute('''
CREATE TABLE documents (
id TEXT PRIMARY KEY, filename TEXT NOT NULL, doc_type TEXT NOT NULL,
file_path TEXT NOT NULL, page_count INTEGER NOT NULL DEFAULT 0,
rotation INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL,
updated_at TEXT NOT NULL)
''');
await db.execute('''
CREATE TABLE annotations (
id TEXT PRIMARY KEY, uuid TEXT NOT NULL, document_id TEXT NOT NULL,
page_number INTEGER NOT NULL, annotation_json TEXT NOT NULL,
created_at TEXT NOT NULL, updated_at TEXT NOT NULL)
''');
await db.execute('''
CREATE TABLE bookmarks (
id TEXT PRIMARY KEY, document_id TEXT NOT NULL,
page_number INTEGER NOT NULL, label TEXT NOT NULL DEFAULT '',
color INTEGER NOT NULL DEFAULT 4283215696, created_at TEXT NOT NULL)
''');
await db.execute('''
CREATE TABLE scratchpads (
id TEXT PRIMARY KEY, document_id TEXT UNIQUE NOT NULL,
strokes_json TEXT NOT NULL DEFAULT '[]', created_at TEXT NOT NULL,
updated_at TEXT NOT NULL)
''');
await db.execute('''
CREATE TABLE ink (
id TEXT PRIMARY KEY, host_kind TEXT NOT NULL, host_id TEXT NOT NULL,
stroke_json TEXT NOT NULL, ordinal INTEGER NOT NULL,
updated_at INTEGER NOT NULL)
''');
await db.execute('''
CREATE TABLE scratch_links (
id TEXT PRIMARY KEY, document_id TEXT NOT NULL,
page_index INTEGER NOT NULL, nx REAL NOT NULL, ny REAL NOT NULL,
created_at TEXT NOT NULL)
''');
await db.execute('''
CREATE TABLE notes (
id TEXT PRIMARY KEY, title TEXT NOT NULL, created_at TEXT NOT NULL,
updated_at TEXT NOT NULL, tags TEXT NOT NULL DEFAULT '[]')
''');
await db.execute('''
CREATE TABLE strokes (
id TEXT PRIMARY KEY, note_id TEXT NOT NULL, tool TEXT NOT NULL,
color INTEGER NOT NULL, stroke_width REAL NOT NULL,
created_at TEXT NOT NULL, points TEXT NOT NULL)
''');
},
),
);
}
/// A minimal EditorStroke JSON (the exact shape `ink.stroke_json` stores).
String _editorStrokeJson(String id) => jsonEncode({
'id': id,
'points': [
{'x': 0.1, 'y': 0.2, 'pressure': 0.5, 'tilt': 0.0},
{'x': 0.3, 'y': 0.4, 'pressure': 0.6, 'tilt': 0.0},
],
'tool': 'pen',
'color': 0xFF000000,
'width': 0.01,
});
/// A minimal InkStroke JSON (the exact shape `scratchpads.strokes_json` and
/// `strokes.points` produce).
Map<String, dynamic> _inkStrokeJson(String id,
{double x = 100, double y = 200}) {
return {
'id': id,
'points': [
{
'x': x,
'y': y,
'pressure': 0.5,
'tilt': 0.0,
'timestamp': 0,
'pointerDeviceKind': 'stylus',
},
],
'tool': 'pen',
'color': 0xFF112233,
'strokeWidth': 3.0,
'createdAt': '2024-01-01T00:00:00.000Z',
};
}
void main() {
TestWidgetsFlutterBinding.ensureInitialized();
sqfliteFfiInit();
late Directory tempRoot; // holds both the legacy db and the vault
late Directory vaultDir;
late Directory sourceDir; // where the "original" source file lives
late String legacyDbPath;
late VaultService vault;
setUp(() async {
SharedPreferences.setMockInitialValues({});
tempRoot = await Directory.systemTemp.createTemp('migrator_test_');
vaultDir = Directory(p.join(tempRoot.path, 'vault'));
await vaultDir.create(recursive: true);
sourceDir = Directory(p.join(tempRoot.path, 'src'));
await sourceDir.create(recursive: true);
legacyDbPath = p.join(tempRoot.path, 'badnote.db');
final prefs = await SharedPreferences.getInstance();
vault = VaultService.forTest(prefs);
await vault.setVaultRoot(vaultDir.path);
});
tearDown(() async {
if (await tempRoot.exists()) await tempRoot.delete(recursive: true);
});
/// Seed the legacy DB with one file-backed document and one note. Returns the
/// on-disk source file path used for the document (so callers can delete it to
/// exercise the missing-source case).
Future<String> seedLegacy({bool createSourceFile = true}) async {
final srcPath = p.join(sourceDir.path, 'Lecture.pdf');
if (createSourceFile) {
await File(srcPath).writeAsString('PDFDATA');
}
final db = await _openLegacyDb(legacyDbPath);
try {
const docId = 'doc-1';
await db.insert('documents', {
'id': docId,
'filename': 'Lecture.pdf',
'doc_type': 'pdf',
'file_path': srcPath,
'page_count': 5,
'rotation': 90,
'created_at': '2024-01-01T00:00:00.000Z',
'updated_at': '2024-02-01T00:00:00.000Z',
});
// Ink on page 0 and page 3.
await db.insert('ink', {
'id': 's0',
'host_kind': 'page',
'host_id': 'doc:$docId:page:0',
'stroke_json': _editorStrokeJson('s0'),
'ordinal': 0,
'updated_at': 0,
});
await db.insert('ink', {
'id': 's3',
'host_kind': 'page',
'host_id': 'doc:$docId:page:3',
'stroke_json': _editorStrokeJson('s3'),
'ordinal': 0,
'updated_at': 0,
});
// A bookmark.
await db.insert('bookmarks', {
'id': 'bm-1',
'document_id': docId,
'page_number': 5,
'label': 'Proof',
'color': 4283215696,
'created_at': '2024-01-01T00:00:00.000Z',
});
// A scratch link + its private scratchpad (keyed by the anchor id).
await db.insert('scratch_links', {
'id': 'anchor-1',
'document_id': docId,
'page_index': 7,
'nx': 0.83,
'ny': 0.41,
'created_at': '2024-01-01T00:00:00.000Z',
});
await db.insert('scratchpads', {
'id': 'sp-1',
'document_id': 'anchor-1',
'strokes_json': jsonEncode([_inkStrokeJson('pad-stroke')]),
'created_at': '2024-01-01T00:00:00.000Z',
'updated_at': '2024-01-01T00:00:00.000Z',
});
// A legacy per-page annotation blob (dead path → preserved verbatim).
await db.insert('annotations', {
'id': 'ann-1',
'uuid': 'u-1',
'document_id': docId,
'page_number': 2,
'annotation_json': '{"legacy":"blob"}',
'created_at': '2024-01-01T00:00:00.000Z',
'updated_at': '2024-01-01T00:00:00.000Z',
});
// A free-ink note with one freehand stroke (absolute px on the note page).
await db.insert('notes', {
'id': 'note-1',
'title': 'My Algebra Notes',
'created_at': '2024-03-01T00:00:00.000Z',
'updated_at': '2024-03-02T00:00:00.000Z',
'tags': '[]',
});
await db.insert('strokes', {
'id': 'ns-1',
'note_id': 'note-1',
'tool': 'pen',
'color': 0xFF112233,
'stroke_width': 3.0,
'created_at': '2024-03-01T00:00:00.000Z',
'points': jsonEncode([
{
'x': 500.0,
'y': 707.0,
'pressure': 0.5,
'tilt': 0.0,
'timestamp': 0,
'pointerDeviceKind': 'stylus',
}
]),
});
} finally {
await db.close();
}
return srcPath;
}
test('migrates a document: file copied, all tables → sidecar fields', () async {
await seedLegacy();
final report = await SqliteToSidecarMigrator(vault, legacyDbPath: legacyDbPath)
.run();
expect(report.legacyDbFound, isTrue);
expect(report.documentsMigrated, 1);
expect(report.notesMigrated, 1);
expect(report.missingSources, isEmpty);
// The source file was copied into a notebook folder.
final docFolder = Directory(p.join(vaultDir.path, 'Lecture'));
expect(docFolder.existsSync(), isTrue);
final copiedPdf = File(p.join(docFolder.path, 'Lecture.pdf'));
expect(copiedPdf.existsSync(), isTrue);
expect(copiedPdf.readAsStringSync(), 'PDFDATA');
// The sidecar holds the migrated annotations.
final sidecar = await SidecarStore.read(
File(p.join(docFolder.path, 'Lecture.pdf.badnote.json')),
);
expect(sidecar, isNotNull);
expect(sidecar!.docType, 'pdf');
expect(sidecar.pageCount, 5);
expect(sidecar.rotation, 90);
expect(sidecar.legacyId, 'doc-1');
// ink → strokes[pageIndex]
expect(sidecar.strokes.keys.toSet(), {0, 3});
expect(sidecar.strokes[0]!.single.id, 's0');
expect(sidecar.strokes[3]!.single.id, 's3');
// bookmarks → bookmarks
expect(sidecar.bookmarks.single.label, 'Proof');
expect(sidecar.bookmarks.single.pageNumber, 5);
// scratch_links + scratchpads → scratchLinks[].link + .scratchpad
expect(sidecar.scratchLinks.single.link.id, 'anchor-1');
expect(sidecar.scratchLinks.single.link.pageIndex, 7);
expect(sidecar.scratchLinks.single.scratchpad.strokes.single.id,
'pad-stroke');
// annotations → legacyAnnotations (preserved verbatim, not dropped)
expect(sidecar.legacyAnnotations[2], '{"legacy":"blob"}');
// No legacy highlights → empty.
expect(sidecar.highlights, isEmpty);
});
test('migrates a note: standalone notebook sidecar with normalized strokes',
() async {
await seedLegacy();
await SqliteToSidecarMigrator(vault, legacyDbPath: legacyDbPath).run();
final notes = await vault.scanNotes();
expect(notes.length, 1);
expect(notes.single.title, 'My Algebra Notes');
final sidecar =
await SidecarStore.read(File('${notes.single.notePath}.badnote.json'));
expect(sidecar, isNotNull);
expect(sidecar!.docType, 'notebook');
expect(sidecar.title, 'My Algebra Notes');
expect(sidecar.legacyId, 'note-1');
// The legacy InkStroke (absolute px on the note logical page) was
// normalized onto page 0 exactly as the runtime note editor stores it.
final migrated = sidecar.strokes[0]!.single;
expect(migrated.id, 'ns-1');
expect(migrated.points.first.x, closeTo(500.0 / kNoteLogicalPage.width, 1e-9));
expect(
migrated.points.first.y, closeTo(707.0 / kNoteLogicalPage.height, 1e-9));
});
test('preserves the legacy DB as *.premigration (never deletes)', () async {
await seedLegacy();
expect(File(legacyDbPath).existsSync(), isTrue);
final report = await SqliteToSidecarMigrator(vault, legacyDbPath: legacyDbPath)
.run();
// Original gone, preserved copy present.
expect(File(legacyDbPath).existsSync(), isFalse);
final preserved = File('$legacyDbPath.premigration');
expect(preserved.existsSync(), isTrue);
expect(report.legacyDbPreservedPath, preserved.path);
// The preserved DB is non-empty (real data, not truncated).
expect(preserved.lengthSync(), greaterThan(0));
});
test('is idempotent: a re-run migrates nothing new, no duplicate folders',
() async {
await seedLegacy();
await SqliteToSidecarMigrator(vault, legacyDbPath: legacyDbPath).run();
// The legacy DB is now renamed; a naive second run against the ORIGINAL
// path is a no-op (file gone). But the migrator must also be idempotent if
// pointed back at the preserved DB — re-run against it and assert nothing
// duplicates.
final preservedPath = '$legacyDbPath.premigration';
final report = await SqliteToSidecarMigrator(vault,
legacyDbPath: preservedPath)
.run();
expect(report.documentsMigrated, 0, reason: 'already migrated');
expect(report.notesMigrated, 0, reason: 'already migrated');
expect(report.documentsSkipped, 1);
expect(report.notesSkipped, 1);
// Exactly one document folder and one note folder — no `Lecture 2` /
// `My Algebra Notes 2` duplicates.
final folders = vaultDir
.listSync()
.whereType<Directory>()
.map((d) => p.basename(d.path))
.where((n) => !n.startsWith('.'))
.toList();
expect(folders.toSet(), {'Lecture', 'My Algebra Notes'});
});
test('missing source file: annotations still migrate into a folder', () async {
await seedLegacy(createSourceFile: false);
final report = await SqliteToSidecarMigrator(vault, legacyDbPath: legacyDbPath)
.run();
expect(report.documentsMigrated, 1);
expect(report.missingSources, ['Lecture.pdf']);
// A folder exists holding ONLY the sidecar (no copied source file).
final docFolder = Directory(p.join(vaultDir.path, 'Lecture'));
expect(docFolder.existsSync(), isTrue);
expect(File(p.join(docFolder.path, 'Lecture.pdf')).existsSync(), isFalse);
final sidecar = await SidecarStore.read(
File(p.join(docFolder.path, 'Lecture.pdf.badnote.json')),
);
expect(sidecar, isNotNull);
expect(sidecar!.strokes.keys.toSet(), {0, 3}, reason: 'ink preserved');
expect(sidecar.scratchLinks.single.scratchpad.strokes.single.id,
'pad-stroke');
});
test('fresh install (no legacy DB) is a safe no-op', () async {
final report = await SqliteToSidecarMigrator(vault,
legacyDbPath: p.join(tempRoot.path, 'does-not-exist.db'))
.run();
expect(report.legacyDbFound, isFalse);
expect(report.didAnything, isFalse);
expect(report.legacyDbPreservedPath, isNull);
expect(vaultDir.listSync().whereType<Directory>().length, 0);
});
test('throws when the vault root is invalid', () async {
await vault.setVaultRoot(p.join(tempRoot.path, 'gone$_sep'));
expect(
() => SqliteToSidecarMigrator(vault, legacyDbPath: legacyDbPath).run(),
throwsStateError,
);
});
}