From 597aba12544c51f9cda739a0991317c8c3b3460f Mon Sep 17 00:00:00 2001 From: ngfchl Date: Sat, 18 Jul 2026 13:02:33 +0800 Subject: [PATCH] feat: persist cloud metadata and incremental media scans --- lib/core/storage/file_metadata_cache.dart | 132 ++++++++++ lib/core/storage/storage_manager.dart | 3 + lib/providers/file_provider.dart | 54 ++++ lib/providers/media_library_provider.dart | 299 ++++++++++++++++------ 4 files changed, 409 insertions(+), 79 deletions(-) create mode 100644 lib/core/storage/file_metadata_cache.dart diff --git a/lib/core/storage/file_metadata_cache.dart b/lib/core/storage/file_metadata_cache.dart new file mode 100644 index 0000000..627706d --- /dev/null +++ b/lib/core/storage/file_metadata_cache.dart @@ -0,0 +1,132 @@ +import '../../models/cloud_file.dart'; +import 'storage_manager.dart'; + +/// Permanent cloud-file indexes used by the workspace and media scanner. +/// +/// Folder records retain ordered child IDs as their index; the snapshots make +/// an offline scan possible before the server needs to be consulted again. +class FileMetadataCache { + static const _rootFolderID = '@root'; + static Future _writes = Future.value(); + + static String _folderKey(String? folderID) => folderID ?? _rootFolderID; + + static Future cacheFolderChildren( + String? folderID, + List files, + ) { + return _enqueue(() async { + final folders = _map(StorageKeys.folderChildrenIndex); + final fileIndex = _map(StorageKeys.fileGcidIndex); + final details = _map(StorageKeys.gcidDetails); + folders[_folderKey(folderID)] = { + 'childIds': files.map((file) => file.id).toList(), + 'children': files.map((file) => file.toJson()).toList(), + }; + _indexFiles(files, fileIndex, details); + await StorageManager.set(StorageKeys.folderChildrenIndex, folders); + await StorageManager.set(StorageKeys.fileGcidIndex, fileIndex); + await StorageManager.set(StorageKeys.gcidDetails, details); + }); + } + + static Future cacheFiles(List files) { + return _enqueue(() async { + final fileIndex = _map(StorageKeys.fileGcidIndex); + final details = _map(StorageKeys.gcidDetails); + _indexFiles(files, fileIndex, details); + await StorageManager.set(StorageKeys.fileGcidIndex, fileIndex); + await StorageManager.set(StorageKeys.gcidDetails, details); + }); + } + + static Future updateFolderChildren( + String? folderID, { + Iterable removeIDs = const [], + Iterable addOrReplace = const [], + bool invalidate = false, + }) { + return _enqueue(() async { + final folders = _map(StorageKeys.folderChildrenIndex); + final key = _folderKey(folderID); + if (invalidate) { + folders.remove(key); + await StorageManager.set(StorageKeys.folderChildrenIndex, folders); + return; + } + final entry = folders[key]; + if (entry is! Map || entry['children'] is! List) return; + final removed = removeIDs.toSet(); + final children = (entry['children'] as List) + .whereType() + .map((value) => CloudFile.fromJson(Map.from(value))) + .where((file) => !removed.contains(file.id)) + .toList(); + final replacement = {for (final file in addOrReplace) file.id: file}; + children.removeWhere((file) => replacement.containsKey(file.id)); + children.addAll(replacement.values); + folders[key] = { + 'childIds': children.map((file) => file.id).toList(), + 'children': children.map((file) => file.toJson()).toList(), + }; + final fileIndex = _map(StorageKeys.fileGcidIndex); + final details = _map(StorageKeys.gcidDetails); + _indexFiles(replacement.values.toList(), fileIndex, details); + await StorageManager.set(StorageKeys.folderChildrenIndex, folders); + await StorageManager.set(StorageKeys.fileGcidIndex, fileIndex); + await StorageManager.set(StorageKeys.gcidDetails, details); + }); + } + + static List? folderChildren(String? folderID) { + final folders = _map(StorageKeys.folderChildrenIndex); + final entry = folders[_folderKey(folderID)]; + if (entry is! Map || entry['children'] is! List) return null; + try { + return (entry['children'] as List) + .whereType() + .map((value) => CloudFile.fromJson(Map.from(value))) + .toList(); + } catch (_) { + return null; + } + } + + static CloudFile? file(String fileID) { + final fileIndex = _map(StorageKeys.fileGcidIndex); + final gcid = fileIndex[fileID]?.toString(); + if (gcid == null || gcid.isEmpty) return null; + final details = _map(StorageKeys.gcidDetails); + final value = details[gcid]; + if (value is! Map) return null; + try { + return CloudFile.fromJson(Map.from(value)); + } catch (_) { + return null; + } + } + + static Map _map(String key) { + final raw = StorageManager.get(key); + if (raw is! Map) return {}; + return raw.map((mapKey, value) => MapEntry(mapKey.toString(), value)); + } + + static void _indexFiles( + List files, + Map fileIndex, + Map details, + ) { + for (final file in files) { + final gcid = file.gcid?.trim(); + if (gcid == null || gcid.isEmpty) continue; + fileIndex[file.id] = gcid; + details[gcid] = file.toJson(); + } + } + + static Future _enqueue(Future Function() operation) { + _writes = _writes.then((_) => operation()); + return _writes; + } +} diff --git a/lib/core/storage/storage_manager.dart b/lib/core/storage/storage_manager.dart index a2b9f78..7e2b274 100644 --- a/lib/core/storage/storage_manager.dart +++ b/lib/core/storage/storage_manager.dart @@ -17,6 +17,9 @@ class StorageKeys { static const String mediaCategoryRules = 'guangya.mediaCategoryRules'; static const String fileListCache = 'guangya.fileListCache'; static const String fileMetadataCache = 'guangya.fileMetadataCache'; + static const String fileGcidIndex = 'guangya.fileGcidIndex'; + static const String gcidDetails = 'guangya.gcidDetails'; + static const String folderChildrenIndex = 'guangya.folderChildrenIndex'; static const String httpProxyHost = 'guangya.httpProxyHost'; static const String httpProxyPort = 'guangya.httpProxyPort'; static const String mediaScanConcurrency = 'guangya.mediaScanConcurrency'; diff --git a/lib/providers/file_provider.dart b/lib/providers/file_provider.dart index d930a15..d73c216 100644 --- a/lib/providers/file_provider.dart +++ b/lib/providers/file_provider.dart @@ -4,6 +4,7 @@ import 'package:flutter/services.dart'; import 'package:flutter_riverpod/legacy.dart'; import 'package:url_launcher/url_launcher.dart'; import '../api/guangya_api.dart'; +import '../core/storage/file_metadata_cache.dart'; import '../core/storage/storage_manager.dart'; import '../models/cloud_file.dart'; @@ -178,6 +179,7 @@ class FileNotifier extends StateNotifier { final totalPages = _extractTotalPages(result, extracted.length); state = state.copyWith(files: extracted, totalPages: totalPages); await _writeCachedFiles(cacheKey, extracted); + await FileMetadataCache.cacheFolderChildren(resolvedParentID, extracted); unawaited(_enrichFolderSizes(extracted, generation)); } catch (e) { state = state.copyWith(errorMessage: e.toString()); @@ -519,6 +521,7 @@ class FileNotifier extends StateNotifier { try { await _api!.fsCreateDir(name, parentID: _currentParentID); state = state.copyWith(statusMessage: '文件夹已创建'); + await _invalidateFolderCaches(_currentParentID); await loadFiles(parentID: _currentParentID); } catch (e) { state = state.copyWith(errorMessage: e.toString(), clearStatus: true); @@ -530,6 +533,11 @@ class FileNotifier extends StateNotifier { try { await _api!.fsRename(file.id, newName); state = state.copyWith(statusMessage: '重命名成功'); + await FileMetadataCache.updateFolderChildren( + _currentParentID, + addOrReplace: [file.copyWith(name: newName)], + ); + await _invalidateListCache(_currentParentID); await loadFiles(parentID: _currentParentID); } catch (e) { state = state.copyWith(errorMessage: e.toString()); @@ -541,6 +549,11 @@ class FileNotifier extends StateNotifier { state = state.copyWith(statusMessage: '正在删除…'); try { await _api!.fsDelete(files.map((f) => f.id).toList()); + await FileMetadataCache.updateFolderChildren( + _currentParentID, + removeIDs: files.map((file) => file.id), + ); + await _invalidateListCache(_currentParentID); state = state.copyWith( statusMessage: '已删除 ${files.length} 个项目', selectedIDs: {}, @@ -555,6 +568,11 @@ class FileNotifier extends StateNotifier { if (_api == null) return; try { await _api!.fsRecycle(files.map((f) => f.id).toList()); + await FileMetadataCache.updateFolderChildren( + _currentParentID, + removeIDs: files.map((file) => file.id), + ); + await _invalidateListCache(_currentParentID); state = state.copyWith(statusMessage: '已移入回收站', selectedIDs: {}); await loadFiles(parentID: _currentParentID); } catch (e) { @@ -566,6 +584,7 @@ class FileNotifier extends StateNotifier { if (_api == null) return; try { await _api!.fsClearRecycleBin(); + await _invalidateFolderCaches(_currentParentID); state = state.copyWith(statusMessage: '回收站已清空'); await loadFiles(); } catch (e) { @@ -590,9 +609,18 @@ class FileNotifier extends StateNotifier { final ids = state.clipboard!.map((f) => f.id).toList(); if (state.clipboardIsMove) { await _api!.fsMove(ids, parentID: _currentParentID); + await FileMetadataCache.updateFolderChildren( + _currentParentID, + addOrReplace: state.clipboard!, + ); } else { await _api!.fsCopy(ids, parentID: _currentParentID); + await FileMetadataCache.updateFolderChildren( + _currentParentID, + invalidate: true, + ); } + await _invalidateListCache(_currentParentID); state = state.copyWith(statusMessage: '操作完成'); await loadFiles(parentID: _currentParentID); } catch (e) { @@ -646,6 +674,7 @@ class FileNotifier extends StateNotifier { state = state.copyWith(statusMessage: '正在恢复 ${files.length} 个项目…'); await _api!.fsRecycle(files.map((file) => file.id).toList()); state = state.copyWith(statusMessage: '已恢复 ${files.length} 个项目'); + await _invalidateFolderCaches(_currentParentID); await loadFiles(parentID: _currentParentID); } catch (e) { state = state.copyWith(errorMessage: e.toString()); @@ -699,6 +728,7 @@ class FileNotifier extends StateNotifier { completed += 1; } state = state.copyWith(statusMessage: '已上传 $completed 个文件'); + await _invalidateFolderCaches(targetParentID); await loadFiles(parentID: _currentParentID); } catch (e) { state = state.copyWith(errorMessage: e.toString()); @@ -713,6 +743,16 @@ class FileNotifier extends StateNotifier { files.map((file) => file.id).toList(), parentID: parentID, ); + await FileMetadataCache.updateFolderChildren( + _currentParentID, + removeIDs: files.map((file) => file.id), + ); + await FileMetadataCache.updateFolderChildren( + parentID, + addOrReplace: files, + ); + await _invalidateListCache(_currentParentID); + if (parentID != _currentParentID) await _invalidateListCache(parentID); state = state.copyWith(statusMessage: '移动完成', selectedIDs: {}); await loadFiles(parentID: _currentParentID); } catch (e) { @@ -728,6 +768,20 @@ class FileNotifier extends StateNotifier { state = state.copyWith(clearStatus: true); } + Future _invalidateFolderCaches(String? parentID) async { + await FileMetadataCache.updateFolderChildren(parentID, invalidate: true); + await _invalidateListCache(parentID); + } + + Future _invalidateListCache(String? parentID) async { + final raw = StorageManager.get(StorageKeys.fileListCache); + if (raw is! Map) return; + final cache = Map.from(raw); + final prefix = '${state.section.name}:${parentID ?? 'root'}:'; + cache.removeWhere((key, _) => key.toString().startsWith(prefix)); + await StorageManager.set(StorageKeys.fileListCache, cache); + } + // ── Helpers ───────────────────────────────────────────────────── List _extractFiles(Map json) { diff --git a/lib/providers/media_library_provider.dart b/lib/providers/media_library_provider.dart index 5105eb8..5d08471 100644 --- a/lib/providers/media_library_provider.dart +++ b/lib/providers/media_library_provider.dart @@ -3,6 +3,7 @@ import 'dart:async'; import 'package:flutter_riverpod/legacy.dart'; import '../api/guangya_api.dart'; +import '../core/storage/file_metadata_cache.dart'; import '../core/storage/storage_manager.dart'; import '../models/cloud_file.dart'; import '../models/media_library.dart'; @@ -191,84 +192,86 @@ class MediaLibraryNotifier extends StateNotifier { ); try { - final discovered = []; - for (final source in library.sources) { - if (_cancelScan) break; - state = state.copyWith( - progress: MediaLibraryScanProgress( - phase: '扫描 ${source.path}', - completed: discovered.length, - ), - ); - final files = await _scanSource( - source.rootID, - source.path, - recursive: library.recursive, - minimumSizeBytes: library.minimumSizeMB * 1024 * 1024, - ); - discovered.addAll(files); - } - - final unique = {}; + final initialItems = _loadAllItems() + ..removeWhere((item) => item.libraryID == library.id); + final unique = { + for (final item in _loadItems(library.id)) item.file.id: item, + }; + state = state.copyWith(items: unique.values.toList()); final tmdbApiKey = StorageManager.get(StorageKeys.tmdbApiKey) ?? ''; final tmdbProxyHost = StorageManager.get(StorageKeys.tmdbProxyHost) ?? ''; final tmdbProxyPort = StorageManager.get(StorageKeys.tmdbProxyPort) ?? ''; - final input = { - for (final file in discovered) file.id: file, - }.values.toList(); - var nextIndex = 0; var completed = 0; Future pendingPersistence = Future.value(); - final initialItems = _loadAllItems() - ..removeWhere((item) => item.libraryID == library.id); - Future worker() async { - while (!_cancelScan) { - if (nextIndex >= input.length) return; - final file = input[nextIndex++]; - final fallback = MediaLibraryItem.fromFile(library.id, file); - final item = await _recognizeMediaItem( - fallback, - tmdbApiKey, - proxyHost: tmdbProxyHost, - proxyPort: tmdbProxyPort, - ); - unique[file.id] = item; - completed += 1; - final visible = unique.values.toList() - ..sort( - (a, b) => a.title.toLowerCase().compareTo(b.title.toLowerCase()), + Future indexBatch(List files) async { + final pending = files + .where((file) => !unique.containsKey(file.id)) + .toList(); + if (pending.isEmpty || _cancelScan) return; + var next = 0; + Future worker() async { + while (!_cancelScan && next < pending.length) { + final file = pending[next++]; + final fallback = MediaLibraryItem.fromFile(library.id, file); + final item = await _recognizeMediaItem( + fallback, + tmdbApiKey, + proxyHost: tmdbProxyHost, + proxyPort: tmdbProxyPort, ); - // Serialize writes so an earlier, smaller snapshot cannot overwrite - // a later batch while recognition workers finish out of order. - pendingPersistence = pendingPersistence.then( - (_) => _saveAllItems([...initialItems, ...unique.values]), - ); - await pendingPersistence; - state = state.copyWith( - items: visible, - progress: MediaLibraryScanProgress( - phase: tmdbApiKey.isEmpty ? '正在建立本地索引' : '正在识别 ${file.name}', - completed: completed, - total: input.length, - ), - ); + unique[file.id] = item; + completed += 1; + final visible = unique.values.toList() + ..sort( + (a, b) => + a.title.toLowerCase().compareTo(b.title.toLowerCase()), + ); + pendingPersistence = pendingPersistence.then( + (_) => _saveAllItems([...initialItems, ...unique.values]), + ); + await pendingPersistence; + state = state.copyWith( + items: visible, + progress: MediaLibraryScanProgress( + phase: tmdbApiKey.isEmpty ? '正在建立本地索引' : '正在识别 ${file.name}', + completed: completed, + ), + ); + } } + + final concurrency = + (int.tryParse( + StorageManager.get( + StorageKeys.mediaScanConcurrency, + ) ?? + '3', + ) ?? + 3) + .clamp(1, 20); + await Future.wait(List.generate(concurrency, (_) => worker())); } - final concurrency = - (int.tryParse( - StorageManager.get( - StorageKeys.mediaScanConcurrency, - ) ?? - '3', - ) ?? - 3) - .clamp(1, 20); - await Future.wait(List.generate(concurrency, (_) => worker())); + for (final source in library.sources) { + if (_cancelScan) break; + state = state.copyWith( + progress: MediaLibraryScanProgress( + phase: '扫描 ${source.path}', + completed: completed, + ), + ); + await _scanSource( + source.rootID, + source.path, + recursive: library.recursive, + minimumSizeBytes: library.minimumSizeMB * 1024 * 1024, + onMediaFiles: indexBatch, + ); + } await pendingPersistence; final items = unique.values.toList() ..sort( @@ -390,15 +393,16 @@ class MediaLibraryNotifier extends StateNotifier { return int.tryParse(value?.toString() ?? ''); } - Future> _scanSource( + Future _scanSource( String? rootID, String rootPath, { required bool recursive, required int minimumSizeBytes, + required Future Function(List files) onMediaFiles, }) async { final folders = <_ScanFolder>[_ScanFolder(rootID, rootPath)]; final visited = {}; - final mediaFiles = []; + var discovered = 0; while (folders.isNotEmpty && !_cancelScan) { final folder = folders.removeAt(0); @@ -406,38 +410,175 @@ class MediaLibraryNotifier extends StateNotifier { if (!visited.add(visitKey)) continue; var page = 0; + final cachedFolder = FileMetadataCache.folderChildren(folder.id); + final folderSnapshot = []; while (!_cancelScan) { - final response = await _api!.fsFiles( - parentID: folder.id, - page: page, - pageSize: 200, - orderBy: 0, - sortType: 0, - ); - final files = _extractFiles(response); + late List files; + if (cachedFolder != null) { + files = cachedFolder; + } else { + final response = await _api!.fsFiles( + parentID: folder.id, + page: page, + pageSize: 200, + orderBy: 0, + sortType: 0, + ); + files = _extractFiles(response); + files = await _enrichAndCacheFiles(files); + folderSnapshot.addAll(files); + } + final mediaBatch = []; for (final file in files) { if (file.isDirectory) { if (recursive) { folders.add(_ScanFolder(file.id, '${folder.path}/${file.name}')); } } else if (file.isVideo && (file.size ?? 0) >= minimumSizeBytes) { - mediaFiles.add(_withPath(file, '${folder.path}/${file.name}')); + mediaBatch.add(_withPath(file, '${folder.path}/${file.name}')); } } + discovered += mediaBatch.length; + if (mediaBatch.isNotEmpty) await onMediaFiles(mediaBatch); state = state.copyWith( progress: MediaLibraryScanProgress( phase: '扫描 ${folder.path}', - completed: mediaFiles.length, + completed: discovered, total: folders.length + visited.length, ), ); - if (files.length < 200) break; + if (cachedFolder != null || files.length < 200) { + if (cachedFolder == null) { + await FileMetadataCache.cacheFolderChildren( + folder.id, + folderSnapshot, + ); + } + break; + } page += 1; } } - return mediaFiles; + } + + Future> _enrichAndCacheFiles(List files) async { + final resolved = []; + final pending = []; + for (final file in files) { + final cached = FileMetadataCache.file(file.id); + if (cached != null) { + resolved.add( + file.copyWith( + size: cached.size, + gcid: cached.gcid, + modifiedAt: cached.modifiedAt, + cloudPath: file.cloudPath, + ), + ); + } else if (!file.isDirectory && + (file.gcid == null || file.gcid!.isEmpty)) { + pending.add(file); + } else { + resolved.add(file); + } + } + if (pending.isEmpty) { + await FileMetadataCache.cacheFiles(resolved); + return resolved; + } + + final enriched = { + for (final file in resolved) file.id: file, + }; + var next = 0; + Future worker() async { + while (next < pending.length) { + final file = pending[next++]; + try { + final detail = await _api!.fsDetail(file.id); + final detailFile = _extractFiles( + detail, + ).where((candidate) => candidate.id == file.id).firstOrNull; + final gcid = + detailFile?.gcid ?? + _findStringDeep(detail, const [ + 'gcid', + 'gcId', + 'gcidValue', + 'hash', + ]); + final size = + detailFile?.size ?? + _findIntDeep(detail, const [ + 'size', + 'fileSize', + 'resSize', + 'totalSize', + ]); + enriched[file.id] = file.copyWith(gcid: gcid, size: size); + } catch (_) { + enriched[file.id] = file; + } + } + } + + await Future.wait(List.generate(6, (_) => worker())); + final values = [for (final file in files) enriched[file.id] ?? file]; + await FileMetadataCache.cacheFiles(values); + return values; + } + + String? _findStringDeep(Map value, List keys) { + for (final entry in value.entries) { + if (keys.contains(entry.key) && entry.value != null) { + final text = entry.value.toString().trim(); + if (text.isNotEmpty) return text; + } + if (entry.value is Map) { + final found = _findStringDeep( + Map.from(entry.value), + keys, + ); + if (found != null) return found; + } else if (entry.value is List) { + for (final child in entry.value as List) { + if (child is Map) { + final found = _findStringDeep( + Map.from(child), + keys, + ); + if (found != null) return found; + } + } + } + } + return null; + } + + int? _findIntDeep(Map value, List keys) { + for (final entry in value.entries) { + if (keys.contains(entry.key)) { + final parsed = int.tryParse(entry.value?.toString() ?? ''); + if (parsed != null) return parsed; + } + if (entry.value is Map) { + final found = _findIntDeep( + Map.from(entry.value), + keys, + ); + if (found != null) return found; + } else if (entry.value is List) { + for (final child in entry.value as List) { + if (child is Map) { + final found = _findIntDeep(Map.from(child), keys); + if (found != null) return found; + } + } + } + } + return null; } CloudFile _withPath(CloudFile file, String path) {