feat: persist cloud metadata and incremental media scans

2026-07-18 13:02:33 +08:00
parent a969eba473
commit 597aba1254
4 changed files with 409 additions and 79 deletions
@@ -0,0 +1,132 @@
+import '../../models/cloud_file.dart';
+import 'storage_manager.dart';
+
+/// Permanent cloud-file indexes used by the workspace and media scanner.
+///
+/// Folder records retain ordered child IDs as their index; the snapshots make
+/// an offline scan possible before the server needs to be consulted again.
+class FileMetadataCache {
+  static const _rootFolderID = '@root';
+  static Future<void> _writes = Future.value();
+
+  static String _folderKey(String? folderID) => folderID ?? _rootFolderID;
+
+  static Future<void> cacheFolderChildren(
+    String? folderID,
+    List<CloudFile> files,
+  ) {
+    return _enqueue(() async {
+      final folders = _map(StorageKeys.folderChildrenIndex);
+      final fileIndex = _map(StorageKeys.fileGcidIndex);
+      final details = _map(StorageKeys.gcidDetails);
+      folders[_folderKey(folderID)] = {
+        'childIds': files.map((file) => file.id).toList(),
+        'children': files.map((file) => file.toJson()).toList(),
+      };
+      _indexFiles(files, fileIndex, details);
+      await StorageManager.set(StorageKeys.folderChildrenIndex, folders);
+      await StorageManager.set(StorageKeys.fileGcidIndex, fileIndex);
+      await StorageManager.set(StorageKeys.gcidDetails, details);
+    });
+  }
+
+  static Future<void> cacheFiles(List<CloudFile> files) {
+    return _enqueue(() async {
+      final fileIndex = _map(StorageKeys.fileGcidIndex);
+      final details = _map(StorageKeys.gcidDetails);
+      _indexFiles(files, fileIndex, details);
+      await StorageManager.set(StorageKeys.fileGcidIndex, fileIndex);
+      await StorageManager.set(StorageKeys.gcidDetails, details);
+    });
+  }
+
+  static Future<void> updateFolderChildren(
+    String? folderID, {
+    Iterable<String> removeIDs = const [],
+    Iterable<CloudFile> addOrReplace = const [],
+    bool invalidate = false,
+  }) {
+    return _enqueue(() async {
+      final folders = _map(StorageKeys.folderChildrenIndex);
+      final key = _folderKey(folderID);
+      if (invalidate) {
+        folders.remove(key);
+        await StorageManager.set(StorageKeys.folderChildrenIndex, folders);
+        return;
+      }
+      final entry = folders[key];
+      if (entry is! Map || entry['children'] is! List) return;
+      final removed = removeIDs.toSet();
+      final children = (entry['children'] as List)
+          .whereType<Map>()
+          .map((value) => CloudFile.fromJson(Map<String, dynamic>.from(value)))
+          .where((file) => !removed.contains(file.id))
+          .toList();
+      final replacement = {for (final file in addOrReplace) file.id: file};
+      children.removeWhere((file) => replacement.containsKey(file.id));
+      children.addAll(replacement.values);
+      folders[key] = {
+        'childIds': children.map((file) => file.id).toList(),
+        'children': children.map((file) => file.toJson()).toList(),
+      };
+      final fileIndex = _map(StorageKeys.fileGcidIndex);
+      final details = _map(StorageKeys.gcidDetails);
+      _indexFiles(replacement.values.toList(), fileIndex, details);
+      await StorageManager.set(StorageKeys.folderChildrenIndex, folders);
+      await StorageManager.set(StorageKeys.fileGcidIndex, fileIndex);
+      await StorageManager.set(StorageKeys.gcidDetails, details);
+    });
+  }
+
+  static List<CloudFile>? folderChildren(String? folderID) {
+    final folders = _map(StorageKeys.folderChildrenIndex);
+    final entry = folders[_folderKey(folderID)];
+    if (entry is! Map || entry['children'] is! List) return null;
+    try {
+      return (entry['children'] as List)
+          .whereType<Map>()
+          .map((value) => CloudFile.fromJson(Map<String, dynamic>.from(value)))
+          .toList();
+    } catch (_) {
+      return null;
+    }
+  }
+
+  static CloudFile? file(String fileID) {
+    final fileIndex = _map(StorageKeys.fileGcidIndex);
+    final gcid = fileIndex[fileID]?.toString();
+    if (gcid == null || gcid.isEmpty) return null;
+    final details = _map(StorageKeys.gcidDetails);
+    final value = details[gcid];
+    if (value is! Map) return null;
+    try {
+      return CloudFile.fromJson(Map<String, dynamic>.from(value));
+    } catch (_) {
+      return null;
+    }
+  }
+
+  static Map<String, dynamic> _map(String key) {
+    final raw = StorageManager.get<dynamic>(key);
+    if (raw is! Map) return <String, dynamic>{};
+    return raw.map((mapKey, value) => MapEntry(mapKey.toString(), value));
+  }
+
+  static void _indexFiles(
+    List<CloudFile> files,
+    Map<String, dynamic> fileIndex,
+    Map<String, dynamic> details,
+  ) {
+    for (final file in files) {
+      final gcid = file.gcid?.trim();
+      if (gcid == null || gcid.isEmpty) continue;
+      fileIndex[file.id] = gcid;
+      details[gcid] = file.toJson();
+    }
+  }
+
+  static Future<void> _enqueue(Future<void> Function() operation) {
+    _writes = _writes.then((_) => operation());
+    return _writes;
+  }
+}
@@ -17,6 +17,9 @@ class StorageKeys {
  static const String mediaCategoryRules = 'guangya.mediaCategoryRules';
  static const String fileListCache = 'guangya.fileListCache';
  static const String fileMetadataCache = 'guangya.fileMetadataCache';
+  static const String fileGcidIndex = 'guangya.fileGcidIndex';
+  static const String gcidDetails = 'guangya.gcidDetails';
+  static const String folderChildrenIndex = 'guangya.folderChildrenIndex';
  static const String httpProxyHost = 'guangya.httpProxyHost';
  static const String httpProxyPort = 'guangya.httpProxyPort';
  static const String mediaScanConcurrency = 'guangya.mediaScanConcurrency';
@@ -4,6 +4,7 @@ import 'package:flutter/services.dart';
 import 'package:flutter_riverpod/legacy.dart';
 import 'package:url_launcher/url_launcher.dart';
 import '../api/guangya_api.dart';
+import '../core/storage/file_metadata_cache.dart';
 import '../core/storage/storage_manager.dart';
 import '../models/cloud_file.dart';

@@ -178,6 +179,7 @@ class FileNotifier extends StateNotifier<FileState> {
      final totalPages = _extractTotalPages(result, extracted.length);
      state = state.copyWith(files: extracted, totalPages: totalPages);
      await _writeCachedFiles(cacheKey, extracted);
+      await FileMetadataCache.cacheFolderChildren(resolvedParentID, extracted);
      unawaited(_enrichFolderSizes(extracted, generation));
    } catch (e) {
      state = state.copyWith(errorMessage: e.toString());
@@ -519,6 +521,7 @@ class FileNotifier extends StateNotifier<FileState> {
    try {
      await _api!.fsCreateDir(name, parentID: _currentParentID);
      state = state.copyWith(statusMessage: '文件夹已创建');
+      await _invalidateFolderCaches(_currentParentID);
      await loadFiles(parentID: _currentParentID);
    } catch (e) {
      state = state.copyWith(errorMessage: e.toString(), clearStatus: true);
@@ -530,6 +533,11 @@ class FileNotifier extends StateNotifier<FileState> {
    try {
      await _api!.fsRename(file.id, newName);
      state = state.copyWith(statusMessage: '重命名成功');
+      await FileMetadataCache.updateFolderChildren(
+        _currentParentID,
+        addOrReplace: [file.copyWith(name: newName)],
+      );
+      await _invalidateListCache(_currentParentID);
      await loadFiles(parentID: _currentParentID);
    } catch (e) {
      state = state.copyWith(errorMessage: e.toString());
@@ -541,6 +549,11 @@ class FileNotifier extends StateNotifier<FileState> {
    state = state.copyWith(statusMessage: '正在删除…');
    try {
      await _api!.fsDelete(files.map((f) => f.id).toList());
+      await FileMetadataCache.updateFolderChildren(
+        _currentParentID,
+        removeIDs: files.map((file) => file.id),
+      );
+      await _invalidateListCache(_currentParentID);
      state = state.copyWith(
        statusMessage: '已删除 ${files.length} 个项目',
        selectedIDs: {},
@@ -555,6 +568,11 @@ class FileNotifier extends StateNotifier<FileState> {
    if (_api == null) return;
    try {
      await _api!.fsRecycle(files.map((f) => f.id).toList());
+      await FileMetadataCache.updateFolderChildren(
+        _currentParentID,
+        removeIDs: files.map((file) => file.id),
+      );
+      await _invalidateListCache(_currentParentID);
      state = state.copyWith(statusMessage: '已移入回收站', selectedIDs: {});
      await loadFiles(parentID: _currentParentID);
    } catch (e) {
@@ -566,6 +584,7 @@ class FileNotifier extends StateNotifier<FileState> {
    if (_api == null) return;
    try {
      await _api!.fsClearRecycleBin();
+      await _invalidateFolderCaches(_currentParentID);
      state = state.copyWith(statusMessage: '回收站已清空');
      await loadFiles();
    } catch (e) {
@@ -590,9 +609,18 @@ class FileNotifier extends StateNotifier<FileState> {
      final ids = state.clipboard!.map((f) => f.id).toList();
      if (state.clipboardIsMove) {
        await _api!.fsMove(ids, parentID: _currentParentID);
+        await FileMetadataCache.updateFolderChildren(
+          _currentParentID,
+          addOrReplace: state.clipboard!,
+        );
      } else {
        await _api!.fsCopy(ids, parentID: _currentParentID);
+        await FileMetadataCache.updateFolderChildren(
+          _currentParentID,
+          invalidate: true,
+        );
      }
+      await _invalidateListCache(_currentParentID);
      state = state.copyWith(statusMessage: '操作完成');
      await loadFiles(parentID: _currentParentID);
    } catch (e) {
@@ -646,6 +674,7 @@ class FileNotifier extends StateNotifier<FileState> {
      state = state.copyWith(statusMessage: '正在恢复 ${files.length} 个项目…');
      await _api!.fsRecycle(files.map((file) => file.id).toList());
      state = state.copyWith(statusMessage: '已恢复 ${files.length} 个项目');
+      await _invalidateFolderCaches(_currentParentID);
      await loadFiles(parentID: _currentParentID);
    } catch (e) {
      state = state.copyWith(errorMessage: e.toString());
@@ -699,6 +728,7 @@ class FileNotifier extends StateNotifier<FileState> {
        completed += 1;
      }
      state = state.copyWith(statusMessage: '已上传 $completed 个文件');
+      await _invalidateFolderCaches(targetParentID);
      await loadFiles(parentID: _currentParentID);
    } catch (e) {
      state = state.copyWith(errorMessage: e.toString());
@@ -713,6 +743,16 @@ class FileNotifier extends StateNotifier<FileState> {
        files.map((file) => file.id).toList(),
        parentID: parentID,
      );
+      await FileMetadataCache.updateFolderChildren(
+        _currentParentID,
+        removeIDs: files.map((file) => file.id),
+      );
+      await FileMetadataCache.updateFolderChildren(
+        parentID,
+        addOrReplace: files,
+      );
+      await _invalidateListCache(_currentParentID);
+      if (parentID != _currentParentID) await _invalidateListCache(parentID);
      state = state.copyWith(statusMessage: '移动完成', selectedIDs: {});
      await loadFiles(parentID: _currentParentID);
    } catch (e) {
@@ -728,6 +768,20 @@ class FileNotifier extends StateNotifier<FileState> {
    state = state.copyWith(clearStatus: true);
  }

+  Future<void> _invalidateFolderCaches(String? parentID) async {
+    await FileMetadataCache.updateFolderChildren(parentID, invalidate: true);
+    await _invalidateListCache(parentID);
+  }
+
+  Future<void> _invalidateListCache(String? parentID) async {
+    final raw = StorageManager.get<dynamic>(StorageKeys.fileListCache);
+    if (raw is! Map) return;
+    final cache = Map<dynamic, dynamic>.from(raw);
+    final prefix = '${state.section.name}:${parentID ?? 'root'}:';
+    cache.removeWhere((key, _) => key.toString().startsWith(prefix));
+    await StorageManager.set(StorageKeys.fileListCache, cache);
+  }
+
  // ── Helpers ─────────────────────────────────────────────────────

  List<CloudFile> _extractFiles(Map<String, dynamic> json) {
@@ -3,6 +3,7 @@ import 'dart:async';
 import 'package:flutter_riverpod/legacy.dart';

 import '../api/guangya_api.dart';
+import '../core/storage/file_metadata_cache.dart';
 import '../core/storage/storage_manager.dart';
 import '../models/cloud_file.dart';
 import '../models/media_library.dart';
@@ -191,84 +192,86 @@ class MediaLibraryNotifier extends StateNotifier<MediaLibraryState> {
    );

    try {
-      final discovered = <CloudFile>[];
-      for (final source in library.sources) {
-        if (_cancelScan) break;
-        state = state.copyWith(
-          progress: MediaLibraryScanProgress(
-            phase: '扫描 ${source.path}',
-            completed: discovered.length,
-          ),
-        );
-        final files = await _scanSource(
-          source.rootID,
-          source.path,
-          recursive: library.recursive,
-          minimumSizeBytes: library.minimumSizeMB * 1024 * 1024,
-        );
-        discovered.addAll(files);
-      }
-
-      final unique = <String, MediaLibraryItem>{};
+      final initialItems = _loadAllItems()
+        ..removeWhere((item) => item.libraryID == library.id);
+      final unique = <String, MediaLibraryItem>{
+        for (final item in _loadItems(library.id)) item.file.id: item,
+      };
+      state = state.copyWith(items: unique.values.toList());
      final tmdbApiKey =
          StorageManager.get<String>(StorageKeys.tmdbApiKey) ?? '';
      final tmdbProxyHost =
          StorageManager.get<String>(StorageKeys.tmdbProxyHost) ?? '';
      final tmdbProxyPort =
          StorageManager.get<String>(StorageKeys.tmdbProxyPort) ?? '';
-      final input = <String, CloudFile>{
-        for (final file in discovered) file.id: file,
-      }.values.toList();
-      var nextIndex = 0;
      var completed = 0;
      Future<void> pendingPersistence = Future.value();
-      final initialItems = _loadAllItems()
-        ..removeWhere((item) => item.libraryID == library.id);

-      Future<void> worker() async {
-        while (!_cancelScan) {
-          if (nextIndex >= input.length) return;
-          final file = input[nextIndex++];
-          final fallback = MediaLibraryItem.fromFile(library.id, file);
-          final item = await _recognizeMediaItem(
-            fallback,
-            tmdbApiKey,
-            proxyHost: tmdbProxyHost,
-            proxyPort: tmdbProxyPort,
-          );
-          unique[file.id] = item;
-          completed += 1;
-          final visible = unique.values.toList()
-            ..sort(
-              (a, b) => a.title.toLowerCase().compareTo(b.title.toLowerCase()),
+      Future<void> indexBatch(List<CloudFile> files) async {
+        final pending = files
+            .where((file) => !unique.containsKey(file.id))
+            .toList();
+        if (pending.isEmpty || _cancelScan) return;
+        var next = 0;
+        Future<void> worker() async {
+          while (!_cancelScan && next < pending.length) {
+            final file = pending[next++];
+            final fallback = MediaLibraryItem.fromFile(library.id, file);
+            final item = await _recognizeMediaItem(
+              fallback,
+              tmdbApiKey,
+              proxyHost: tmdbProxyHost,
+              proxyPort: tmdbProxyPort,
            );
-          // Serialize writes so an earlier, smaller snapshot cannot overwrite
-          // a later batch while recognition workers finish out of order.
-          pendingPersistence = pendingPersistence.then(
-            (_) => _saveAllItems([...initialItems, ...unique.values]),
-          );
-          await pendingPersistence;
-          state = state.copyWith(
-            items: visible,
-            progress: MediaLibraryScanProgress(
-              phase: tmdbApiKey.isEmpty ? '正在建立本地索引' : '正在识别 ${file.name}',
-              completed: completed,
-              total: input.length,
-            ),
-          );
+            unique[file.id] = item;
+            completed += 1;
+            final visible = unique.values.toList()
+              ..sort(
+                (a, b) =>
+                    a.title.toLowerCase().compareTo(b.title.toLowerCase()),
+              );
+            pendingPersistence = pendingPersistence.then(
+              (_) => _saveAllItems([...initialItems, ...unique.values]),
+            );
+            await pendingPersistence;
+            state = state.copyWith(
+              items: visible,
+              progress: MediaLibraryScanProgress(
+                phase: tmdbApiKey.isEmpty ? '正在建立本地索引' : '正在识别 ${file.name}',
+                completed: completed,
+              ),
+            );
+          }
        }
+
+        final concurrency =
+            (int.tryParse(
+                      StorageManager.get<String>(
+                            StorageKeys.mediaScanConcurrency,
+                          ) ??
+                          '3',
+                    ) ??
+                    3)
+                .clamp(1, 20);
+        await Future.wait(List.generate(concurrency, (_) => worker()));
      }

-      final concurrency =
-          (int.tryParse(
-                    StorageManager.get<String>(
-                          StorageKeys.mediaScanConcurrency,
-                        ) ??
-                        '3',
-                  ) ??
-                  3)
-              .clamp(1, 20);
-      await Future.wait(List.generate(concurrency, (_) => worker()));
+      for (final source in library.sources) {
+        if (_cancelScan) break;
+        state = state.copyWith(
+          progress: MediaLibraryScanProgress(
+            phase: '扫描 ${source.path}',
+            completed: completed,
+          ),
+        );
+        await _scanSource(
+          source.rootID,
+          source.path,
+          recursive: library.recursive,
+          minimumSizeBytes: library.minimumSizeMB * 1024 * 1024,
+          onMediaFiles: indexBatch,
+        );
+      }
      await pendingPersistence;
      final items = unique.values.toList()
        ..sort(
@@ -390,15 +393,16 @@ class MediaLibraryNotifier extends StateNotifier<MediaLibraryState> {
    return int.tryParse(value?.toString() ?? '');
  }

-  Future<List<CloudFile>> _scanSource(
+  Future<void> _scanSource(
    String? rootID,
    String rootPath, {
    required bool recursive,
    required int minimumSizeBytes,
+    required Future<void> Function(List<CloudFile> files) onMediaFiles,
  }) async {
    final folders = <_ScanFolder>[_ScanFolder(rootID, rootPath)];
    final visited = <String>{};
-    final mediaFiles = <CloudFile>[];
+    var discovered = 0;

    while (folders.isNotEmpty && !_cancelScan) {
      final folder = folders.removeAt(0);
@@ -406,38 +410,175 @@ class MediaLibraryNotifier extends StateNotifier<MediaLibraryState> {
      if (!visited.add(visitKey)) continue;

      var page = 0;
+      final cachedFolder = FileMetadataCache.folderChildren(folder.id);
+      final folderSnapshot = <CloudFile>[];
      while (!_cancelScan) {
-        final response = await _api!.fsFiles(
-          parentID: folder.id,
-          page: page,
-          pageSize: 200,
-          orderBy: 0,
-          sortType: 0,
-        );
-        final files = _extractFiles(response);
+        late List<CloudFile> files;
+        if (cachedFolder != null) {
+          files = cachedFolder;
+        } else {
+          final response = await _api!.fsFiles(
+            parentID: folder.id,
+            page: page,
+            pageSize: 200,
+            orderBy: 0,
+            sortType: 0,
+          );
+          files = _extractFiles(response);
+          files = await _enrichAndCacheFiles(files);
+          folderSnapshot.addAll(files);
+        }
+        final mediaBatch = <CloudFile>[];
        for (final file in files) {
          if (file.isDirectory) {
            if (recursive) {
              folders.add(_ScanFolder(file.id, '${folder.path}/${file.name}'));
            }
          } else if (file.isVideo && (file.size ?? 0) >= minimumSizeBytes) {
-            mediaFiles.add(_withPath(file, '${folder.path}/${file.name}'));
+            mediaBatch.add(_withPath(file, '${folder.path}/${file.name}'));
          }
        }
+        discovered += mediaBatch.length;
+        if (mediaBatch.isNotEmpty) await onMediaFiles(mediaBatch);

        state = state.copyWith(
          progress: MediaLibraryScanProgress(
            phase: '扫描 ${folder.path}',
-            completed: mediaFiles.length,
+            completed: discovered,
            total: folders.length + visited.length,
          ),
        );

-        if (files.length < 200) break;
+        if (cachedFolder != null || files.length < 200) {
+          if (cachedFolder == null) {
+            await FileMetadataCache.cacheFolderChildren(
+              folder.id,
+              folderSnapshot,
+            );
+          }
+          break;
+        }
        page += 1;
      }
    }
-    return mediaFiles;
+  }
+
+  Future<List<CloudFile>> _enrichAndCacheFiles(List<CloudFile> files) async {
+    final resolved = <CloudFile>[];
+    final pending = <CloudFile>[];
+    for (final file in files) {
+      final cached = FileMetadataCache.file(file.id);
+      if (cached != null) {
+        resolved.add(
+          file.copyWith(
+            size: cached.size,
+            gcid: cached.gcid,
+            modifiedAt: cached.modifiedAt,
+            cloudPath: file.cloudPath,
+          ),
+        );
+      } else if (!file.isDirectory &&
+          (file.gcid == null || file.gcid!.isEmpty)) {
+        pending.add(file);
+      } else {
+        resolved.add(file);
+      }
+    }
+    if (pending.isEmpty) {
+      await FileMetadataCache.cacheFiles(resolved);
+      return resolved;
+    }
+
+    final enriched = <String, CloudFile>{
+      for (final file in resolved) file.id: file,
+    };
+    var next = 0;
+    Future<void> worker() async {
+      while (next < pending.length) {
+        final file = pending[next++];
+        try {
+          final detail = await _api!.fsDetail(file.id);
+          final detailFile = _extractFiles(
+            detail,
+          ).where((candidate) => candidate.id == file.id).firstOrNull;
+          final gcid =
+              detailFile?.gcid ??
+              _findStringDeep(detail, const [
+                'gcid',
+                'gcId',
+                'gcidValue',
+                'hash',
+              ]);
+          final size =
+              detailFile?.size ??
+              _findIntDeep(detail, const [
+                'size',
+                'fileSize',
+                'resSize',
+                'totalSize',
+              ]);
+          enriched[file.id] = file.copyWith(gcid: gcid, size: size);
+        } catch (_) {
+          enriched[file.id] = file;
+        }
+      }
+    }
+
+    await Future.wait(List.generate(6, (_) => worker()));
+    final values = [for (final file in files) enriched[file.id] ?? file];
+    await FileMetadataCache.cacheFiles(values);
+    return values;
+  }
+
+  String? _findStringDeep(Map<String, dynamic> value, List<String> keys) {
+    for (final entry in value.entries) {
+      if (keys.contains(entry.key) && entry.value != null) {
+        final text = entry.value.toString().trim();
+        if (text.isNotEmpty) return text;
+      }
+      if (entry.value is Map) {
+        final found = _findStringDeep(
+          Map<String, dynamic>.from(entry.value),
+          keys,
+        );
+        if (found != null) return found;
+      } else if (entry.value is List) {
+        for (final child in entry.value as List) {
+          if (child is Map) {
+            final found = _findStringDeep(
+              Map<String, dynamic>.from(child),
+              keys,
+            );
+            if (found != null) return found;
+          }
+        }
+      }
+    }
+    return null;
+  }
+
+  int? _findIntDeep(Map<String, dynamic> value, List<String> keys) {
+    for (final entry in value.entries) {
+      if (keys.contains(entry.key)) {
+        final parsed = int.tryParse(entry.value?.toString() ?? '');
+        if (parsed != null) return parsed;
+      }
+      if (entry.value is Map) {
+        final found = _findIntDeep(
+          Map<String, dynamic>.from(entry.value),
+          keys,
+        );
+        if (found != null) return found;
+      } else if (entry.value is List) {
+        for (final child in entry.value as List) {
+          if (child is Map) {
+            final found = _findIntDeep(Map<String, dynamic>.from(child), keys);
+            if (found != null) return found;
+          }
+        }
+      }
+    }
+    return null;
  }

  CloudFile _withPath(CloudFile file, String path) {