feat: 流水线刮削并即时写回媒体信息

This commit is contained in:
2026-07-16 14:58:24 +08:00
parent 8f78b25313
commit ec7f0024e7
2 changed files with 172 additions and 122 deletions
+166 -122
View File
@@ -42,6 +42,7 @@ final class AppModel: ObservableObject {
@Published var mediaMetadataWriteProgress = MediaLibraryScanProgress()
@Published var mediaLibraryLiveLibraryID: String?
@Published var mediaLibraryLiveItems: [MediaLibraryItem] = []
@Published var mediaLibraryDestination: MediaLibraryDestination = .all
@Published var isRunningTMDBJob = false
@Published var isAnalyzing = false
@Published var activeScanRequest: ScanRequest?
@@ -83,6 +84,9 @@ final class AppModel: ObservableObject {
private var isHandlingAuthorizationExpiry = false
private var scanTask: Task<Void, Never>?
private var mediaMetadataWriteTask: Task<Void, Never>?
private var mediaMetadataWriteQueue: [(MediaLibraryItem, MediaSidecarFiles)] = []
private var mediaMetadataWrittenKeys = Set<String>()
private var mediaMetadataWriteFailures = 0
init() {
let defaults = UserDefaults.standard
@@ -419,44 +423,41 @@ final class AppModel: ObservableObject {
guard !isScanningMediaLibrary else { return await cachedMediaLibraryItems(libraryID: library.id) }
isScanningMediaLibrary = true
mediaLibraryScanProgress = MediaLibraryScanProgress(phase: "正在读取云端目录…")
defer { isScanningMediaLibrary = false }
var discoveredFiles: [CloudFile] = []
var discoveredSidecars: [String: MediaSidecarFiles] = [:]
for (index, source) in library.sources.enumerated() {
mediaLibraryScanProgress.phase = "正在读取目录 \(index + 1)/\(library.sources.count)"
let snapshot = try await discoverMediaFiles(parentID: source.rootID, rootPath: source.path, recursive: library.recursive)
discoveredFiles += snapshot.files
discoveredSidecars.merge(snapshot.sidecars) { current, _ in current }
mediaMetadataWrittenKeys = []
mediaMetadataWriteFailures = 0
mediaMetadataWriteProgress = MediaLibraryScanProgress()
defer {
isScanningMediaLibrary = false
if Task.isCancelled { mediaLibraryScanProgress.phase = "扫描已取消" }
}
mediaLibraryScanProgress.phase = "正在校验 GCID…"
let uniqueByID = Array(Dictionary(grouping: discoveredFiles, by: \.id).compactMap { $0.value.first })
let enriched = await enrichMediaFingerprints(uniqueByID)
var preferredByFingerprint: [String: CloudFile] = [:]
for file in enriched {
let key = file.gcid.flatMap { $0.isEmpty ? nil : "gcid:\($0)" } ?? "file:\(file.id)"
guard let existing = preferredByFingerprint[key] else { preferredByFingerprint[key] = file; continue }
let candidateLanguage = Self.mediaLanguageProfile(fileName: file.name, subtitles: discoveredSidecars[file.id]?.subtitles ?? [])
let existingLanguage = Self.mediaLanguageProfile(fileName: existing.name, subtitles: discoveredSidecars[existing.id]?.subtitles ?? [])
let candidateScore = (candidateLanguage.audio ? 2 : 0) + (candidateLanguage.subtitle ? 1 : 0)
let existingScore = (existingLanguage.audio ? 2 : 0) + (existingLanguage.subtitle ? 1 : 0)
if candidateScore > existingScore || (candidateScore == existingScore && file.cloudPath.count < existing.cloudPath.count) {
preferredByFingerprint[key] = file
}
}
let files = preferredByFingerprint.values.sorted { $0.cloudPath.localizedStandardCompare($1.cloudPath) == .orderedAscending }
let snapshot = MediaScanSnapshot(files: files, sidecars: discoveredSidecars)
mediaLibraryScanProgress = MediaLibraryScanProgress(phase: "正在读取已有刮削信息…", completed: 0, total: snapshot.files.count)
await mediaLibraryStore.save(library: library)
mediaLibraryLiveLibraryID = library.id
mediaLibraryLiveItems = await mediaLibraryStore.items(libraryID: library.id)
let items = await scrapeMediaFiles(snapshot.files, library: library, sidecars: snapshot.sidecars)
var items: [MediaLibraryItem] = []
var allSidecars: [String: MediaSidecarFiles] = [:]
var seenFingerprints = Set<String>()
mediaLibraryScanProgress = MediaLibraryScanProgress(phase: "正在发现媒体…")
for try await discovered in mediaDirectorySnapshots(for: library) {
try Task.checkCancellation()
allSidecars.merge(discovered.sidecars) { current, _ in current }
let enriched = await enrichMediaFingerprints(discovered.files)
let newFiles = enriched.filter { file in
let key = file.gcid.flatMap { $0.isEmpty ? nil : "gcid:\($0.lowercased())" } ?? "file:\(file.id)"
return seenFingerprints.insert(key).inserted
}
guard !newFiles.isEmpty else { continue }
mediaLibraryScanProgress.total += newFiles.count
mediaLibraryScanProgress.phase = "\(library.kind.title)模式 · 边扫描边刮削…"
let batchItems = try await scrapeMediaFiles(newFiles, library: library, sidecars: allSidecars, progressOffset: items.count)
items += batchItems
}
items.sort { $0.title.localizedStandardCompare($1.title) == .orderedAscending }
var updatedLibrary = library
updatedLibrary.updatedAt = Date()
await mediaLibraryStore.save(library: updatedLibrary)
guard await mediaLibraryStore.replaceItems(items, library: updatedLibrary) else { throw MediaLibraryStoreError.writeFailed }
await loadMediaLibraries()
mediaLibraryScanProgress.phase = "刮削完成"
startMediaMetadataWriteback(items, sidecars: snapshot.sidecars)
return items
}
@@ -478,56 +479,72 @@ final class AppModel: ObservableObject {
}
for _ in 0..<min(4, files.count) { addNext() }
var values: [CloudFile] = []
while let file = await group.next() { values.append(file); addNext() }
while let file = await group.next() {
if Task.isCancelled { group.cancelAll(); break }
values.append(file); addNext()
}
return values
}
}
private func discoverMediaFiles(parentID: String?, rootPath: String, recursive: Bool) async throws -> MediaScanSnapshot {
let initialPath = parentID == nil ? [] : rootPath.split(separator: "/").map(String.init)
var queue: [(id: String?, path: [String])] = [(parentID, initialPath)]
var media: [CloudFile] = []
var sidecars: [String: MediaSidecarFiles] = [:]
while !queue.isEmpty {
try Task.checkCancellation()
let current = queue.removeFirst()
var children = try await allFiles(parentID: current.id)
for index in children.indices {
children[index].cloudPath = "/" + (current.path + [children[index].name]).joined(separator: "/")
}
let directMedia = children.filter(\.isVideo)
media += directMedia
let ordinaryFiles = children.filter { !$0.isDirectory && !$0.isVideo }
for video in directMedia {
let stem = (video.name as NSString).deletingPathExtension.lowercased()
func first(named names: Set<String>, extensions: Set<String>) -> CloudFile? {
ordinaryFiles.first { file in
let lower = file.name.lowercased()
let ext = (lower as NSString).pathExtension
return names.contains(lower) || (extensions.contains(ext) && (lower.hasPrefix(stem + ".") || lower.hasPrefix(stem + "-") || lower.hasPrefix(stem + "_")))
private func mediaDirectorySnapshots(for library: MediaLibraryDefinition) -> AsyncThrowingStream<MediaScanSnapshot, Error> {
AsyncThrowingStream { continuation in
let producer = Task { @MainActor [weak self] in
guard let self else { continuation.finish(); return }
do {
var queue: [(id: String?, path: [String])] = library.sources.map { source in
(source.rootID, source.rootID == nil ? [] : source.path.split(separator: "/").map(String.init))
}
while !queue.isEmpty {
try Task.checkCancellation()
let current = queue.removeFirst()
var children = try await self.allFiles(parentID: current.id)
for index in children.indices {
children[index].cloudPath = "/" + (current.path + [children[index].name]).joined(separator: "/")
}
let snapshot = self.mediaSnapshot(children: children, parentID: current.id)
if !snapshot.files.isEmpty { continuation.yield(snapshot) }
if library.recursive { queue += children.filter(\.isDirectory).map { ($0.id, current.path + [$0.name]) } }
}
continuation.finish()
} catch {
continuation.finish(throwing: error)
}
let nfoFiles = ordinaryFiles.filter { ($0.name as NSString).pathExtension.lowercased() == "nfo" }
let nfo = first(named: ["movie.nfo", "tvshow.nfo"], extensions: ["nfo"]) ?? (nfoFiles.count == 1 ? nfoFiles.first : nil)
let poster = first(named: ["poster.jpg", "poster.jpeg", "poster.png", "folder.jpg", "folder.png"], extensions: ["jpg", "jpeg", "png", "webp"])
let backdrop = first(named: ["fanart.jpg", "fanart.png", "backdrop.jpg", "background.jpg"], extensions: ["jpg", "jpeg", "png", "webp"])
let subtitleExtensions: Set<String> = ["srt", "ass", "ssa", "sub", "vtt", "sup"]
let subtitles = ordinaryFiles.filter { file in
let lower = file.name.lowercased()
guard subtitleExtensions.contains((lower as NSString).pathExtension) else { return false }
let subtitleStem = (lower as NSString).deletingPathExtension
return subtitleStem == stem || subtitleStem.hasPrefix(stem + ".") || subtitleStem.hasPrefix(stem + "-") || subtitleStem.hasPrefix(stem + "_")
}
sidecars[video.id] = MediaSidecarFiles(parentID: current.id, nfo: nfo, poster: poster, backdrop: backdrop, subtitles: subtitles)
}
if recursive { queue += children.filter(\.isDirectory).map { ($0.id, current.path + [$0.name]) } }
continuation.onTermination = { @Sendable _ in producer.cancel() }
}
let unique = Array(Dictionary(grouping: media, by: \.id).compactMap { $0.value.first })
.sorted { $0.cloudPath.localizedStandardCompare($1.cloudPath) == .orderedAscending }
return MediaScanSnapshot(files: unique, sidecars: sidecars)
}
private func scrapeMediaFiles(_ files: [CloudFile], library: MediaLibraryDefinition, sidecars: [String: MediaSidecarFiles]) async -> [MediaLibraryItem] {
private func mediaSnapshot(children: [CloudFile], parentID: String?) -> MediaScanSnapshot {
let directMedia = children.filter(\.isVideo)
let ordinaryFiles = children.filter { !$0.isDirectory && !$0.isVideo }
var sidecars: [String: MediaSidecarFiles] = [:]
for video in directMedia {
let stem = (video.name as NSString).deletingPathExtension.lowercased()
func first(named names: Set<String>, extensions: Set<String>) -> CloudFile? {
ordinaryFiles.first { file in
let lower = file.name.lowercased()
let ext = (lower as NSString).pathExtension
return names.contains(lower) || (extensions.contains(ext) && (lower.hasPrefix(stem + ".") || lower.hasPrefix(stem + "-") || lower.hasPrefix(stem + "_")))
}
}
let nfoFiles = ordinaryFiles.filter { ($0.name as NSString).pathExtension.lowercased() == "nfo" }
let nfo = first(named: ["movie.nfo", "tvshow.nfo"], extensions: ["nfo"]) ?? (nfoFiles.count == 1 ? nfoFiles.first : nil)
let poster = first(named: ["poster.jpg", "poster.jpeg", "poster.png", "folder.jpg", "folder.png"], extensions: ["jpg", "jpeg", "png", "webp"])
let backdrop = first(named: ["fanart.jpg", "fanart.png", "backdrop.jpg", "background.jpg"], extensions: ["jpg", "jpeg", "png", "webp"])
let subtitleExtensions: Set<String> = ["srt", "ass", "ssa", "sub", "vtt", "sup"]
let subtitles = ordinaryFiles.filter { file in
let lower = file.name.lowercased()
guard subtitleExtensions.contains((lower as NSString).pathExtension) else { return false }
let subtitleStem = (lower as NSString).deletingPathExtension
return subtitleStem == stem || subtitleStem.hasPrefix(stem + ".") || subtitleStem.hasPrefix(stem + "-") || subtitleStem.hasPrefix(stem + "_")
}
sidecars[video.id] = MediaSidecarFiles(parentID: parentID, nfo: nfo, poster: poster, backdrop: backdrop, subtitles: subtitles)
}
return MediaScanSnapshot(files: directMedia, sidecars: sidecars)
}
private func scrapeMediaFiles(_ files: [CloudFile], library: MediaLibraryDefinition, sidecars: [String: MediaSidecarFiles], progressOffset: Int) async throws -> [MediaLibraryItem] {
let kind = library.kind
let requests = files.enumerated().map { index, file in
(index: index, file: file, parsed: parseMediaName(file.name))
@@ -554,11 +571,27 @@ final class AppModel: ObservableObject {
case .series: preferredKind = .tv
case .mixed: preferredKind = request.parsed.isEpisode ? .tv : .automatic
}
let response = try await api.tmdbSearch(query: request.parsed.title, apiKey: apiKey, mediaKind: preferredKind, proxyHost: proxyHost, proxyPort: proxyPort)
guard let candidate = (response["results"]?.arrayValue ?? []).compactMap({ TMDBCandidate(json: $0, forcedKind: preferredKind == .automatic ? nil : preferredKind) }).first else {
return (request.index, fallback)
let candidate: TMDBCandidate
let details: JSONValue?
if let embeddedID = Self.embeddedTMDBID(in: request.file.cloudPath) {
let initialKind: TMDBMediaKind = preferredKind == .automatic ? (request.parsed.isEpisode ? .tv : .movie) : preferredKind
var embeddedDetails = try? await api.tmdbDetails(id: embeddedID, mediaKind: initialKind, apiKey: apiKey, proxyHost: proxyHost, proxyPort: proxyPort)
var resolvedKind = initialKind
if embeddedDetails == nil, preferredKind == .automatic {
resolvedKind = initialKind == .movie ? .tv : .movie
embeddedDetails = try? await api.tmdbDetails(id: embeddedID, mediaKind: resolvedKind, apiKey: apiKey, proxyHost: proxyHost, proxyPort: proxyPort)
}
guard let embeddedDetails, let embeddedCandidate = TMDBCandidate(json: embeddedDetails, forcedKind: resolvedKind) else { return (request.index, fallback) }
candidate = embeddedCandidate
details = embeddedDetails
} else {
let response = try await api.tmdbSearch(query: request.parsed.title, apiKey: apiKey, mediaKind: preferredKind, proxyHost: proxyHost, proxyPort: proxyPort)
guard let searched = (response["results"]?.arrayValue ?? []).compactMap({ TMDBCandidate(json: $0, forcedKind: preferredKind == .automatic ? nil : preferredKind) }).first else {
return (request.index, fallback)
}
candidate = searched
details = try? await api.tmdbDetails(id: searched.id, mediaKind: searched.mediaType, apiKey: apiKey, proxyHost: proxyHost, proxyPort: proxyPort)
}
let details = try? await api.tmdbDetails(id: candidate.id, mediaKind: candidate.mediaType, apiKey: apiKey, proxyHost: proxyHost, proxyPort: proxyPort)
let localizedTitle = details?.firstString(["title", "name"]) ?? candidate.title
let originalTitle = details?.firstString(["original_title", "original_name"]) ?? candidate.originalTitle
let localizedOverview = details?.firstString(["overview"]) ?? candidate.overview
@@ -588,24 +621,21 @@ final class AppModel: ObservableObject {
}
for _ in 0..<min(4, requests.count) { addNext() }
var values: [(Int, MediaLibraryItem)] = []
var pendingDatabaseItems: [MediaLibraryItem] = []
while let value = await group.next() {
if Task.isCancelled { group.cancelAll(); break }
values.append(value)
pendingDatabaseItems.append(value.1)
mediaLibraryScanProgress.completed = values.count
if pendingDatabaseItems.count >= 8 {
_ = await mediaLibraryStore.upsertItems(pendingDatabaseItems, library: library)
pendingDatabaseItems.removeAll(keepingCapacity: true)
mediaLibraryLiveItems = await mediaLibraryStore.items(libraryID: library.id)
mediaLibraryScanProgress.completed = progressOffset + values.count
if await mediaLibraryStore.upsertItems([value.1], library: library) {
if let index = mediaLibraryLiveItems.firstIndex(where: { $0.id == value.1.id }) { mediaLibraryLiveItems[index] = value.1 }
else { mediaLibraryLiveItems.append(value.1) }
mediaLibraryLiveItems.sort { $0.title.localizedStandardCompare($1.title) == .orderedAscending }
if let files = sidecars[value.1.id] { enqueueMediaMetadataWriteback(value.1, files: files) }
}
addNext()
}
if !pendingDatabaseItems.isEmpty {
_ = await mediaLibraryStore.upsertItems(pendingDatabaseItems, library: library)
mediaLibraryLiveItems = await mediaLibraryStore.items(libraryID: library.id)
}
return values
}
try Task.checkCancellation()
return results.sorted { $0.0 < $1.0 }.map(\.1)
}
@@ -640,6 +670,14 @@ final class AppModel: ObservableObject {
return (hasAudio, hasSubtitle)
}
private nonisolated static func embeddedTMDBID(in cloudPath: String) -> Int? {
let folderPath = (cloudPath as NSString).deletingLastPathComponent
guard let regex = try? NSRegularExpression(pattern: "(?i)tmdb(?:id)?[\\s._:=\\-\\[\\(]*(\\d{2,})"),
let match = regex.matches(in: folderPath, range: NSRange(folderPath.startIndex..., in: folderPath)).last,
let range = Range(match.range(at: 1), in: folderPath) else { return nil }
return Int(folderPath[range])
}
private nonisolated static func preferredArtworkPath(in details: JSONValue, collection: String) -> String? {
let images = details["images"]?[collection]?.arrayValue ?? []
let languageOrder: [String?] = ["zh", "zh-CN", nil, "en"]
@@ -664,52 +702,58 @@ final class AppModel: ObservableObject {
return await sidecarData(for: file, api: api)
}
private func writeMediaScrapeToCloud(_ items: [MediaLibraryItem], sidecars: [String: MediaSidecarFiles]) async -> Int {
var failures = 0
var writtenKeys = Set<String>()
for (index, item) in items.enumerated() {
if Task.isCancelled { break }
defer { mediaMetadataWriteProgress.completed = index + 1 }
guard item.isMatched, let files = sidecars[item.file.id] else { continue }
let parentKey = files.parentID ?? "root"
do {
if files.nfo == nil {
let stem = (item.file.name as NSString).deletingPathExtension
let name = "\(stem).nfo"
let key = "\(parentKey)/\(name.lowercased())"
if writtenKeys.insert(key).inserted {
try await uploadScrapeData(Data(mediaNFO(for: item).utf8), name: name, contentType: "application/xml", parentID: files.parentID)
}
}
if files.poster == nil, let data = item.posterData {
let key = "\(parentKey)/poster.jpg"
if writtenKeys.insert(key).inserted { try await uploadScrapeData(data, name: "poster.jpg", contentType: "image/jpeg", parentID: files.parentID) }
}
if files.backdrop == nil, let data = item.backdropData {
let key = "\(parentKey)/fanart.jpg"
if writtenKeys.insert(key).inserted { try await uploadScrapeData(data, name: "fanart.jpg", contentType: "image/jpeg", parentID: files.parentID) }
}
} catch {
failures += 1
}
await Task.yield()
}
return failures
}
private func startMediaMetadataWriteback(_ items: [MediaLibraryItem], sidecars: [String: MediaSidecarFiles]) {
mediaMetadataWriteTask?.cancel()
private func enqueueMediaMetadataWriteback(_ item: MediaLibraryItem, files: MediaSidecarFiles) {
guard item.isMatched else { return }
mediaMetadataWriteQueue.append((item, files))
mediaMetadataWriteProgress.total += 1
mediaMetadataWriteProgress.phase = "刮削入库并写回资源目录…"
guard mediaMetadataWriteTask == nil else { return }
isWritingMediaMetadata = true
mediaMetadataWriteProgress = MediaLibraryScanProgress(phase: "正在写回资源目录…", completed: 0, total: items.count)
mediaMetadataWriteTask = Task { [weak self] in
guard let self else { return }
let failures = await self.writeMediaScrapeToCloud(items, sidecars: sidecars)
guard !Task.isCancelled else { self.isWritingMediaMetadata = false; return }
self.mediaMetadataWriteProgress.phase = failures == 0 ? "资源目录写回完成" : "写回完成,\(failures) 项失败"
while !self.mediaMetadataWriteQueue.isEmpty, !Task.isCancelled {
let (item, files) = self.mediaMetadataWriteQueue.removeFirst()
if !(await self.writeMediaScrapeItem(item, files: files)) { self.mediaMetadataWriteFailures += 1 }
self.mediaMetadataWriteProgress.completed += 1
}
guard !Task.isCancelled else { self.isWritingMediaMetadata = false; self.mediaMetadataWriteTask = nil; return }
self.mediaMetadataWriteProgress.phase = self.mediaMetadataWriteFailures == 0 ? "资源目录写回完成" : "写回完成,\(self.mediaMetadataWriteFailures) 项失败"
self.isWritingMediaMetadata = false
self.mediaMetadataWriteTask = nil
}
}
private func writeMediaScrapeItem(_ item: MediaLibraryItem, files: MediaSidecarFiles) async -> Bool {
let parentKey = files.parentID ?? "root"
do {
if files.nfo == nil {
let stem = (item.file.name as NSString).deletingPathExtension
let name = "\(stem).nfo"
let key = "\(parentKey)/\(name.lowercased())"
if mediaMetadataWrittenKeys.insert(key).inserted {
try await uploadScrapeData(Data(mediaNFO(for: item).utf8), name: name, contentType: "application/xml", parentID: files.parentID)
}
}
if files.poster == nil, let data = item.posterData {
let key = "\(parentKey)/poster.jpg"
if mediaMetadataWrittenKeys.insert(key).inserted { try await uploadScrapeData(data, name: "poster.jpg", contentType: "image/jpeg", parentID: files.parentID) }
}
if files.backdrop == nil, let data = item.backdropData {
let key = "\(parentKey)/fanart.jpg"
if mediaMetadataWrittenKeys.insert(key).inserted { try await uploadScrapeData(data, name: "fanart.jpg", contentType: "image/jpeg", parentID: files.parentID) }
}
return true
} catch { return false }
}
func cancelMediaMetadataWriteback() {
mediaMetadataWriteTask?.cancel()
mediaMetadataWriteTask = nil
mediaMetadataWriteQueue.removeAll()
isWritingMediaMetadata = false
mediaMetadataWriteProgress.phase = "资源目录写回已停止"
}
private func mediaNFO(for item: MediaLibraryItem) -> String {
let parsed = parseMediaName(item.file.name)
let root = parsed.isEpisode ? "episodedetails" : (item.mediaKind == .tv ? "tvshow" : "movie")
+6
View File
@@ -516,6 +516,12 @@ struct MediaLibrarySource: Identifiable, Hashable, Sendable {
}
}
enum MediaLibraryDestination: Equatable, Sendable {
case all
case library(String)
case tasks
}
enum MediaLibraryKind: String, CaseIterable, Identifiable, Codable, Sendable {
case movies
case series