-
Notifications
You must be signed in to change notification settings - Fork 16
feat: Add batch PV conversion tool #63
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,8 @@ | ||
| using System.Text.Json; | ||
| using System.Threading.Channels; | ||
| using MaiChartManager.Utils; | ||
| using Microsoft.AspNetCore.Mvc; | ||
| using Microsoft.VisualBasic.FileIO; | ||
|
|
||
| namespace MaiChartManager.Controllers.Tools; | ||
|
|
||
|
|
@@ -83,4 +86,270 @@ await VideoConvert.ConvertVideoToUsm( | |
| await Response.Body.FlushAsync(); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public enum BatchConvertPvDirection | ||
| { | ||
| /// <summary>USM/DAT → MP4</summary> | ||
| UsmToMp4, | ||
| /// <summary>MP4 → USM/DAT</summary> | ||
| Mp4ToUsm | ||
| } | ||
|
|
||
| public enum BatchConvertPvEventType | ||
| { | ||
| /// <summary>整体 + 当前文件进度,data 为 JSON</summary> | ||
| Progress, | ||
| /// <summary>单文件失败,仍然继续处理后续文件</summary> | ||
| FileError, | ||
| /// <summary>全部完成,data 为 "processed/total|failedCount"</summary> | ||
| Success, | ||
| /// <summary>致命错误,停止</summary> | ||
| Error, | ||
| /// <summary>被取消,data 为 "processed/total"</summary> | ||
| Cancelled | ||
| } | ||
|
|
||
| private static readonly JsonSerializerOptions BatchJsonOptions = new() | ||
| { | ||
| PropertyNamingPolicy = JsonNamingPolicy.CamelCase, | ||
| }; | ||
|
|
||
| private record BatchProgressPayload(int Processed, int Total, int FileProgress, string FileName, int Failed); | ||
|
|
||
| /// <summary> | ||
| /// 批量转换用户选择的文件夹内所有 PV:USM/DAT ↔ MP4。 | ||
| /// 使用 SSE 实时推送整体进度(已处理/总数)+ 当前文件进度。 | ||
| /// 客户端断开连接时通过 RequestAborted 触发取消,循环在下一个文件之间退出。 | ||
| /// 所有 SSE 写入通过单写者 Channel 串行化,避免 Xabe 同步进度事件触发的 async-void 写入交错。 | ||
| /// </summary> | ||
| [HttpPost] | ||
| public async Task BatchConvertPvTool([FromQuery] string folderPath, [FromQuery] BatchConvertPvDirection direction) | ||
| { | ||
| Response.Headers.Append("Content-Type", "text/event-stream"); | ||
|
|
||
| // PV 转换属于赞助功能 | ||
| if (IapManager.License != IapManager.LicenseStatus.Active) | ||
| { | ||
| await Response.WriteAsync($"event: {BatchConvertPvEventType.Error}\ndata: {SanitizeSseLine(Locale.BatchConvertPvNeedLicense)}\n\n"); | ||
| await Response.Body.FlushAsync(); | ||
| return; | ||
| } | ||
|
|
||
| if (string.IsNullOrWhiteSpace(folderPath) || !Directory.Exists(folderPath)) | ||
| { | ||
| await Response.WriteAsync($"event: {BatchConvertPvEventType.Error}\ndata: {SanitizeSseLine(Locale.BatchConvertPvFolderNotFound)}\n\n"); | ||
| await Response.Body.FlushAsync(); | ||
| return; | ||
| } | ||
|
|
||
| // 直接枚举用户选择的文件夹(不递归),按方向筛选源扩展名 | ||
| var sourceExtensions = direction == BatchConvertPvDirection.UsmToMp4 | ||
| ? new HashSet<string>(StringComparer.OrdinalIgnoreCase) { ".dat", ".usm" } | ||
| : new HashSet<string>(StringComparer.OrdinalIgnoreCase) { ".mp4" }; | ||
|
|
||
| var files = Directory.EnumerateFiles(folderPath) | ||
| .Where(f => sourceExtensions.Contains(Path.GetExtension(f))) | ||
| .OrderBy(f => f, StringComparer.OrdinalIgnoreCase) | ||
| .ToList(); | ||
|
|
||
| if (files.Count == 0) | ||
| { | ||
| await Response.WriteAsync($"event: {BatchConvertPvEventType.Error}\ndata: {SanitizeSseLine(Locale.BatchConvertPvNoFiles)}\n\n"); | ||
| await Response.Body.FlushAsync(); | ||
| return; | ||
| } | ||
|
|
||
| var total = files.Count; | ||
| var processed = 0; | ||
| var failedCount = 0; | ||
| var cancellationToken = HttpContext.RequestAborted; | ||
|
|
||
| // 单写者 Channel:所有 SSE 帧(不论来自循环还是 OnProgress)都进入这条队列 | ||
| var sseChannel = Channel.CreateUnbounded<string>(new UnboundedChannelOptions | ||
| { | ||
| SingleReader = true, | ||
| SingleWriter = false, | ||
| }); | ||
|
|
||
| var writer = WriteSseFrames(sseChannel.Reader, cancellationToken); | ||
|
Comment on lines
+168
to
+174
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. suggestion (bug_risk): 建议对 SSE 进度通道进行限界或节流,以避免在客户端较慢或进度事件非常频繁的情况下导致内存无限增长。 由于这里使用的是无界的 Suggested implementation: // 单写者 Channel:所有 SSE 帧(不论来自循环还是 OnProgress)都进入这条队列
// 使用有界 Channel,并在队列满时丢弃最旧的进度帧,防止在慢客户端/高频进度回调下无限占用内存
var sseChannel = Channel.CreateBounded<string>(new BoundedChannelOptions(100)
{
SingleReader = true,
SingleWriter = false,
FullMode = BoundedChannelFullMode.DropOldest,
});
Original comment in Englishsuggestion (bug_risk): Consider bounding or throttling the SSE progress channel to avoid unbounded memory growth under slow clients or very frequent progress events. Because this is an unbounded Suggested implementation: // 单写者 Channel:所有 SSE 帧(不论来自循环还是 OnProgress)都进入这条队列
// 使用有界 Channel,并在队列满时丢弃最旧的进度帧,防止在慢客户端/高频进度回调下无限占用内存
var sseChannel = Channel.CreateBounded<string>(new BoundedChannelOptions(100)
{
SingleReader = true,
SingleWriter = false,
FullMode = BoundedChannelFullMode.DropOldest,
});
|
||
|
|
||
| try | ||
| { | ||
| foreach (var inputPath in files) | ||
| { | ||
| if (cancellationToken.IsCancellationRequested) break; | ||
|
|
||
| var fileName = Path.GetFileName(inputPath); | ||
| await EnqueueProgress(sseChannel.Writer, processed, total, 0, fileName, failedCount); | ||
|
|
||
| try | ||
| { | ||
| var directory = Path.GetDirectoryName(inputPath)!; | ||
| var nameWithoutExt = Path.GetFileNameWithoutExtension(inputPath); | ||
|
|
||
| if (direction == BatchConvertPvDirection.UsmToMp4) | ||
| { | ||
| var outputPath = Path.Combine(directory, nameWithoutExt + ".mp4"); | ||
| var snapshot = (Processed: processed, Failed: failedCount); | ||
| await VideoConvert.ConvertUsmToMp4( | ||
| inputPath, | ||
| outputPath, | ||
| percent => EnqueueProgressFireAndForget(sseChannel.Writer, snapshot.Processed, total, percent, fileName, snapshot.Failed)); | ||
| } | ||
| else | ||
| { | ||
| // MP4 → USM(VP9):先输出到临时文件,验证后再覆盖目标 | ||
| var finalPath = Path.Combine(directory, nameWithoutExt + ".dat"); | ||
| var tempPath = finalPath + ".tmp"; | ||
| var snapshot = (Processed: processed, Failed: failedCount); | ||
| try | ||
| { | ||
| await VideoConvert.ConvertVideo(new VideoConvert.VideoConvertOptions | ||
| { | ||
| InputPath = inputPath, | ||
| OutputPath = tempPath, | ||
| NoScale = StaticSettings.Config.NoScale, | ||
| UseH264 = false, | ||
| UseYuv420p = StaticSettings.Config.Yuv420p, | ||
| Padding = 0, | ||
| TaskbarProgress = false, | ||
| OnProgress = percent => EnqueueProgressFireAndForget(sseChannel.Writer, snapshot.Processed, total, percent, fileName, snapshot.Failed) | ||
| }); | ||
|
|
||
| if (!System.IO.File.Exists(tempPath) || new FileInfo(tempPath).Length == 0) | ||
| { | ||
| throw new Exception("Converted DAT is missing or empty"); | ||
| } | ||
|
|
||
| // 取消检查必须在覆盖/删除前,避免取消时仍然损毁源文件 | ||
| cancellationToken.ThrowIfCancellationRequested(); | ||
|
|
||
| if (System.IO.File.Exists(finalPath)) | ||
| { | ||
| System.IO.File.Delete(finalPath); | ||
| } | ||
| System.IO.File.Move(tempPath, finalPath); | ||
|
|
||
| // 源 MP4 送进回收站,而非永久删除,最大程度避免用户数据丢失 | ||
| try | ||
| { | ||
| FileSystem.DeleteFile(inputPath, UIOption.OnlyErrorDialogs, RecycleOption.SendToRecycleBin); | ||
| } | ||
| catch (Exception delEx) | ||
| { | ||
| logger.LogWarning(delEx, "Failed to move source MP4 to recycle bin after batch convert: {Path}", inputPath); | ||
| await EnqueueEvent(sseChannel.Writer, BatchConvertPvEventType.FileError, $"{fileName}: moved to .dat but failed to remove source MP4 ({delEx.Message})"); | ||
| } | ||
| } | ||
| catch | ||
| { | ||
| try { if (System.IO.File.Exists(tempPath)) System.IO.File.Delete(tempPath); } | ||
| catch { /* ignored */ } | ||
| throw; | ||
| } | ||
| } | ||
|
|
||
| processed++; | ||
| await EnqueueProgress(sseChannel.Writer, processed, total, 100, fileName, failedCount); | ||
| } | ||
| catch (OperationCanceledException) | ||
| { | ||
| throw; | ||
| } | ||
| catch (Exception fileEx) | ||
| { | ||
| logger.LogError(fileEx, "Failed to convert PV file {File}", inputPath); | ||
| failedCount++; | ||
| processed++; | ||
| await EnqueueEvent(sseChannel.Writer, BatchConvertPvEventType.FileError, $"{fileName}: {fileEx.Message}"); | ||
| await EnqueueProgress(sseChannel.Writer, processed, total, 100, fileName, failedCount); | ||
| } | ||
| } | ||
|
|
||
| if (cancellationToken.IsCancellationRequested) | ||
| { | ||
| await EnqueueEvent(sseChannel.Writer, BatchConvertPvEventType.Cancelled, $"{processed}/{total}"); | ||
| } | ||
| else | ||
| { | ||
| await EnqueueEvent(sseChannel.Writer, BatchConvertPvEventType.Success, $"{processed}/{total}|{failedCount}"); | ||
| } | ||
| } | ||
| catch (OperationCanceledException) | ||
| { | ||
| await EnqueueEvent(sseChannel.Writer, BatchConvertPvEventType.Cancelled, $"{processed}/{total}"); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| logger.LogError(ex, "Batch PV conversion failed"); | ||
| SentrySdk.CaptureException(ex); | ||
| try | ||
| { | ||
| await EnqueueEvent(sseChannel.Writer, BatchConvertPvEventType.Error, string.Format(Locale.ConvertFailed, ex.Message)); | ||
| } | ||
| catch | ||
| { | ||
| // 客户端可能已断开 | ||
| } | ||
| } | ||
| finally | ||
| { | ||
| sseChannel.Writer.TryComplete(); | ||
| try | ||
| { | ||
| await writer; | ||
| } | ||
| catch | ||
| { | ||
| // writer 自己负责吞掉客户端断开异常 | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private static string SanitizeSseLine(string data) => | ||
| data.Replace("\r", " ").Replace("\n", " "); | ||
|
|
||
| private static ValueTask EnqueueEvent(ChannelWriter<string> writer, BatchConvertPvEventType eventType, string data) => | ||
| writer.WriteAsync($"event: {eventType}\ndata: {SanitizeSseLine(data)}\n\n"); | ||
|
|
||
| private static ValueTask EnqueueProgress(ChannelWriter<string> writer, int processed, int total, int fileProgress, string fileName, int failed) | ||
| { | ||
| var payload = JsonSerializer.Serialize(new BatchProgressPayload(processed, total, fileProgress, fileName, failed), BatchJsonOptions); | ||
| return writer.WriteAsync($"event: {BatchConvertPvEventType.Progress}\ndata: {payload}\n\n"); | ||
| } | ||
|
|
||
| private static void EnqueueProgressFireAndForget(ChannelWriter<string> writer, int processed, int total, int fileProgress, string fileName, int failed) | ||
| { | ||
| var payload = JsonSerializer.Serialize(new BatchProgressPayload(processed, total, fileProgress, fileName, failed), BatchJsonOptions); | ||
| // Channel 是无界的,TryWrite 同步入队,避免在 Xabe 的同步进度事件里 await | ||
| writer.TryWrite($"event: {BatchConvertPvEventType.Progress}\ndata: {payload}\n\n"); | ||
| } | ||
|
|
||
| private async Task WriteSseFrames(ChannelReader<string> reader, CancellationToken cancellationToken) | ||
| { | ||
| try | ||
| { | ||
| await foreach (var frame in reader.ReadAllAsync(cancellationToken)) | ||
| { | ||
| try | ||
| { | ||
| await Response.WriteAsync(frame, cancellationToken); | ||
| await Response.Body.FlushAsync(cancellationToken); | ||
| } | ||
| catch (OperationCanceledException) | ||
| { | ||
| return; | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| logger.LogDebug(ex, "SSE frame write failed (client disconnected?)"); | ||
| return; | ||
| } | ||
| } | ||
| } | ||
| catch (OperationCanceledException) | ||
| { | ||
| // ignore | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P2: Batch conversion no longer refreshes
MovieDataMapafter file changes, which can leave metadata out of sync with disk state until another scan runs.Prompt for AI agents