Improve DiffSinger variance curve and pitch editing functionality

OpenUtau.Core/Commands/ExpCommands.cs

@@ -317,6 +317,8 @@ public class SetCurveCommand : ExpCommand {
         readonly int lastY;
         int[] oldXs;
         int[] oldYs;
+        public int StartTick => Math.Min(x, lastX);
+        public int EndTick => Math.Max(x, lastX) + 1;
         public override ValidateOptions ValidateOptions
             => new ValidateOptions {
                 SkipTiming = true,
@@ -384,6 +386,14 @@ public class MergedSetCurveCommand : ExpCommand {
         readonly int[] newXs;
         readonly int[] newYs;
         readonly bool setReal;
+        public int StartTick => (oldXs ?? Array.Empty<int>())
+            .Concat(newXs ?? Array.Empty<int>())
+            .DefaultIfEmpty(0)
+            .Min();
+        public int EndTick => (oldXs ?? Array.Empty<int>())
+            .Concat(newXs ?? Array.Empty<int>())
+            .DefaultIfEmpty(Part.Duration)
+            .Max() + 1;
         public MergedSetCurveCommand(UProject project, UVoicePart part,
             string abbr, int[] oldXs, int[] oldYs, int[] newXs, int[] newYs, bool setReal = false) : base(part) {
             this.project = project;

OpenUtau.Core/Commands/Notifications.cs

@@ -222,8 +222,55 @@ public FocusNoteNotification(UPart part, UNote note) {
         public override string ToString() => $"Focus note {note.lyric} at {note.position}.";
     }
 
+    public class PreRenderPriority {
+        public readonly UVoicePart part;
+        public readonly int startTick;
+        public readonly int endTick;
+
+        public PreRenderPriority(UVoicePart part, int startTick, int endTick) {
+            this.part = part;
+            this.startTick = startTick;
+            this.endTick = endTick;
+        }
+    }
+
     public class PreRenderNotification : UNotification {
-        public override string ToString() => $"Pre-render notification.";
+        public readonly PreRenderPriority[] priorities;
+        public UVoicePart? priorityPart => System.Linq.Enumerable.FirstOrDefault(priorities)?.part;
+        public int priorityStartTick => System.Linq.Enumerable.FirstOrDefault(priorities)?.startTick ?? -1;
+        public int priorityEndTick => System.Linq.Enumerable.FirstOrDefault(priorities)?.endTick ?? -1;
+        public bool HasPriorityRange => priorities.Length > 0;
+
+        public PreRenderNotification() {
+            priorities = Array.Empty<PreRenderPriority>();
+        }
+
+        public PreRenderNotification(UVoicePart priorityPart, int priorityStartTick, int priorityEndTick)
+            : this(new[] { new PreRenderPriority(priorityPart, priorityStartTick, priorityEndTick) }) { }
+
+        public PreRenderNotification(System.Collections.Generic.IEnumerable<PreRenderPriority> priorities) {
+            this.priorities = System.Linq.Enumerable.ToArray(
+                System.Linq.Enumerable.Where(priorities, priority => priority.endTick > priority.startTick));
+            part = priorityPart;
+        }
+
+        public override string ToString() => HasPriorityRange
+            ? $"Pre-render notification {priorities.Length} prioritized range(s)."
+            : $"Pre-render notification.";
+    }
+
+    public class PhraseRenderedNotification : UNotification {
+        public PhraseRenderedNotification(UVoicePart part) {
+            this.part = part;
+        }
+        public override string ToString() => "Phrase rendered.";
+    }
+
+    public class RealCurvesRenderedNotification : UNotification {
+        public RealCurvesRenderedNotification(UVoicePart part) {
+            this.part = part;
+        }
+        public override string ToString() => "Real curves rendered.";
     }
 
     public class PartRenderedNotification : UNotification {

OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs

@@ -81,6 +81,9 @@ public RenderResult Layout(RenderPhrase phrase) {
 
         public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) {
             var task = Task.Run(() => {
+                if (cancellation.IsCancellationRequested) {
+                    return new RenderResult();
+                }
                 lock (lockObj) {
                     if (cancellation.IsCancellationRequested) {
                         return new RenderResult();

OpenUtau.Core/DiffSinger/DiffSingerVariance.cs

@@ -24,6 +24,7 @@ public struct VarianceResult{
         public int totalFrames;
     }
     public class DsVariance : IDisposable{
+        const ulong AcceptedVarianceCacheSalt = 0x9E3779B97F4A7C15UL;
         string rootPath;
         DsConfig dsConfig;
         Dictionary<string, int> languageIds = new Dictionary<string, int>();
@@ -257,6 +258,17 @@ public VarianceResult Process(RenderPhrase phrase){
                 varianceInputs.Add(NamedOnnxValue.CreateFromTensor("spk_embed", spkEmbedTensor));
             }
             Onnx.VerifyInputNames(varianceModel, varianceInputs);
+            var acceptedVarianceCache = Preferences.Default.DiffSingerTensorCache
+                ? new DiffSingerCache(varianceHash ^ AcceptedVarianceCacheSalt, AcceptedVarianceCacheInputs(varianceInputs))
+                : null;
+            ulong pitchHash = DiffSingerVariancePatcher.HashPitch(pitch);
+            var acceptedVariance = acceptedVarianceCache == null
+                ? null
+                : LoadAcceptedVariance(acceptedVarianceCache, frameMs, headFrames, tailFrames, totalFrames);
+            if (acceptedVariance != null && acceptedVariance.PitchHash == pitchHash) {
+                phrase.AddCacheFile(acceptedVarianceCache?.Filename);
+                return acceptedVariance.Result;
+            }
             var varianceCache = Preferences.Default.DiffSingerTensorCache
                 ? new DiffSingerCache(varianceHash, varianceInputs)
                 : null;
@@ -290,7 +302,7 @@ public VarianceResult Process(RenderPhrase phrase){
                     .First()
                     .AsTensor<float>()
                 : null;
-            return new VarianceResult{
+            var fullResult = new VarianceResult{
                 energy = energy_pred?.ToArray(),
                 breathiness = breathiness_pred?.ToArray(),
                 voicing = voicing_pred?.ToArray(),
@@ -300,6 +312,111 @@ public VarianceResult Process(RenderPhrase phrase){
                 tailFrames = tailFrames,
                 totalFrames = totalFrames,
             };
+            if (acceptedVarianceCache == null) {
+                return fullResult;
+            }
+            var result = fullResult;
+            if (acceptedVariance?.Pitch != null) {
+                result = DiffSingerVariancePatcher.PatchByPitchChange(
+                    acceptedVariance.Result, fullResult, acceptedVariance.Pitch, pitch);
+            }
+            SaveAcceptedVariance(acceptedVarianceCache, result, pitchHash, pitch);
+            phrase.AddCacheFile(acceptedVarianceCache.Filename);
+            return result;
+        }
+
+        ICollection<NamedOnnxValue> AcceptedVarianceCacheInputs(ICollection<NamedOnnxValue> varianceInputs) {
+            return varianceInputs
+                .Where(input => input.Name != "pitch" && input.Name != "retake")
+                .ToArray();
+        }
+
+        class AcceptedVariance {
+            public VarianceResult Result;
+            public ulong PitchHash;
+            public float[]? Pitch;
+        }
+
+        AcceptedVariance? LoadAcceptedVariance(
+            DiffSingerCache cache,
+            float expectedFrameMs,
+            int expectedHeadFrames,
+            int expectedTailFrames,
+            int expectedTotalFrames) {
+            var outputs = cache.Load();
+            if (outputs == null) {
+                return null;
+            }
+            try {
+                var frameMsValue = GetRequiredFloat(outputs, "frame_ms");
+                var headFramesValue = (int)GetRequiredLong(outputs, "head_frames");
+                var tailFramesValue = (int)GetRequiredLong(outputs, "tail_frames");
+                var totalFramesValue = (int)GetRequiredLong(outputs, "total_frames");
+                if (Math.Abs(frameMsValue - expectedFrameMs) > 0.0001f ||
+                    headFramesValue != expectedHeadFrames ||
+                    tailFramesValue != expectedTailFrames ||
+                    totalFramesValue != expectedTotalFrames) {
+                    return null;
+                }
+                return new AcceptedVariance {
+                    PitchHash = unchecked((ulong)GetRequiredLong(outputs, "pitch_hash")),
+                    Pitch = GetOptionalFloatArray(outputs, "pitch"),
+                    Result = new VarianceResult {
+                        energy = GetOptionalFloatArray(outputs, "energy_pred"),
+                        breathiness = GetOptionalFloatArray(outputs, "breathiness_pred"),
+                        voicing = GetOptionalFloatArray(outputs, "voicing_pred"),
+                        tension = GetOptionalFloatArray(outputs, "tension_pred"),
+                        frameMs = frameMsValue,
+                        headFrames = headFramesValue,
+                        tailFrames = tailFramesValue,
+                        totalFrames = totalFramesValue,
+                    }
+                };
+            } catch (Exception e) {
+                Log.Error(e, "Failed to load accepted variance cache.");
+                cache.Delete();
+                return null;
+            }
+        }
+
+        void SaveAcceptedVariance(DiffSingerCache cache, VarianceResult result, ulong pitchHash, float[] pitch) {
+            var outputs = new List<NamedOnnxValue> {
+                NamedOnnxValue.CreateFromTensor("pitch_hash",
+                    new DenseTensor<long>(new[] { unchecked((long)pitchHash) }, new[] { 1 })),
+                NamedOnnxValue.CreateFromTensor("frame_ms",
+                    new DenseTensor<float>(new[] { result.frameMs }, new[] { 1 })),
+                NamedOnnxValue.CreateFromTensor("head_frames",
+                    new DenseTensor<long>(new[] { (long)result.headFrames }, new[] { 1 })),
+                NamedOnnxValue.CreateFromTensor("tail_frames",
+                    new DenseTensor<long>(new[] { (long)result.tailFrames }, new[] { 1 })),
+                NamedOnnxValue.CreateFromTensor("total_frames",
+                    new DenseTensor<long>(new[] { (long)result.totalFrames }, new[] { 1 })),
+            };
+            AddOptionalFloatArray(outputs, "pitch", pitch);
+            AddOptionalFloatArray(outputs, "energy_pred", result.energy);
+            AddOptionalFloatArray(outputs, "breathiness_pred", result.breathiness);
+            AddOptionalFloatArray(outputs, "voicing_pred", result.voicing);
+            AddOptionalFloatArray(outputs, "tension_pred", result.tension);
+            cache.Save(outputs);
+        }
+
+        static void AddOptionalFloatArray(List<NamedOnnxValue> outputs, string name, float[]? values) {
+            if (values != null) {
+                outputs.Add(NamedOnnxValue.CreateFromTensor(
+                    name, new DenseTensor<float>(values, new[] { values.Length })));
+            }
+        }
+
+        static float[]? GetOptionalFloatArray(ICollection<NamedOnnxValue> outputs, string name) {
+            return outputs.FirstOrDefault(output => output.Name == name)?.AsTensor<float>().ToArray();
+        }
+
+        static float GetRequiredFloat(ICollection<NamedOnnxValue> outputs, string name) {
+            return outputs.First(output => output.Name == name).AsTensor<float>().First();
+        }
+
+        static long GetRequiredLong(ICollection<NamedOnnxValue> outputs, string name) {
+            return outputs.First(output => output.Name == name).AsTensor<long>().First();
         }
 
         private bool disposedValue;