Refresh real curves after phrase rendering

OpenUtau.Core/Classic/ClassicRenderer.cs

@@ -48,7 +48,7 @@ public RenderResult Layout(RenderPhrase phrase) {
             };
         }
 
-        public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) {
+        public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender, RenderPhraseEvents? renderEvents = null) {
             if (phrase.wavtool == SharpWavtool.nameConvergence || phrase.wavtool == SharpWavtool.nameSimple) {
                 return RenderInternal(phrase, progress, trackNo, cancellation, isPreRender);
             } else {

OpenUtau.Core/Classic/WorldlineRenderer.cs

@@ -63,7 +63,7 @@ public RenderResult Layout(RenderPhrase phrase) {
             };
         }
 
-        public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) {
+        public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender, RenderPhraseEvents? renderEvents = null) {
             var resamplerItems = new List<ResamplerItem>();
             foreach (var phone in phrase.phones) {
                 resamplerItems.Add(new ResamplerItem(phrase, phone));

OpenUtau.Core/Commands/ExpCommands.cs

@@ -327,6 +327,7 @@ public override ValidateOptions ValidateOptions
         public SetCurveCommand(UProject project, UVoicePart part, string abbr, int x, int y, int lastX, int lastY) : base(part) {
             this.project = project;
             this.abbr = abbr;
+            Key = abbr;
             this.x = x;
             this.y = y;
             this.lastX = lastX;
@@ -388,6 +389,7 @@ public MergedSetCurveCommand(UProject project, UVoicePart part,
             string abbr, int[] oldXs, int[] oldYs, int[] newXs, int[] newYs, bool setReal = false) : base(part) {
             this.project = project;
             this.abbr = abbr;
+            Key = setReal ? string.Empty : abbr;
             this.oldXs = oldXs;
             this.oldYs = oldYs;
             this.newXs = newXs;
@@ -439,6 +441,7 @@ public class PasteCurveCommand : ExpCommand {
         public PasteCurveCommand(UProject project, UVoicePart part, string abbr, IEnumerable<int> xs, IEnumerable<int> ys) : base(part) {
             this.project = project;
             this.abbr = abbr;
+            Key = abbr;
             this.xs = xs.ToArray();
             this.ys = ys.ToArray();
             var curve = part.curves.FirstOrDefault(c => c.abbr == abbr);
@@ -448,6 +451,7 @@ public PasteCurveCommand(UProject project, UVoicePart part, string abbr, IEnumer
         public PasteCurveCommand(UProject project, UVoicePart part, string abbr, int startX, int startY, int endX, int endY) : base(part) {
             this.project = project;
             this.abbr = abbr;
+            Key = abbr;
             this.xs = new int[] { startX, endX };
             this.ys = new int[] { startY, endY };
             var curve = part.curves.FirstOrDefault(c => c.abbr == abbr);
@@ -499,6 +503,7 @@ public class ClearCurveCommand : ExpCommand {
         readonly int[] oldYs;
         public ClearCurveCommand(UVoicePart part, string abbr) : base(part) {
             this.abbr = abbr;
+            Key = abbr;
             var curve = Part.curves.FirstOrDefault(curve => curve.abbr == abbr);
             if (curve != null) {
                 oldXs = curve.xs.ToArray();

OpenUtau.Core/Commands/Notifications.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using OpenUtau.Core.Render;
 using OpenUtau.Core.Ustx;
 
 namespace OpenUtau.Core {
@@ -233,6 +235,30 @@ public PartRenderedNotification(UVoicePart part) {
         public override string ToString() => "Part rendered.";
     }
 
+    public class PhraseRenderedNotification : UNotification {
+        public readonly RenderPhrase phrase;
+        public readonly RenderResult result;
+        public readonly int trackNo;
+        public override bool Silent => true;
+        public PhraseRenderedNotification(UVoicePart part, RenderPhrase phrase, RenderResult result, int trackNo) {
+            this.part = part;
+            this.phrase = phrase;
+            this.result = result;
+            this.trackNo = trackNo;
+        }
+        public override string ToString() => "Phrase rendered.";
+    }
+
+    public class RealCurvesUpdatedNotification : UNotification {
+        public readonly IReadOnlyList<RealCurveUpdate> updates;
+        public override bool Silent => true;
+        public RealCurvesUpdatedNotification(UVoicePart part, IReadOnlyList<RealCurveUpdate> updates) {
+            this.part = part;
+            this.updates = updates;
+        }
+        public override string ToString() => "Real curves updated.";
+    }
+
     public class GotoOtoNotification : UNotification {
         public readonly USinger? singer;
         public readonly UOto? oto;

OpenUtau.Core/DiffSinger/DiffSingerRealCurveScheduler.cs

@@ -0,0 +1,103 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using OpenUtau.Core.Render;
+using OpenUtau.Core.Ustx;
+using Serilog;
+
+namespace OpenUtau.Core.DiffSinger {
+    internal static class DiffSingerRealCurveScheduler {
+        // Coalesce drag-generated curve commands without adding pointer-level preview logic.
+        const int DebounceMs = 200;
+        static readonly object lockObj = new object();
+        static readonly Dictionary<UVoicePart, CancellationTokenSource> pending = new Dictionary<UVoicePart, CancellationTokenSource>();
+
+        public static void TrySchedule(UProject project, UVoicePart part, UCommand command) {
+            if (command is not ExpCommand expCommand ||
+                !IsCurveEditCommand(command) ||
+                string.IsNullOrEmpty(expCommand.Key) ||
+                expCommand.Part != part ||
+                !CanRefresh(project, part, expCommand.Key)) {
+                return;
+            }
+            Schedule(project, part);
+        }
+
+        static bool IsCurveEditCommand(UCommand command) {
+            return command is SetCurveCommand ||
+                command is MergedSetCurveCommand ||
+                command is PasteCurveCommand ||
+                command is ClearCurveCommand;
+        }
+
+        static bool CanRefresh(UProject project, UVoicePart part, string abbr) {
+            if (!DiffSingerRenderer.ShouldRefreshRealCurvesOnCurveEdit(abbr) ||
+                !project.parts.Contains(part) ||
+                part.trackNo < 0 ||
+                part.trackNo >= project.tracks.Count) {
+                return false;
+            }
+            return project.tracks[part.trackNo].RendererSettings.Renderer is DiffSingerRenderer;
+        }
+
+        static void Schedule(UProject project, UVoicePart part) {
+            var cancellation = new CancellationTokenSource();
+            lock (lockObj) {
+                if (pending.TryGetValue(part, out var previous)) {
+                    previous.Cancel();
+                }
+                pending[part] = cancellation;
+            }
+            _ = Task.Run(() => RefreshAsync(project, part, cancellation));
+        }
+
+        static async Task RefreshAsync(UProject project, UVoicePart part, CancellationTokenSource cancellation) {
+            try {
+                await Task.Delay(DebounceMs, cancellation.Token);
+                var updates = LoadPartUpdates(project, part, cancellation.Token);
+                if (updates.Count > 0 && !cancellation.IsCancellationRequested) {
+                    DocManager.Inst.ExecuteCmd(new RealCurvesUpdatedNotification(part, updates));
+                }
+            } catch (OperationCanceledException) {
+            } catch (Exception e) {
+                Log.Debug(e, "Failed to refresh DiffSinger real curves after curve edit.");
+            } finally {
+                lock (lockObj) {
+                    if (pending.TryGetValue(part, out var current) && current == cancellation) {
+                        pending.Remove(part);
+                    }
+                }
+                cancellation.Dispose();
+            }
+        }
+
+        static IReadOnlyList<RealCurveUpdate> LoadPartUpdates(
+            UProject project,
+            UVoicePart part,
+            CancellationToken cancellationToken) {
+            RenderPhrase[] phrases;
+            lock (project) {
+                if (!project.parts.Contains(part) ||
+                    part.trackNo < 0 ||
+                    part.trackNo >= project.tracks.Count ||
+                    project.tracks[part.trackNo].RendererSettings.Renderer is not DiffSingerRenderer) {
+                    return Array.Empty<RealCurveUpdate>();
+                }
+                phrases = part.renderPhrases.ToArray();
+            }
+            if (phrases.Length == 0) {
+                return Array.Empty<RealCurveUpdate>();
+            }
+            var updates = new List<RealCurveUpdate>();
+            foreach (var phrase in phrases) {
+                if (cancellationToken.IsCancellationRequested) {
+                    return Array.Empty<RealCurveUpdate>();
+                }
+                updates.AddRange(RealCurveUpdater.LoadPhraseUpdates(part, phrase));
+            }
+            return updates;
+        }
+    }
+}

OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs

@@ -79,7 +79,7 @@ public RenderResult Layout(RenderPhrase phrase) {
             };
         }
 
-        public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) {
+        public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender, RenderPhraseEvents? renderEvents = null) {
             var task = Task.Run(() => {
                 lock (lockObj) {
                     if (cancellation.IsCancellationRequested) {
@@ -114,7 +114,7 @@ public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int tra
                         }
                     }
                     if (result.samples == null) {
-                        result.samples = InvokeDiffsinger(phrase, depth, steps, cancellation);
+                        result.samples = InvokeDiffsinger(phrase, depth, steps, cancellation, renderEvents);
                         if (result.samples != null) {
                             var source = new WaveSource(0, 0, 0, 1);
                             source.SetSamples(result.samples);
@@ -135,7 +135,7 @@ public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int tra
         leadingMs、positionMs、estimatedLengthMs: timeaxis layout in Ms, double
          */
 
-        float[] InvokeDiffsinger(RenderPhrase phrase, double depth, int steps, CancellationTokenSource cancellation) {
+        float[] InvokeDiffsinger(RenderPhrase phrase, double depth, int steps, CancellationTokenSource cancellation, RenderPhraseEvents? renderEvents) {
             var singer = phrase.singer as DiffSingerSinger;
             //Check if dsconfig.yaml is correct
             if(String.IsNullOrEmpty(singer.dsConfig.vocoder) ||
@@ -348,6 +348,7 @@ float[] InvokeDiffsinger(RenderPhrase phrase, double depth, int steps, Cancellat
                     }
                     varianceResult = singer.getVariancePredictor().Process(phrase);
                 }
+                renderEvents?.ReportRealCurves(BuildRenderedRealCurves(phrase, varianceResult));
                 //TODO: let user edit variance curves
                 if(singer.dsConfig.useEnergyEmbed){
                     var energyCurve = phrase.curves.FirstOrDefault(curve => curve.Item1 == ENE);
@@ -508,6 +509,10 @@ public RenderPitchResult LoadRenderedPitch(RenderPhrase phrase) {
             }
         }
 
+        public void ScheduleRealCurveRefresh(UProject project, UVoicePart part, UCommand command) {
+            DiffSingerRealCurveScheduler.TrySchedule(project, part, command);
+        }
+
         public List<RenderRealCurveResult> LoadRenderedRealCurves(RenderPhrase phrase) {
             if (!Preferences.Default.DiffSingerTensorCache) {
                 throw new Exception("Please enable DiffSinger tensor cache and re-render the phrase to display correct base curves.");
@@ -519,57 +524,68 @@ public List<RenderRealCurveResult> LoadRenderedRealCurves(RenderPhrase phrase) {
             var variancePredictor = singer.getVariancePredictor()!;
             lock (variancePredictor) {
                 var result = variancePredictor.Process(phrase);
-                var frameMs = result.frameMs;
-                var headFrames = result.headFrames;
-                var tailFrames = result.tailFrames;
-                var startMs = phrase.positionMs - headFrames * frameMs;
-                var realCurves = new (string, float[], float[], Func<float, float>)[] {
-                    (
-                        ENE, result.energy ?? Array.Empty<float>(),
-                        phrase.curves.FirstOrDefault(curve => curve.Item1 == ENE)?.Item2
-                        ?? Array.Empty<float>(),
-                        x => Math.Clamp(x, -96f, 0f) / 96f + 1f
-                    ),
-                    (
-                        Format.Ustx.BREC, result.breathiness ?? Array.Empty<float>(), phrase.breathiness,
-                        x => Math.Clamp(x, -96f, 0f) / 96f + 1f
-                    ),
-                    (
-                        Format.Ustx.VOIC, result.voicing ?? Array.Empty<float>(), phrase.voicing,
-                        x => Math.Clamp(x, -96f, 0f) / 96f + 1f
-                    ),
-                    (
-                        Format.Ustx.TENC, result.tension ?? Array.Empty<float>(), phrase.tension,
-                        x => Math.Clamp(x, -10f, 10f) / 20f + 0.5f
-                    ),
-                }.Select(t => {
-                    var abbr = t.Item1;
-                    var realCurve = t.Item2;
-                    if (realCurve.Length == 0) {
-                        return new RenderRealCurveResult {
-                            abbr = abbr,
-                            ticks = Array.Empty<float>(),
-                            values = Array.Empty<float>(),
-                        };
-                    }
-                    var deltaCurve = DiffSingerUtils.SampleCurve(
-                        phrase, t.Item3, 0, frameMs, realCurve.Length,
-                        headFrames, tailFrames, x => x)
-                        .Select(x => (float)x)
-                        .ToArray();
-                    var normFunc = t.Item4;
+                return BuildRenderedRealCurves(phrase, result);
+            }
+        }
+
+        internal static bool ShouldRefreshRealCurvesOnCurveEdit(string abbr) {
+            return abbr == ENE ||
+                abbr == Format.Ustx.BREC ||
+                abbr == Format.Ustx.VOIC ||
+                abbr == Format.Ustx.TENC;
+        }
+
+        private List<RenderRealCurveResult> BuildRenderedRealCurves(RenderPhrase phrase, VarianceResult result) {
+            var frameMs = result.frameMs;
+            var headFrames = result.headFrames;
+            var tailFrames = result.tailFrames;
+            var startMs = phrase.positionMs - headFrames * frameMs;
+            var realCurves = new (string, float[], float[], Func<float, float>)[] {
+                (
+                    ENE, result.energy ?? Array.Empty<float>(),
+                    phrase.curves.FirstOrDefault(curve => curve.Item1 == ENE)?.Item2
+                    ?? Array.Empty<float>(),
+                    x => Math.Clamp(x, -96f, 0f) / 96f + 1f
+                ),
+                (
+                    Format.Ustx.BREC, result.breathiness ?? Array.Empty<float>(), phrase.breathiness,
+                    x => Math.Clamp(x, -96f, 0f) / 96f + 1f
+                ),
+                (
+                    Format.Ustx.VOIC, result.voicing ?? Array.Empty<float>(), phrase.voicing,
+                    x => Math.Clamp(x, -96f, 0f) / 96f + 1f
+                ),
+                (
+                    Format.Ustx.TENC, result.tension ?? Array.Empty<float>(), phrase.tension,
+                    x => Math.Clamp(x, -10f, 10f) / 20f + 0.5f
+                ),
+            }.Select(t => {
+                var abbr = t.Item1;
+                var realCurve = t.Item2;
+                if (realCurve.Length == 0) {
                     return new RenderRealCurveResult {
                         abbr = abbr,
-                        ticks = Enumerable.Range(0, realCurve.Length)
-                            .Select(i => (float)phrase.timeAxis.MsPosToTickPos(startMs + i * frameMs) - phrase.position)
-                            .ToArray(),
-                        values = realCurve.Zip(deltaCurve, varianceDeltaFunctions[abbr])
-                            .Select(normFunc)
-                            .ToArray()
+                        ticks = Array.Empty<float>(),
+                        values = Array.Empty<float>(),
                     };
-                }).ToList();
-                return realCurves;
-            }
+                }
+                var deltaCurve = DiffSingerUtils.SampleCurve(
+                    phrase, t.Item3, 0, frameMs, realCurve.Length,
+                    headFrames, tailFrames, x => x)
+                    .Select(x => (float)x)
+                    .ToArray();
+                var normFunc = t.Item4;
+                return new RenderRealCurveResult {
+                    abbr = abbr,
+                    ticks = Enumerable.Range(0, realCurve.Length)
+                        .Select(i => (float)phrase.timeAxis.MsPosToTickPos(startMs + i * frameMs) - phrase.position)
+                        .ToArray(),
+                    values = realCurve.Zip(deltaCurve, varianceDeltaFunctions[abbr])
+                        .Select(normFunc)
+                        .ToArray()
+                };
+            }).ToList();
+            return realCurves;
         }
 
         public UExpressionDescriptor[] GetSuggestedExpressions(USinger singer, URenderSettings renderSettings) {