From 3e827dc94740c938f07dce6d41cbce556d8a8db7 Mon Sep 17 00:00:00 2001 From: KakaruHayate Date: Tue, 2 Jun 2026 14:37:11 +0800 Subject: [PATCH 1/4] Diffsinger renderer's queue jumping mechanism --- OpenUtau.Core/Commands/ExpCommands.cs | 10 ++ OpenUtau.Core/Commands/Notifications.cs | 42 +++++- .../DiffSinger/DiffSingerRenderer.cs | 3 + OpenUtau.Core/DocManager.cs | 76 +++++++++- OpenUtau.Core/PlaybackManager.cs | 14 +- OpenUtau.Core/Properties/AssemblyInfo.cs | 3 + OpenUtau.Core/Render/RenderEngine.cs | 137 ++++++++++++++++-- OpenUtau.Core/Render/RenderPriority.cs | 53 +++++++ .../Core/Render/RenderPriorityTest.cs | 90 ++++++++++++ OpenUtau/ViewModels/NotesViewModel.cs | 2 +- 10 files changed, 408 insertions(+), 22 deletions(-) create mode 100644 OpenUtau.Core/Properties/AssemblyInfo.cs create mode 100644 OpenUtau.Core/Render/RenderPriority.cs create mode 100644 OpenUtau.Test/Core/Render/RenderPriorityTest.cs diff --git a/OpenUtau.Core/Commands/ExpCommands.cs b/OpenUtau.Core/Commands/ExpCommands.cs index 072e12c71..e338b733a 100644 --- a/OpenUtau.Core/Commands/ExpCommands.cs +++ b/OpenUtau.Core/Commands/ExpCommands.cs @@ -317,6 +317,8 @@ public class SetCurveCommand : ExpCommand { readonly int lastY; int[] oldXs; int[] oldYs; + public int StartTick => Math.Min(x, lastX); + public int EndTick => Math.Max(x, lastX) + 1; public override ValidateOptions ValidateOptions => new ValidateOptions { SkipTiming = true, @@ -384,6 +386,14 @@ public class MergedSetCurveCommand : ExpCommand { readonly int[] newXs; readonly int[] newYs; readonly bool setReal; + public int StartTick => (oldXs ?? Array.Empty()) + .Concat(newXs ?? Array.Empty()) + .DefaultIfEmpty(0) + .Min(); + public int EndTick => (oldXs ?? Array.Empty()) + .Concat(newXs ?? Array.Empty()) + .DefaultIfEmpty(Part.Duration) + .Max() + 1; public MergedSetCurveCommand(UProject project, UVoicePart part, string abbr, int[] oldXs, int[] oldYs, int[] newXs, int[] newYs, bool setReal = false) : base(part) { this.project = project; diff --git a/OpenUtau.Core/Commands/Notifications.cs b/OpenUtau.Core/Commands/Notifications.cs index 0ee6448fa..4724f90af 100644 --- a/OpenUtau.Core/Commands/Notifications.cs +++ b/OpenUtau.Core/Commands/Notifications.cs @@ -222,8 +222,48 @@ public FocusNoteNotification(UPart part, UNote note) { public override string ToString() => $"Focus note {note.lyric} at {note.position}."; } + public class PreRenderPriority { + public readonly UVoicePart part; + public readonly int startTick; + public readonly int endTick; + + public PreRenderPriority(UVoicePart part, int startTick, int endTick) { + this.part = part; + this.startTick = startTick; + this.endTick = endTick; + } + } + public class PreRenderNotification : UNotification { - public override string ToString() => $"Pre-render notification."; + public readonly PreRenderPriority[] priorities; + public UVoicePart? priorityPart => System.Linq.Enumerable.FirstOrDefault(priorities)?.part; + public int priorityStartTick => System.Linq.Enumerable.FirstOrDefault(priorities)?.startTick ?? -1; + public int priorityEndTick => System.Linq.Enumerable.FirstOrDefault(priorities)?.endTick ?? -1; + public bool HasPriorityRange => priorities.Length > 0; + + public PreRenderNotification() { + priorities = Array.Empty(); + } + + public PreRenderNotification(UVoicePart priorityPart, int priorityStartTick, int priorityEndTick) + : this(new[] { new PreRenderPriority(priorityPart, priorityStartTick, priorityEndTick) }) { } + + public PreRenderNotification(System.Collections.Generic.IEnumerable priorities) { + this.priorities = System.Linq.Enumerable.ToArray( + System.Linq.Enumerable.Where(priorities, priority => priority.endTick > priority.startTick)); + part = priorityPart; + } + + public override string ToString() => HasPriorityRange + ? $"Pre-render notification {priorities.Length} prioritized range(s)." + : $"Pre-render notification."; + } + + public class PhraseRenderedNotification : UNotification { + public PhraseRenderedNotification(UVoicePart part) { + this.part = part; + } + public override string ToString() => "Phrase rendered."; } public class PartRenderedNotification : UNotification { diff --git a/OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs b/OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs index 7844eea55..32556105a 100644 --- a/OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs +++ b/OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs @@ -81,6 +81,9 @@ public RenderResult Layout(RenderPhrase phrase) { public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { var task = Task.Run(() => { + if (cancellation.IsCancellationRequested) { + return new RenderResult(); + } lock (lockObj) { if (cancellation.IsCancellationRequested) { return new RenderResult(); diff --git a/OpenUtau.Core/DocManager.cs b/OpenUtau.Core/DocManager.cs index 22fb2fbba..16904f955 100644 --- a/OpenUtau.Core/DocManager.cs +++ b/OpenUtau.Core/DocManager.cs @@ -278,10 +278,80 @@ public void EndUndoGroup() { if (undoGroup.DeferValidate) { Project.ValidateFull(); } + var preRenderNotification = CreatePreRenderNotification(undoGroup.Commands); undoGroup.Merge(); undoGroup = null; Log.Information("undoGroup ended"); - ExecuteCmd(new PreRenderNotification()); + ExecuteCmd(preRenderNotification); + } + + PreRenderNotification CreatePreRenderNotification(IEnumerable commands) { + var priorityRanges = new Dictionary(); + foreach (var cmd in commands) { + if (!TryGetPreRenderRange(cmd, out var part, out var cmdStartTick, out var cmdEndTick) || + part == null || + !Project.parts.Contains(part)) { + continue; + } + if (priorityRanges.TryGetValue(part, out var range)) { + priorityRanges[part] = ( + Math.Min(range.startTick, cmdStartTick), + Math.Max(range.endTick, cmdEndTick)); + } else { + priorityRanges[part] = (cmdStartTick, cmdEndTick); + } + } + if (priorityRanges.Count > 0) { + return new PreRenderNotification(priorityRanges.Select(range => + new PreRenderPriority(range.Key, range.Value.startTick, range.Value.endTick))); + } + return new PreRenderNotification(); + } + + bool TryGetPreRenderRange(UCommand cmd, out UVoicePart? part, out int startTick, out int endTick) { + part = null; + startTick = 0; + endTick = 0; + switch (cmd) { + case NoteCommand noteCommand when noteCommand.Notes.Length > 0: + part = noteCommand.Part; + startTick = part.position + noteCommand.Notes.Min(note => note.position); + endTick = part.position + noteCommand.Notes.Max(note => note.End); + return endTick > startTick; + case SetCurveCommand curveCommand: + part = curveCommand.Part; + startTick = part.position + curveCommand.StartTick; + endTick = part.position + curveCommand.EndTick; + return endTick > startTick; + case MergedSetCurveCommand curveCommand: + part = curveCommand.Part; + startTick = part.position + curveCommand.StartTick; + endTick = part.position + curveCommand.EndTick; + return endTick > startTick; + case ExpCommand expCommand when expCommand.Note != null: + part = expCommand.Part; + startTick = part.position + expCommand.Note.position; + endTick = part.position + expCommand.Note.End; + return endTick > startTick; + case ExpCommand expCommand: + part = expCommand.Part; + startTick = part.position; + endTick = part.End; + return endTick > startTick; + case PartCommand partCommand when partCommand.part is UVoicePart voicePart: + part = voicePart; + startTick = part.position; + endTick = part.End; + return endTick > startTick; + default: + if (cmd.ValidateOptions.Part is UVoicePart validatePart) { + part = validatePart; + startTick = part.position; + endTick = part.End; + return endTick > startTick; + } + return false; + } } public void RollBackUndoGroup() { @@ -314,7 +384,7 @@ public void Undo() { Publish(cmd, true); } redoQueue.AddToBack(group); - ExecuteCmd(new PreRenderNotification()); + ExecuteCmd(CreatePreRenderNotification(group.Commands)); } public void Redo() { @@ -331,7 +401,7 @@ public void Redo() { Publish(cmd); } undoQueue.AddToBack(group); - ExecuteCmd(new PreRenderNotification()); + ExecuteCmd(CreatePreRenderNotification(group.Commands)); } public bool GetUndoState(out string? key) { diff --git a/OpenUtau.Core/PlaybackManager.cs b/OpenUtau.Core/PlaybackManager.cs index 9bc67a6ed..a0183480f 100644 --- a/OpenUtau.Core/PlaybackManager.cs +++ b/OpenUtau.Core/PlaybackManager.cs @@ -373,9 +373,15 @@ private void CheckFileWritable(string filePath) { } } - void SchedulePreRender() { - Log.Information("SchedulePreRender"); - var engine = new RenderEngine(DocManager.Inst.Project); + void SchedulePreRender(PreRenderNotification? notification = null) { + Log.Information(notification?.HasPriorityRange == true + ? $"SchedulePreRender {notification.priorities.Length} prioritized range(s)" + : "SchedulePreRender"); + var engine = notification?.HasPriorityRange == true + ? new RenderEngine( + DocManager.Inst.Project, + priorityRanges: notification.priorities) + : new RenderEngine(DocManager.Inst.Project); engine.PreRenderProject(ref renderCancellation); } @@ -404,7 +410,7 @@ public void OnNext(UCommand cmd, bool isUndo) { } if (cmd is PreRenderNotification || cmd is LoadProjectNotification) { if (Util.Preferences.Default.PreRender) { - SchedulePreRender(); + SchedulePreRender(cmd as PreRenderNotification); } } } diff --git a/OpenUtau.Core/Properties/AssemblyInfo.cs b/OpenUtau.Core/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..a1f94316e --- /dev/null +++ b/OpenUtau.Core/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("OpenUtau.Test")] diff --git a/OpenUtau.Core/Render/RenderEngine.cs b/OpenUtau.Core/Render/RenderEngine.cs index b8bde5fa8..0af6b8f5e 100644 --- a/OpenUtau.Core/Render/RenderEngine.cs +++ b/OpenUtau.Core/Render/RenderEngine.cs @@ -47,12 +47,30 @@ class RenderEngine { readonly int startTick; readonly int endTick; readonly int trackNo; + readonly PreRenderPriority[] priorityRanges; - public RenderEngine(UProject project, int startTick = 0, int endTick = -1, int trackNo = -1) { + public RenderEngine( + UProject project, + int startTick = 0, + int endTick = -1, + int trackNo = -1, + UVoicePart? priorityPart = null, + int priorityStartTick = -1, + int priorityEndTick = -1, + IEnumerable? priorityRanges = null) { this.project = project; this.startTick = startTick; this.endTick = endTick; this.trackNo = trackNo; + if (priorityRanges != null) { + this.priorityRanges = priorityRanges + .Where(priority => priority.endTick > priority.startTick) + .ToArray(); + } else if (priorityPart != null && priorityEndTick > priorityStartTick) { + this.priorityRanges = new[] { new PreRenderPriority(priorityPart, priorityStartTick, priorityEndTick) }; + } else { + this.priorityRanges = Array.Empty(); + } } // for playback or export @@ -220,35 +238,128 @@ private void RenderRequests( } var tuples = requests .SelectMany(req => req.phrases - .Zip(req.sources, (phrase, source) => Tuple.Create(phrase, source, req))) + .Zip(req.sources, (phrase, source) => (phrase, source, request: req))) .ToArray(); - if (playing) { - var orderedTuples = tuples - .Where(tuple => tuple.Item1.end > startTick) - .OrderBy(tuple => tuple.Item1.end) - .Concat(tuples.Where(tuple => tuple.Item1.end <= startTick)) - .ToArray(); - tuples = orderedTuples; + if (tuples.Length == 0) { + return; + } + if (tuples.Any(tuple => IsDiffSinger(tuple.phrase))) { + if (playing) { + tuples = OrderForPlayback(tuples); + } else if (GetDiffSingerPriorityRanges().Length > 0) { + tuples = OrderForPreRender(tuples); + } } var progress = new Progress(tuples.Sum(t => t.Item1.phones.Length)); foreach (var tuple in tuples) { - var phrase = tuple.Item1; - var source = tuple.Item2; - var request = tuple.Item3; + if (cancellation.IsCancellationRequested) { + break; + } + var phrase = tuple.phrase; + var source = tuple.source; + var request = tuple.request; var task = phrase.renderer.Render(phrase, progress, request.trackNo, cancellation, true); task.Wait(); if (cancellation.IsCancellationRequested) { break; } source.SetSamples(task.Result.samples); - if (request.sources.All(s => s.HasSamples)) { + bool partReady = request.sources.All(s => s.HasSamples); + if (IsDiffSinger(phrase)) { request.part.SetMix(request.mix); + DocManager.Inst.ExecuteCmd(new PhraseRenderedNotification(request.part)); + } + if (partReady) { + if (!IsDiffSinger(phrase)) { + request.part.SetMix(request.mix); + } DocManager.Inst.ExecuteCmd(new PartRenderedNotification(request.part)); } } progress.Clear(); } + private (RenderPhrase phrase, WaveSource source, RenderPartRequest request)[] OrderForPlayback( + (RenderPhrase phrase, WaveSource source, RenderPartRequest request)[] tuples) { + double playbackStartMs = project.timeAxis.TickPosToMsPos(startTick); + return tuples + .Select((tuple, index) => (tuple, index)) + .OrderBy(item => RenderPriority.PlaybackBucket( + item.tuple.source.offsetMs, item.tuple.source.EndMs, playbackStartMs)) + .ThenBy(item => RenderPriority.PlaybackDistance( + item.tuple.source.offsetMs, item.tuple.source.EndMs, playbackStartMs)) + .ThenBy(item => item.index) + .Select(item => item.tuple) + .ToArray(); + } + + private (RenderPhrase phrase, WaveSource source, RenderPartRequest request)[] OrderForPreRender( + (RenderPhrase phrase, WaveSource source, RenderPartRequest request)[] tuples) { + var priorities = GetDiffSingerPriorityRanges(); + return tuples + .Select((tuple, index) => (tuple, index)) + .OrderBy(item => PreRenderPriorityBucket(item.tuple, priorities)) + .ThenBy(item => PreRenderPriorityIndex(item.tuple, priorities)) + .ThenBy(item => PreRenderPriorityDistance(item.tuple.phrase, priorities)) + .ThenBy(item => item.index) + .Select(item => item.tuple) + .ToArray(); + } + + private int PreRenderPriorityBucket( + (RenderPhrase phrase, WaveSource source, RenderPartRequest request) tuple, + PreRenderPriority[] priorities) { + bool isDiffSinger = IsDiffSinger(tuple.phrase); + bool isPriorityPart = priorities.Any(priority => ReferenceEquals(tuple.request.part, priority.part)); + bool overlapsPriority = priorities.Any(priority => + ReferenceEquals(tuple.request.part, priority.part) && + RenderPriority.Overlaps(tuple.phrase.position, tuple.phrase.end, priority.startTick, priority.endTick)); + int earliestPriorityStart = priorities.Min(priority => priority.startTick); + return RenderPriority.PreRenderBucket( + isDiffSinger, + isPriorityPart, + overlapsPriority, + tuple.phrase.end > earliestPriorityStart); + } + + private int PreRenderPriorityIndex( + (RenderPhrase phrase, WaveSource source, RenderPartRequest request) tuple, + PreRenderPriority[] priorities) { + for (int i = 0; i < priorities.Length; ++i) { + var priority = priorities[i]; + if (ReferenceEquals(tuple.request.part, priority.part) && + RenderPriority.Overlaps(tuple.phrase.position, tuple.phrase.end, priority.startTick, priority.endTick)) { + return i; + } + } + for (int i = 0; i < priorities.Length; ++i) { + if (ReferenceEquals(tuple.request.part, priorities[i].part)) { + return i; + } + } + return int.MaxValue; + } + + private int PreRenderPriorityDistance(RenderPhrase phrase, PreRenderPriority[] priorities) { + return priorities + .Select(priority => RenderPriority.PreRenderDistance(phrase.position, phrase.end, priority.startTick)) + .DefaultIfEmpty(0) + .Min(); + } + + private bool IsDiffSinger(RenderPhrase phrase) { + return phrase.renderer.SingerType == USingerType.DiffSinger; + } + + private PreRenderPriority[] GetDiffSingerPriorityRanges() { + return priorityRanges + .Where(priority => + priority.part.trackNo >= 0 && + priority.part.trackNo < project.tracks.Count && + project.tracks[priority.part.trackNo].RendererSettings.Renderer?.SingerType == USingerType.DiffSinger) + .ToArray(); + } + public static void ReleaseSourceTemp() { VoicebankFiles.Inst.ReleaseSourceTemp(); } diff --git a/OpenUtau.Core/Render/RenderPriority.cs b/OpenUtau.Core/Render/RenderPriority.cs new file mode 100644 index 000000000..2f79f76ea --- /dev/null +++ b/OpenUtau.Core/Render/RenderPriority.cs @@ -0,0 +1,53 @@ +using System; + +namespace OpenUtau.Core.Render { + internal static class RenderPriority { + internal static int PlaybackBucket(double sourceStartMs, double sourceEndMs, double playbackStartMs) { + if (sourceStartMs <= playbackStartMs && sourceEndMs > playbackStartMs) { + return 0; + } + return sourceStartMs >= playbackStartMs ? 1 : 2; + } + + internal static double PlaybackDistance(double sourceStartMs, double sourceEndMs, double playbackStartMs) { + if (sourceStartMs <= playbackStartMs && sourceEndMs > playbackStartMs) { + return Math.Max(0, playbackStartMs - sourceStartMs); + } + if (sourceStartMs >= playbackStartMs) { + return sourceStartMs - playbackStartMs; + } + return playbackStartMs - sourceEndMs; + } + + internal static bool Overlaps(int startTick, int endTick, int priorityStartTick, int priorityEndTick) { + return endTick > priorityStartTick && startTick < priorityEndTick; + } + + internal static int PreRenderBucket( + bool isDiffSinger, + bool isPriorityPart, + bool overlapsPriority, + bool isAfterPriorityStart) { + if (isDiffSinger && isPriorityPart && overlapsPriority) { + return 0; + } + if (isDiffSinger && isPriorityPart) { + return 1; + } + if (isDiffSinger && isAfterPriorityStart) { + return 2; + } + return 3; + } + + internal static int PreRenderDistance(int phraseStartTick, int phraseEndTick, int priorityStartTick) { + if (phraseStartTick <= priorityStartTick && phraseEndTick > priorityStartTick) { + return 0; + } + if (phraseStartTick >= priorityStartTick) { + return phraseStartTick - priorityStartTick; + } + return priorityStartTick - phraseEndTick; + } + } +} diff --git a/OpenUtau.Test/Core/Render/RenderPriorityTest.cs b/OpenUtau.Test/Core/Render/RenderPriorityTest.cs new file mode 100644 index 000000000..55cac4b1e --- /dev/null +++ b/OpenUtau.Test/Core/Render/RenderPriorityTest.cs @@ -0,0 +1,90 @@ +using System.Linq; +using OpenUtau.Core.Ustx; +using Xunit; + +namespace OpenUtau.Core.Render { + public class RenderPriorityTest { + [Fact] + public void PlaybackPriorityPrefersSourceCoveringStart() { + var ordered = new[] { + ("future", 120.0, 160.0), + ("past", 0.0, 90.0), + ("current", 80.0, 150.0), + } + .OrderBy(item => RenderPriority.PlaybackBucket(item.Item2, item.Item3, 100)) + .ThenBy(item => RenderPriority.PlaybackDistance(item.Item2, item.Item3, 100)) + .Select(item => item.Item1) + .ToArray(); + + Assert.Equal(new[] { "current", "future", "past" }, ordered); + } + + [Fact] + public void PlaybackPriorityUsesDistanceWithinBucket() { + var ordered = new[] { + ("far", 300.0, 360.0), + ("near", 120.0, 180.0), + } + .OrderBy(item => RenderPriority.PlaybackBucket(item.Item2, item.Item3, 100)) + .ThenBy(item => RenderPriority.PlaybackDistance(item.Item2, item.Item3, 100)) + .Select(item => item.Item1) + .ToArray(); + + Assert.Equal(new[] { "near", "far" }, ordered); + } + + [Fact] + public void PreRenderPriorityBucketsEditedDiffSingerPhraseFirst() { + Assert.Equal(0, RenderPriority.PreRenderBucket( + isDiffSinger: true, + isPriorityPart: true, + overlapsPriority: true, + isAfterPriorityStart: true)); + Assert.Equal(1, RenderPriority.PreRenderBucket( + isDiffSinger: true, + isPriorityPart: true, + overlapsPriority: false, + isAfterPriorityStart: true)); + Assert.Equal(2, RenderPriority.PreRenderBucket( + isDiffSinger: true, + isPriorityPart: false, + overlapsPriority: false, + isAfterPriorityStart: true)); + Assert.Equal(3, RenderPriority.PreRenderBucket( + isDiffSinger: false, + isPriorityPart: true, + overlapsPriority: true, + isAfterPriorityStart: true)); + } + + [Fact] + public void PreRenderDistanceIsZeroWhenPhraseCoversPriorityStart() { + Assert.Equal(0, RenderPriority.PreRenderDistance(80, 160, 100)); + Assert.Equal(20, RenderPriority.PreRenderDistance(120, 160, 100)); + Assert.Equal(10, RenderPriority.PreRenderDistance(40, 90, 100)); + } + + [Fact] + public void OverlapsUsesHalfOpenTickRanges() { + Assert.True(RenderPriority.Overlaps(80, 120, 100, 160)); + Assert.False(RenderPriority.Overlaps(80, 100, 100, 160)); + Assert.False(RenderPriority.Overlaps(160, 200, 100, 160)); + } + + [Fact] + public void PreRenderNotificationKeepsMultiplePriorityRanges() { + var part1 = new UVoicePart(); + var part2 = new UVoicePart(); + var notification = new PreRenderNotification(new[] { + new PreRenderPriority(part1, 100, 200), + new PreRenderPriority(part2, 300, 400), + new PreRenderPriority(part2, 500, 500), + }); + + Assert.Equal(2, notification.priorities.Length); + Assert.Same(part1, notification.priorityPart); + Assert.Equal(100, notification.priorityStartTick); + Assert.Equal(200, notification.priorityEndTick); + } + } +} diff --git a/OpenUtau/ViewModels/NotesViewModel.cs b/OpenUtau/ViewModels/NotesViewModel.cs index 1d4d4f582..63f7a4646 100644 --- a/OpenUtau/ViewModels/NotesViewModel.cs +++ b/OpenUtau/ViewModels/NotesViewModel.cs @@ -1101,7 +1101,7 @@ public void OnNext(UCommand cmd, bool isUndo) { } else if (cmd is PhonemizedNotification) { OnPartModified(); MessageBus.Current.SendMessage(new NotesRefreshEvent()); - } else if (notif is PartRenderedNotification && notif.part == Part) { + } else if ((notif is PhraseRenderedNotification || notif is PartRenderedNotification) && notif.part == Part) { MessageBus.Current.SendMessage(new WaveformRefreshEvent()); } } else if (cmd is PartCommand partCommand) { From 6dc14fcefa410b87480b1587abfa0060c00773af Mon Sep 17 00:00:00 2001 From: KakaruHayate Date: Tue, 2 Jun 2026 17:14:32 +0800 Subject: [PATCH 2/4] Improve DiffSinger variance curve refresh --- OpenUtau.Core/Commands/ExpCommands.cs | 9 ++ OpenUtau.Core/Commands/Notifications.cs | 21 ++- .../DiffSinger/DiffSingerVariance.cs | 121 +++++++++++++++- .../DiffSinger/DiffSingerVariancePatcher.cs | 129 ++++++++++++++++++ OpenUtau.Core/DocManager.cs | 92 ++++++++++++- OpenUtau.Core/Render/RealCurveRefresh.cs | 91 ++++++++++++ OpenUtau.Core/Render/RenderContext.cs | 35 +++++ OpenUtau.Core/Render/RenderEngine.cs | 36 +++++ .../DiffSingerVariancePatcherTest.cs | 49 +++++++ OpenUtau/ViewModels/NotesViewModel.cs | 88 ++++++++++++ 10 files changed, 663 insertions(+), 8 deletions(-) create mode 100644 OpenUtau.Core/DiffSinger/DiffSingerVariancePatcher.cs create mode 100644 OpenUtau.Core/Render/RealCurveRefresh.cs create mode 100644 OpenUtau.Core/Render/RenderContext.cs create mode 100644 OpenUtau.Test/Core/DiffSinger/DiffSingerVariancePatcherTest.cs diff --git a/OpenUtau.Core/Commands/ExpCommands.cs b/OpenUtau.Core/Commands/ExpCommands.cs index e338b733a..f17aa4508 100644 --- a/OpenUtau.Core/Commands/ExpCommands.cs +++ b/OpenUtau.Core/Commands/ExpCommands.cs @@ -317,6 +317,7 @@ public class SetCurveCommand : ExpCommand { readonly int lastY; int[] oldXs; int[] oldYs; + public string Abbr => abbr; public int StartTick => Math.Min(x, lastX); public int EndTick => Math.Max(x, lastX) + 1; public override ValidateOptions ValidateOptions @@ -386,6 +387,8 @@ public class MergedSetCurveCommand : ExpCommand { readonly int[] newXs; readonly int[] newYs; readonly bool setReal; + public string Abbr => abbr; + public bool SetReal => setReal; public int StartTick => (oldXs ?? Array.Empty()) .Concat(newXs ?? Array.Empty()) .DefaultIfEmpty(0) @@ -446,6 +449,9 @@ public class PasteCurveCommand : ExpCommand { readonly int[] ys; int[]? oldXs; int[]? oldYs; + public string Abbr => abbr; + public int StartTick => xs.DefaultIfEmpty(0).Min(); + public int EndTick => xs.DefaultIfEmpty(Part.Duration).Max() + 1; public PasteCurveCommand(UProject project, UVoicePart part, string abbr, IEnumerable xs, IEnumerable ys) : base(part) { this.project = project; this.abbr = abbr; @@ -507,6 +513,9 @@ public class ClearCurveCommand : ExpCommand { readonly string abbr; readonly int[] oldXs; readonly int[] oldYs; + public string Abbr => abbr; + public int StartTick => 0; + public int EndTick => Part.Duration; public ClearCurveCommand(UVoicePart part, string abbr) : base(part) { this.abbr = abbr; var curve = Part.curves.FirstOrDefault(curve => curve.abbr == abbr); diff --git a/OpenUtau.Core/Commands/Notifications.cs b/OpenUtau.Core/Commands/Notifications.cs index 4724f90af..a2f416193 100644 --- a/OpenUtau.Core/Commands/Notifications.cs +++ b/OpenUtau.Core/Commands/Notifications.cs @@ -222,15 +222,27 @@ public FocusNoteNotification(UPart part, UNote note) { public override string ToString() => $"Focus note {note.lyric} at {note.position}."; } + public enum PreRenderEditKind { + Generic, + Pitch, + VarianceCurve, + } + public class PreRenderPriority { public readonly UVoicePart part; public readonly int startTick; public readonly int endTick; + public readonly PreRenderEditKind editKind; - public PreRenderPriority(UVoicePart part, int startTick, int endTick) { + public PreRenderPriority( + UVoicePart part, + int startTick, + int endTick, + PreRenderEditKind editKind = PreRenderEditKind.Generic) { this.part = part; this.startTick = startTick; this.endTick = endTick; + this.editKind = editKind; } } @@ -266,6 +278,13 @@ public PhraseRenderedNotification(UVoicePart part) { public override string ToString() => "Phrase rendered."; } + public class RealCurvesRenderedNotification : UNotification { + public RealCurvesRenderedNotification(UVoicePart part) { + this.part = part; + } + public override string ToString() => "Real curves rendered."; + } + public class PartRenderedNotification : UNotification { public PartRenderedNotification(UVoicePart part) { this.part = part; diff --git a/OpenUtau.Core/DiffSinger/DiffSingerVariance.cs b/OpenUtau.Core/DiffSinger/DiffSingerVariance.cs index 5f12b9802..a70d90005 100644 --- a/OpenUtau.Core/DiffSinger/DiffSingerVariance.cs +++ b/OpenUtau.Core/DiffSinger/DiffSingerVariance.cs @@ -24,6 +24,7 @@ public struct VarianceResult{ public int totalFrames; } public class DsVariance : IDisposable{ + const ulong AcceptedVarianceCacheSalt = 0x9E3779B97F4A7C15UL; string rootPath; DsConfig dsConfig; Dictionary languageIds = new Dictionary(); @@ -257,6 +258,22 @@ public VarianceResult Process(RenderPhrase phrase){ varianceInputs.Add(NamedOnnxValue.CreateFromTensor("spk_embed", spkEmbedTensor)); } Onnx.VerifyInputNames(varianceModel, varianceInputs); + var acceptedVarianceCache = Preferences.Default.DiffSingerTensorCache + ? new DiffSingerCache(varianceHash ^ AcceptedVarianceCacheSalt, AcceptedVarianceCacheInputs(varianceInputs)) + : null; + ulong pitchHash = DiffSingerVariancePatcher.HashPitch(pitch); + var acceptedVariance = acceptedVarianceCache == null + ? null + : LoadAcceptedVariance(acceptedVarianceCache, frameMs, headFrames, tailFrames, totalFrames); + if (acceptedVariance != null && acceptedVariance.PitchHash == pitchHash) { + phrase.AddCacheFile(acceptedVarianceCache?.Filename); + return acceptedVariance.Result; + } + var pitchPriorities = RenderContext.PreRenderPriorities + .Where(priority => + priority.editKind == PreRenderEditKind.Pitch && + RenderPriority.Overlaps(phrase.position, phrase.end, priority.startTick, priority.endTick)) + .ToArray(); var varianceCache = Preferences.Default.DiffSingerTensorCache ? new DiffSingerCache(varianceHash, varianceInputs) : null; @@ -290,7 +307,7 @@ public VarianceResult Process(RenderPhrase phrase){ .First() .AsTensor() : null; - return new VarianceResult{ + var fullResult = new VarianceResult{ energy = energy_pred?.ToArray(), breathiness = breathiness_pred?.ToArray(), voicing = voicing_pred?.ToArray(), @@ -300,6 +317,108 @@ public VarianceResult Process(RenderPhrase phrase){ tailFrames = tailFrames, totalFrames = totalFrames, }; + if (acceptedVarianceCache == null) { + return fullResult; + } + var result = fullResult; + if (acceptedVariance != null && pitchPriorities.Length > 0) { + result = DiffSingerVariancePatcher.Patch( + phrase, acceptedVariance.Result, fullResult, pitchPriorities); + } + SaveAcceptedVariance(acceptedVarianceCache, result, pitchHash); + phrase.AddCacheFile(acceptedVarianceCache.Filename); + return result; + } + + ICollection AcceptedVarianceCacheInputs(ICollection varianceInputs) { + return varianceInputs + .Where(input => input.Name != "pitch" && input.Name != "retake") + .ToArray(); + } + + class AcceptedVariance { + public VarianceResult Result; + public ulong PitchHash; + } + + AcceptedVariance? LoadAcceptedVariance( + DiffSingerCache cache, + float expectedFrameMs, + int expectedHeadFrames, + int expectedTailFrames, + int expectedTotalFrames) { + var outputs = cache.Load(); + if (outputs == null) { + return null; + } + try { + var frameMsValue = GetRequiredFloat(outputs, "frame_ms"); + var headFramesValue = (int)GetRequiredLong(outputs, "head_frames"); + var tailFramesValue = (int)GetRequiredLong(outputs, "tail_frames"); + var totalFramesValue = (int)GetRequiredLong(outputs, "total_frames"); + if (Math.Abs(frameMsValue - expectedFrameMs) > 0.0001f || + headFramesValue != expectedHeadFrames || + tailFramesValue != expectedTailFrames || + totalFramesValue != expectedTotalFrames) { + return null; + } + return new AcceptedVariance { + PitchHash = unchecked((ulong)GetRequiredLong(outputs, "pitch_hash")), + Result = new VarianceResult { + energy = GetOptionalFloatArray(outputs, "energy_pred"), + breathiness = GetOptionalFloatArray(outputs, "breathiness_pred"), + voicing = GetOptionalFloatArray(outputs, "voicing_pred"), + tension = GetOptionalFloatArray(outputs, "tension_pred"), + frameMs = frameMsValue, + headFrames = headFramesValue, + tailFrames = tailFramesValue, + totalFrames = totalFramesValue, + } + }; + } catch (Exception e) { + Log.Error(e, "Failed to load accepted variance cache."); + cache.Delete(); + return null; + } + } + + void SaveAcceptedVariance(DiffSingerCache cache, VarianceResult result, ulong pitchHash) { + var outputs = new List { + NamedOnnxValue.CreateFromTensor("pitch_hash", + new DenseTensor(new[] { unchecked((long)pitchHash) }, new[] { 1 })), + NamedOnnxValue.CreateFromTensor("frame_ms", + new DenseTensor(new[] { result.frameMs }, new[] { 1 })), + NamedOnnxValue.CreateFromTensor("head_frames", + new DenseTensor(new[] { (long)result.headFrames }, new[] { 1 })), + NamedOnnxValue.CreateFromTensor("tail_frames", + new DenseTensor(new[] { (long)result.tailFrames }, new[] { 1 })), + NamedOnnxValue.CreateFromTensor("total_frames", + new DenseTensor(new[] { (long)result.totalFrames }, new[] { 1 })), + }; + AddOptionalFloatArray(outputs, "energy_pred", result.energy); + AddOptionalFloatArray(outputs, "breathiness_pred", result.breathiness); + AddOptionalFloatArray(outputs, "voicing_pred", result.voicing); + AddOptionalFloatArray(outputs, "tension_pred", result.tension); + cache.Save(outputs); + } + + static void AddOptionalFloatArray(List outputs, string name, float[]? values) { + if (values != null) { + outputs.Add(NamedOnnxValue.CreateFromTensor( + name, new DenseTensor(values, new[] { values.Length }))); + } + } + + static float[]? GetOptionalFloatArray(ICollection outputs, string name) { + return outputs.FirstOrDefault(output => output.Name == name)?.AsTensor().ToArray(); + } + + static float GetRequiredFloat(ICollection outputs, string name) { + return outputs.First(output => output.Name == name).AsTensor().First(); + } + + static long GetRequiredLong(ICollection outputs, string name) { + return outputs.First(output => output.Name == name).AsTensor().First(); } private bool disposedValue; diff --git a/OpenUtau.Core/DiffSinger/DiffSingerVariancePatcher.cs b/OpenUtau.Core/DiffSinger/DiffSingerVariancePatcher.cs new file mode 100644 index 000000000..815708ef4 --- /dev/null +++ b/OpenUtau.Core/DiffSinger/DiffSingerVariancePatcher.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using K4os.Hash.xxHash; +using OpenUtau.Core.Render; + +namespace OpenUtau.Core.DiffSinger { + internal static class DiffSingerVariancePatcher { + const double InfluenceMarginMs = 250; + const double FadeMs = 100; + + internal static ulong HashPitch(float[] pitch) { + var bytes = new byte[pitch.Length * sizeof(float)]; + Buffer.BlockCopy(pitch, 0, bytes, 0, bytes.Length); + return XXH64.DigestOf(bytes); + } + + internal static bool CanPatch(VarianceResult oldResult, VarianceResult newResult) { + return oldResult.totalFrames == newResult.totalFrames && + oldResult.headFrames == newResult.headFrames && + oldResult.tailFrames == newResult.tailFrames && + Math.Abs(oldResult.frameMs - newResult.frameMs) < 0.0001f && + SameLength(oldResult.energy, newResult.energy) && + SameLength(oldResult.breathiness, newResult.breathiness) && + SameLength(oldResult.voicing, newResult.voicing) && + SameLength(oldResult.tension, newResult.tension); + } + + static bool SameLength(float[]? a, float[]? b) { + return (a == null && b == null) || (a != null && b != null && a.Length == b.Length); + } + + internal static VarianceResult Patch( + RenderPhrase phrase, + VarianceResult oldResult, + VarianceResult newResult, + IEnumerable priorities) { + if (!CanPatch(oldResult, newResult)) { + return newResult; + } + var weights = BuildWeights(phrase, newResult, priorities); + if (weights.All(w => w <= 0)) { + return oldResult; + } + return new VarianceResult { + energy = PatchArray(oldResult.energy, newResult.energy, weights), + breathiness = PatchArray(oldResult.breathiness, newResult.breathiness, weights), + voicing = PatchArray(oldResult.voicing, newResult.voicing, weights), + tension = PatchArray(oldResult.tension, newResult.tension, weights), + frameMs = newResult.frameMs, + headFrames = newResult.headFrames, + tailFrames = newResult.tailFrames, + totalFrames = newResult.totalFrames, + }; + } + + internal static float[] BuildWeights( + RenderPhrase phrase, + VarianceResult result, + IEnumerable priorities) { + double startMs = phrase.positionMs - result.headFrames * result.frameMs; + var ranges = priorities + .Where(priority => + priority.editKind == PreRenderEditKind.Pitch && + RenderPriority.Overlaps(phrase.position, phrase.end, priority.startTick, priority.endTick)) + .Select(priority => ( + startMs: phrase.timeAxis.TickPosToMsPos(priority.startTick), + endMs: phrase.timeAxis.TickPosToMsPos(priority.endTick))); + return BuildWeightsForMsRanges(result.totalFrames, result.frameMs, startMs, ranges); + } + + internal static float[] BuildWeightsForMsRanges( + int totalFrames, + float frameMs, + double startMs, + IEnumerable<(double startMs, double endMs)> ranges) { + var weights = new float[totalFrames]; + if (totalFrames <= 0) { + return weights; + } + int marginFrames = Math.Max(0, (int)Math.Round(InfluenceMarginMs / frameMs)); + int fadeFrames = Math.Max(1, (int)Math.Round(FadeMs / frameMs)); + foreach (var range in ranges) { + int coreStart = FrameIndex(range.startMs, startMs, frameMs, totalFrames); + int coreEnd = FrameIndex(range.endMs, startMs, frameMs, totalFrames); + if (coreEnd <= coreStart) { + coreEnd = Math.Min(totalFrames, coreStart + 1); + } + int replaceStart = Math.Max(0, coreStart - marginFrames); + int replaceEnd = Math.Min(totalFrames, coreEnd + marginFrames); + int fadeStart = Math.Max(0, replaceStart - fadeFrames); + int fadeEnd = Math.Min(totalFrames, replaceEnd + fadeFrames); + for (int i = fadeStart; i < fadeEnd; ++i) { + float weight; + if (i < replaceStart) { + weight = SmoothStep((float)(i - fadeStart + 1) / Math.Max(1, replaceStart - fadeStart + 1)); + } else if (i < replaceEnd) { + weight = 1; + } else { + weight = 1 - SmoothStep((float)(i - replaceEnd + 1) / Math.Max(1, fadeEnd - replaceEnd + 1)); + } + weights[i] = Math.Max(weights[i], weight); + } + } + return weights; + } + + static int FrameIndex(double ms, double startMs, float frameMs, int totalFrames) { + return Math.Clamp((int)Math.Round((ms - startMs) / frameMs), 0, totalFrames); + } + + static float SmoothStep(float x) { + x = Math.Clamp(x, 0, 1); + return x * x * (3 - 2 * x); + } + + internal static float[]? PatchArray(float[]? oldValues, float[]? newValues, float[] weights) { + if (oldValues == null || newValues == null || oldValues.Length != newValues.Length) { + return newValues; + } + var result = new float[newValues.Length]; + for (int i = 0; i < result.Length; ++i) { + float weight = weights[i]; + result[i] = oldValues[i] * (1 - weight) + newValues[i] * weight; + } + return result; + } + } +} diff --git a/OpenUtau.Core/DocManager.cs b/OpenUtau.Core/DocManager.cs index 16904f955..9666bd076 100644 --- a/OpenUtau.Core/DocManager.cs +++ b/OpenUtau.Core/DocManager.cs @@ -286,9 +286,9 @@ public void EndUndoGroup() { } PreRenderNotification CreatePreRenderNotification(IEnumerable commands) { - var priorityRanges = new Dictionary(); + var priorityRanges = new Dictionary(); foreach (var cmd in commands) { - if (!TryGetPreRenderRange(cmd, out var part, out var cmdStartTick, out var cmdEndTick) || + if (!TryGetPreRenderRange(cmd, out var part, out var cmdStartTick, out var cmdEndTick, out var editKind) || part == null || !Project.parts.Contains(part)) { continue; @@ -296,23 +296,52 @@ PreRenderNotification CreatePreRenderNotification(IEnumerable commands if (priorityRanges.TryGetValue(part, out var range)) { priorityRanges[part] = ( Math.Min(range.startTick, cmdStartTick), - Math.Max(range.endTick, cmdEndTick)); + Math.Max(range.endTick, cmdEndTick), + MergeEditKind(range.editKind, editKind)); } else { - priorityRanges[part] = (cmdStartTick, cmdEndTick); + priorityRanges[part] = (cmdStartTick, cmdEndTick, editKind); } } if (priorityRanges.Count > 0) { return new PreRenderNotification(priorityRanges.Select(range => - new PreRenderPriority(range.Key, range.Value.startTick, range.Value.endTick))); + new PreRenderPriority(range.Key, range.Value.startTick, range.Value.endTick, range.Value.editKind))); } return new PreRenderNotification(); } - bool TryGetPreRenderRange(UCommand cmd, out UVoicePart? part, out int startTick, out int endTick) { + PreRenderEditKind MergeEditKind(PreRenderEditKind a, PreRenderEditKind b) { + if (a == PreRenderEditKind.Pitch || b == PreRenderEditKind.Pitch) { + return PreRenderEditKind.Pitch; + } + if (a == PreRenderEditKind.VarianceCurve || b == PreRenderEditKind.VarianceCurve) { + return PreRenderEditKind.VarianceCurve; + } + return PreRenderEditKind.Generic; + } + + bool TryGetPreRenderRange( + UCommand cmd, + out UVoicePart? part, + out int startTick, + out int endTick, + out PreRenderEditKind editKind) { part = null; startTick = 0; endTick = 0; + editKind = PreRenderEditKind.Generic; switch (cmd) { + case PitchExpCommand pitchCommand when pitchCommand.Part != null: + part = pitchCommand.Part; + startTick = GetExpCommandStartTick(pitchCommand); + endTick = GetExpCommandEndTick(pitchCommand); + editKind = PreRenderEditKind.Pitch; + return endTick > startTick; + case VibratoCommand vibratoCommand when vibratoCommand.Notes.Length > 0: + part = vibratoCommand.Part; + startTick = part.position + vibratoCommand.Notes.Min(note => note.position); + endTick = part.position + vibratoCommand.Notes.Max(note => note.End); + editKind = PreRenderEditKind.Pitch; + return endTick > startTick; case NoteCommand noteCommand when noteCommand.Notes.Length > 0: part = noteCommand.Part; startTick = part.position + noteCommand.Notes.Min(note => note.position); @@ -322,16 +351,31 @@ bool TryGetPreRenderRange(UCommand cmd, out UVoicePart? part, out int startTick, part = curveCommand.Part; startTick = part.position + curveCommand.StartTick; endTick = part.position + curveCommand.EndTick; + editKind = GetCurveEditKind(curveCommand.Abbr); return endTick > startTick; case MergedSetCurveCommand curveCommand: part = curveCommand.Part; startTick = part.position + curveCommand.StartTick; endTick = part.position + curveCommand.EndTick; + editKind = curveCommand.SetReal ? PreRenderEditKind.Generic : GetCurveEditKind(curveCommand.Abbr); + return endTick > startTick; + case PasteCurveCommand curveCommand: + part = curveCommand.Part; + startTick = part.position + curveCommand.StartTick; + endTick = part.position + curveCommand.EndTick; + editKind = GetCurveEditKind(curveCommand.Abbr); + return endTick > startTick; + case ClearCurveCommand curveCommand: + part = curveCommand.Part; + startTick = part.position + curveCommand.StartTick; + endTick = part.position + curveCommand.EndTick; + editKind = GetCurveEditKind(curveCommand.Abbr); return endTick > startTick; case ExpCommand expCommand when expCommand.Note != null: part = expCommand.Part; startTick = part.position + expCommand.Note.position; endTick = part.position + expCommand.Note.End; + editKind = GetExpressionEditKind(expCommand.Key); return endTick > startTick; case ExpCommand expCommand: part = expCommand.Part; @@ -354,6 +398,42 @@ bool TryGetPreRenderRange(UCommand cmd, out UVoicePart? part, out int startTick, } } + int GetExpCommandStartTick(ExpCommand command) { + if (command.Note != null) { + return command.Part.position + command.Note.position; + } + return command.Part.position; + } + + int GetExpCommandEndTick(ExpCommand command) { + if (command.Note != null) { + return command.Part.position + command.Note.End; + } + return command.Part.End; + } + + PreRenderEditKind GetExpressionEditKind(string? abbr) { + if (abbr == Format.Ustx.SHFT) { + return PreRenderEditKind.Pitch; + } + return PreRenderEditKind.Generic; + } + + PreRenderEditKind GetCurveEditKind(string? abbr) { + switch (abbr) { + case Format.Ustx.PITD: + case Format.Ustx.SHFC: + return PreRenderEditKind.Pitch; + case OpenUtau.Core.DiffSinger.DiffSingerUtils.ENE: + case Format.Ustx.BREC: + case Format.Ustx.VOIC: + case Format.Ustx.TENC: + return PreRenderEditKind.VarianceCurve; + default: + return PreRenderEditKind.Generic; + } + } + public void RollBackUndoGroup() { if (undoGroup == null) { Log.Error("No active undoGroup to rollback."); diff --git a/OpenUtau.Core/Render/RealCurveRefresh.cs b/OpenUtau.Core/Render/RealCurveRefresh.cs new file mode 100644 index 000000000..a5813dca8 --- /dev/null +++ b/OpenUtau.Core/Render/RealCurveRefresh.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using OpenUtau.Core.DiffSinger; +using OpenUtau.Core.Ustx; +using OpenUtau.Core.Util; + +namespace OpenUtau.Core.Render { + public static class RealCurveRefresh { + public static bool CanRefresh(RenderPhrase phrase, bool allowSessionInitialization = true) { + if (phrase.renderer.SingerType != USingerType.DiffSinger || + !phrase.renderer.SupportsRealCurve || + !Preferences.Default.DiffSingerTensorCache) { + return false; + } + return true; + } + + public static List LoadRenderedRealCurves( + RenderPhrase phrase, + IEnumerable? priorities = null, + bool allowSessionInitialization = true) { + if (!CanRefresh(phrase, allowSessionInitialization)) { + return new List(0); + } + using var context = RenderContext.WithPreRenderPriorities( + priorities ?? Array.Empty()); + return phrase.renderer.LoadRenderedRealCurves(phrase); + } + + public static bool ApplyRealCurveResults( + UProject project, + UVoicePart part, + RenderPhrase phrase, + IEnumerable results) { + bool changed = false; + foreach (var result in results) { + changed |= ApplyRealCurveResult(project, part, phrase, result); + } + return changed; + } + + public static bool ApplyRealCurveResult( + UProject project, + UVoicePart part, + RenderPhrase phrase, + RenderRealCurveResult result) { + int count = Math.Min(result.ticks.Length, result.values.Length); + if (count == 0) { + return false; + } + var curve = part.curves.FirstOrDefault(curve => curve.abbr == result.abbr); + if (curve == null) { + if (!project.expressions.TryGetValue(result.abbr, out var descriptor)) { + return false; + } + curve = new UCurve(descriptor); + part.curves.Add(curve); + } + var ticks = result.ticks + .Take(count) + .Select(tick => phrase.position - part.position + (int)Math.Round(tick)) + .ToArray(); + var values = result.values + .Take(count) + .Select(value => (int)Math.Round(value * 1000.0)) + .ToArray(); + int rangeStart = ticks.First(); + int rangeEnd = ticks.Last(); + int index = curve.realXs.BinarySearch(rangeStart); + if (index < 0) { + index = ~index; + } + int removeCount = 0; + while (index + removeCount < curve.realXs.Count && curve.realXs[index + removeCount] <= rangeEnd) { + removeCount++; + } + if (removeCount > 0) { + curve.realXs.RemoveRange(index, removeCount); + curve.realYs.RemoveRange(index, removeCount); + } + var insertXs = new List(ticks.Length + 1) { rangeStart }; + var insertYs = new List(values.Length + 1) { -1 }; + insertXs.AddRange(ticks); + insertYs.AddRange(values); + curve.realXs.InsertRange(index, insertXs); + curve.realYs.InsertRange(index, insertYs); + return true; + } + } +} diff --git a/OpenUtau.Core/Render/RenderContext.cs b/OpenUtau.Core/Render/RenderContext.cs new file mode 100644 index 000000000..245937bc2 --- /dev/null +++ b/OpenUtau.Core/Render/RenderContext.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +namespace OpenUtau.Core.Render { + internal static class RenderContext { + static readonly AsyncLocal preRenderPriorities = new AsyncLocal(); + + internal static PreRenderPriority[] PreRenderPriorities => + preRenderPriorities.Value ?? Array.Empty(); + + internal static IDisposable WithPreRenderPriorities(IEnumerable priorities) { + var previous = preRenderPriorities.Value; + preRenderPriorities.Value = priorities.ToArray(); + return new Scope(() => preRenderPriorities.Value = previous); + } + + class Scope : IDisposable { + readonly Action dispose; + bool disposed; + + public Scope(Action dispose) { + this.dispose = dispose; + } + + public void Dispose() { + if (!disposed) { + dispose(); + disposed = true; + } + } + } + } +} diff --git a/OpenUtau.Core/Render/RenderEngine.cs b/OpenUtau.Core/Render/RenderEngine.cs index 0af6b8f5e..7a8705f1a 100644 --- a/OpenUtau.Core/Render/RenderEngine.cs +++ b/OpenUtau.Core/Render/RenderEngine.cs @@ -258,6 +258,8 @@ private void RenderRequests( var phrase = tuple.phrase; var source = tuple.source; var request = tuple.request; + using var context = RenderContext.WithPreRenderPriorities( + IsDiffSinger(phrase) ? GetDiffSingerPriorityRanges(request.part) : Array.Empty()); var task = phrase.renderer.Render(phrase, progress, request.trackNo, cancellation, true); task.Wait(); if (cancellation.IsCancellationRequested) { @@ -266,6 +268,7 @@ private void RenderRequests( source.SetSamples(task.Result.samples); bool partReady = request.sources.All(s => s.HasSamples); if (IsDiffSinger(phrase)) { + RefreshRealCurves(phrase, request.part); request.part.SetMix(request.mix); DocManager.Inst.ExecuteCmd(new PhraseRenderedNotification(request.part)); } @@ -351,6 +354,33 @@ private bool IsDiffSinger(RenderPhrase phrase) { return phrase.renderer.SingerType == USingerType.DiffSinger; } + private void RefreshRealCurves(RenderPhrase phrase, UVoicePart part) { + if (!RealCurveRefresh.CanRefresh(phrase)) { + return; + } + List results; + try { + results = RealCurveRefresh.LoadRenderedRealCurves(phrase, GetDiffSingerPriorityRanges(part)); + } catch (Exception e) { + Log.Debug(e, "Failed to refresh DiffSinger real curves."); + return; + } + if (results.Count == 0) { + return; + } + void Apply() { + lock (part) { + RealCurveRefresh.ApplyRealCurveResults(project, part, phrase, results); + } + DocManager.Inst.ExecuteCmd(new RealCurvesRenderedNotification(part)); + } + if (DocManager.Inst.PostOnUIThread != null) { + DocManager.Inst.PostOnUIThread(Apply); + } else { + Apply(); + } + } + private PreRenderPriority[] GetDiffSingerPriorityRanges() { return priorityRanges .Where(priority => @@ -360,6 +390,12 @@ private PreRenderPriority[] GetDiffSingerPriorityRanges() { .ToArray(); } + private PreRenderPriority[] GetDiffSingerPriorityRanges(UVoicePart part) { + return GetDiffSingerPriorityRanges() + .Where(priority => ReferenceEquals(priority.part, part)) + .ToArray(); + } + public static void ReleaseSourceTemp() { VoicebankFiles.Inst.ReleaseSourceTemp(); } diff --git a/OpenUtau.Test/Core/DiffSinger/DiffSingerVariancePatcherTest.cs b/OpenUtau.Test/Core/DiffSinger/DiffSingerVariancePatcherTest.cs new file mode 100644 index 000000000..8b476d668 --- /dev/null +++ b/OpenUtau.Test/Core/DiffSinger/DiffSingerVariancePatcherTest.cs @@ -0,0 +1,49 @@ +using System.Linq; +using Xunit; + +namespace OpenUtau.Core.DiffSinger { + public class DiffSingerVariancePatcherTest { + [Fact] + public void HashPitchChangesWhenPitchChanges() { + var a = DiffSingerVariancePatcher.HashPitch(new[] { 1f, 2f, 3f }); + var b = DiffSingerVariancePatcher.HashPitch(new[] { 1f, 2.1f, 3f }); + + Assert.NotEqual(a, b); + } + + [Fact] + public void BuildWeightsForMsRangesKeepsOutsideRangeAtZero() { + var weights = DiffSingerVariancePatcher.BuildWeightsForMsRanges( + totalFrames: 100, + frameMs: 10, + startMs: 0, + ranges: new[] { (startMs: 500.0, endMs: 600.0) }); + + Assert.Equal(0, weights[0]); + Assert.Equal(0, weights[99]); + Assert.Contains(weights, weight => weight >= 1); + } + + [Fact] + public void BuildWeightsForMsRangesCrossfadesEdges() { + var weights = DiffSingerVariancePatcher.BuildWeightsForMsRanges( + totalFrames: 100, + frameMs: 10, + startMs: 0, + ranges: new[] { (startMs: 500.0, endMs: 600.0) }); + + Assert.Contains(weights, weight => weight > 0 && weight < 1); + } + + [Fact] + public void PatchArrayBlendsOldAndNewByWeight() { + var oldValues = Enumerable.Repeat(0f, 5).ToArray(); + var newValues = Enumerable.Repeat(10f, 5).ToArray(); + var weights = new[] { 0f, 0.25f, 0.5f, 0.75f, 1f }; + + var result = DiffSingerVariancePatcher.PatchArray(oldValues, newValues, weights)!; + + Assert.Equal(new[] { 0f, 2.5f, 5f, 7.5f, 10f }, result); + } + } +} diff --git a/OpenUtau/ViewModels/NotesViewModel.cs b/OpenUtau/ViewModels/NotesViewModel.cs index 63f7a4646..65c494fb5 100644 --- a/OpenUtau/ViewModels/NotesViewModel.cs +++ b/OpenUtau/ViewModels/NotesViewModel.cs @@ -5,6 +5,7 @@ using System.Numerics; using System.Reactive; using System.Reactive.Linq; +using System.Threading; using System.Threading.Tasks; using Avalonia; using Avalonia.Controls; @@ -14,6 +15,7 @@ using DynamicData.Binding; using OpenUtau.App.Views; using OpenUtau.Core; +using OpenUtau.Core.Render; using OpenUtau.Core.Ustx; using OpenUtau.Core.Util; using OpenUtau.ViewModels; @@ -104,6 +106,7 @@ public class NotesViewModel : ViewModelBase, ICmdSubscriber { // See the comments on TracksViewModel.playPosXToTickOffset private double playPosXToTickOffset => Bounds.Width != 0 ? ViewportTicks / Bounds.Width : 0; + private int realCurveRefreshVersion; private readonly ObservableAsPropertyHelper viewportTicks; private readonly ObservableAsPropertyHelper viewportTracks; @@ -1101,6 +1104,10 @@ public void OnNext(UCommand cmd, bool isUndo) { } else if (cmd is PhonemizedNotification) { OnPartModified(); MessageBus.Current.SendMessage(new NotesRefreshEvent()); + } else if (notif is PreRenderNotification preRender) { + ScheduleDiffSingerRealCurveRefresh(preRender.priorities); + } else if (notif is RealCurvesRenderedNotification && notif.part == Part) { + MessageBus.Current.SendMessage(new NotesRefreshEvent()); } else if ((notif is PhraseRenderedNotification || notif is PartRenderedNotification) && notif.part == Part) { MessageBus.Current.SendMessage(new WaveformRefreshEvent()); } @@ -1165,6 +1172,87 @@ public void OnNext(UCommand cmd, bool isUndo) { } } + private void ScheduleDiffSingerRealCurveRefresh(IEnumerable priorities) { + if (Part == null) { + return; + } + var part = Part; + var project = Project; + var relevantPriorities = priorities + .Where(priority => + ReferenceEquals(priority.part, part) && + (priority.editKind == PreRenderEditKind.Pitch || + priority.editKind == PreRenderEditKind.VarianceCurve)) + .ToArray(); + if (relevantPriorities.Length == 0) { + return; + } + var phrases = part.renderPhrases + .Where(phrase => + RealCurveRefresh.CanRefresh(phrase, allowSessionInitialization: false) && + relevantPriorities.Any(priority => + phrase.end > priority.startTick && phrase.position < priority.endTick)) + .ToArray(); + if (phrases.Length == 0) { + return; + } + int version = Interlocked.Increment(ref realCurveRefreshVersion); + _ = Task.Run(() => RefreshDiffSingerRealCurves(project, part, phrases, relevantPriorities, version)); + } + + private void RefreshDiffSingerRealCurves( + UProject project, + UVoicePart part, + RenderPhrase[] phrases, + PreRenderPriority[] priorities, + int version) { + try { + var phraseResults = new List<(RenderPhrase phrase, List results)>(); + foreach (var phrase in phrases) { + if (Volatile.Read(ref realCurveRefreshVersion) != version) { + return; + } + try { + var results = RealCurveRefresh.LoadRenderedRealCurves( + phrase, priorities, allowSessionInitialization: false); + if (results.Count > 0) { + phraseResults.Add((phrase, results)); + } + } catch (Exception e) { + Log.Debug(e, "Failed to refresh DiffSinger real curves."); + } + } + if (phraseResults.Count == 0 || + Volatile.Read(ref realCurveRefreshVersion) != version) { + return; + } + void Apply() { + if (Volatile.Read(ref realCurveRefreshVersion) != version || + !ReferenceEquals(Project, project) || + !ReferenceEquals(Part, part)) { + return; + } + bool changed = false; + lock (part) { + foreach (var phraseResult in phraseResults) { + changed |= RealCurveRefresh.ApplyRealCurveResults( + project, part, phraseResult.phrase, phraseResult.results); + } + } + if (changed) { + MessageBus.Current.SendMessage(new NotesRefreshEvent()); + } + } + if (DocManager.Inst.PostOnUIThread != null) { + DocManager.Inst.PostOnUIThread(Apply); + } else { + Apply(); + } + } catch (Exception e) { + Log.Debug(e, "Failed to refresh DiffSinger real curves."); + } + } + private void MaybeAutoScroll(double positionX) { var autoScrollPreference = Convert.ToBoolean(Preferences.Default.PlaybackAutoScroll); if (autoScrollPreference) { From 14b74507b958e1aa69687c08e3d1af368634a87b Mon Sep 17 00:00:00 2001 From: KakaruHayate Date: Tue, 2 Jun 2026 17:50:50 +0800 Subject: [PATCH 3/4] Fix DiffSinger variance pitch edit ranges --- OpenUtau.Core/Commands/ExpCommands.cs | 20 +++++++++ OpenUtau.Core/DocManager.cs | 10 ++++- OpenUtau.Core/Editing/NoteBatchEdits.cs | 2 +- .../Core/Commands/PitchCommandTest.cs | 41 +++++++++++++++++++ .../ViewModels/NotePropertiesViewModel.cs | 4 +- OpenUtau/ViewModels/PianoRollViewModel.cs | 10 ++--- OpenUtau/Views/NoteEditStates.cs | 2 +- 7 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 OpenUtau.Test/Core/Commands/PitchCommandTest.cs diff --git a/OpenUtau.Core/Commands/ExpCommands.cs b/OpenUtau.Core/Commands/ExpCommands.cs index f17aa4508..667ddd150 100644 --- a/OpenUtau.Core/Commands/ExpCommands.cs +++ b/OpenUtau.Core/Commands/ExpCommands.cs @@ -140,6 +140,8 @@ public override void Unexecute() { public abstract class PitchExpCommand : ExpCommand { public PitchExpCommand(UVoicePart part) : base(part) { } + public virtual IEnumerable AffectedNotes => + Note == null ? Enumerable.Empty() : new[] { Note }; public override ValidateOptions ValidateOptions => new ValidateOptions { SkipTiming = true, Part = Part, @@ -162,14 +164,22 @@ public DeletePitchPointCommand(UVoicePart part, UNote note, int index) : base(pa } public class ChangePitchPointShapeCommand : PitchExpCommand { + UNote[] notes; public PitchPoint Point; public PitchPointShape NewShape; public PitchPointShape OldShape; public ChangePitchPointShapeCommand(UVoicePart part, PitchPoint point, PitchPointShape shape) : base(part) { + notes = Array.Empty(); this.Point = point; this.NewShape = shape; this.OldShape = point.shape; } + public ChangePitchPointShapeCommand(UVoicePart part, UNote note, PitchPoint point, PitchPointShape shape) + : this(part, point, shape) { + Note = note; + notes = new[] { note }; + } + public override IEnumerable AffectedNotes => notes; public override string ToString() { return "Change pitch point shape"; } public override void Execute() { Point.shape = NewShape; } public override void Unexecute() { Point.shape = OldShape; } @@ -188,6 +198,7 @@ public SetPitchPointShapeCommand(UVoicePart part, IEnumerable notes, Pitc .ToArray()) .ToArray(); } + public override IEnumerable AffectedNotes => Notes; public override string ToString() { return "Change pitch point shape"; } public override void Execute() { foreach (var note in Notes) { @@ -245,14 +256,22 @@ public AddPitchPointCommand(UVoicePart part, UNote note, PitchPoint point, int i } public class MovePitchPointCommand : PitchExpCommand { + UNote[] notes; readonly PitchPoint point; readonly float deltaX; readonly float deltaY; public MovePitchPointCommand(UVoicePart part, PitchPoint point, float deltaX, float deltaY) : base(part) { + notes = Array.Empty(); this.point = point; this.deltaX = deltaX; this.deltaY = deltaY; } + public MovePitchPointCommand(UVoicePart part, UNote note, PitchPoint point, float deltaX, float deltaY) + : this(part, point, deltaX, deltaY) { + Note = note; + notes = new[] { note }; + } + public override IEnumerable AffectedNotes => notes; public override string ToString() { return "Move pitch point"; } public override void Execute() { point.X += deltaX; point.Y += deltaY; } public override void Unexecute() { point.X -= deltaX; point.Y -= deltaY; } @@ -291,6 +310,7 @@ public SetPitchPointsCommand(UVoicePart part, IEnumerable notes, UPitch p oldPitch = Notes.Select(note => note.pitch).ToArray(); newPitch = pitch; } + public override IEnumerable AffectedNotes => Notes; public override string ToString() => "Set pitch points"; public override void Execute(){ lock (Part) { diff --git a/OpenUtau.Core/DocManager.cs b/OpenUtau.Core/DocManager.cs index 9666bd076..75365c3dd 100644 --- a/OpenUtau.Core/DocManager.cs +++ b/OpenUtau.Core/DocManager.cs @@ -332,8 +332,14 @@ bool TryGetPreRenderRange( switch (cmd) { case PitchExpCommand pitchCommand when pitchCommand.Part != null: part = pitchCommand.Part; - startTick = GetExpCommandStartTick(pitchCommand); - endTick = GetExpCommandEndTick(pitchCommand); + var pitchNotes = pitchCommand.AffectedNotes.ToArray(); + if (pitchNotes.Length > 0) { + startTick = part.position + pitchNotes.Min(note => note.position); + endTick = part.position + pitchNotes.Max(note => note.End); + } else { + startTick = part.position; + endTick = part.End; + } editKind = PreRenderEditKind.Pitch; return endTick > startTick; case VibratoCommand vibratoCommand when vibratoCommand.Notes.Length > 0: diff --git a/OpenUtau.Core/Editing/NoteBatchEdits.cs b/OpenUtau.Core/Editing/NoteBatchEdits.cs index 817315430..3d2985eae 100644 --- a/OpenUtau.Core/Editing/NoteBatchEdits.cs +++ b/OpenUtau.Core/Editing/NoteBatchEdits.cs @@ -750,7 +750,7 @@ public void Run(UProject project, UVoicePart part, List selectedNotes, Do docManager.ExecuteCmd(new DeletePitchPointCommand(part, note, index)); docManager.ExecuteCmd(new DeletePitchPointCommand(part, note, index)); var lastPitch = note.pitch.data[^1]; - docManager.ExecuteCmd(new MovePitchPointCommand(part, lastPitch, 0, -lastPitch.Y)); + docManager.ExecuteCmd(new MovePitchPointCommand(part, note, lastPitch, 0, -lastPitch.Y)); } } diff --git a/OpenUtau.Test/Core/Commands/PitchCommandTest.cs b/OpenUtau.Test/Core/Commands/PitchCommandTest.cs new file mode 100644 index 000000000..188f4ba0c --- /dev/null +++ b/OpenUtau.Test/Core/Commands/PitchCommandTest.cs @@ -0,0 +1,41 @@ +using System.Linq; +using OpenUtau.Core.Ustx; +using Xunit; + +namespace OpenUtau.Core { + public class PitchCommandTest { + [Fact] + public void MovePitchPointCommandReportsAffectedNote() { + var part = new UVoicePart(); + var note = new UNote { position = 240, duration = 480 }; + var point = new PitchPoint(0, 0, PitchPointShape.io); + + var command = new MovePitchPointCommand(part, note, point, 10, 20); + + Assert.Single(command.AffectedNotes); + Assert.Same(note, command.AffectedNotes.First()); + } + + [Fact] + public void ChangePitchPointShapeCommandReportsAffectedNote() { + var part = new UVoicePart(); + var note = new UNote { position = 240, duration = 480 }; + var point = new PitchPoint(0, 0, PitchPointShape.io); + + var command = new ChangePitchPointShapeCommand(part, note, point, PitchPointShape.l); + + Assert.Single(command.AffectedNotes); + Assert.Same(note, command.AffectedNotes.First()); + } + + [Fact] + public void LegacyPitchPointMoveConstructorHasNoAffectedNote() { + var part = new UVoicePart(); + var point = new PitchPoint(0, 0, PitchPointShape.io); + + var command = new MovePitchPointCommand(part, point, 10, 20); + + Assert.Empty(command.AffectedNotes); + } + } +} diff --git a/OpenUtau/ViewModels/NotePropertiesViewModel.cs b/OpenUtau/ViewModels/NotePropertiesViewModel.cs index fc8749bc4..ec658e416 100644 --- a/OpenUtau/ViewModels/NotePropertiesViewModel.cs +++ b/OpenUtau/ViewModels/NotePropertiesViewModel.cs @@ -532,7 +532,7 @@ public void SetNoteParams(string tag, object? obj) { var newX = firstPoint.X + (pitchPoint.X - firstPoint.X) * scale; var deltaX = newX - pitchPoint.X; if (deltaX != 0) { - DocManager.Inst.ExecuteCmd(new MovePitchPointCommand(Part, pitchPoint, deltaX, 0)); + DocManager.Inst.ExecuteCmd(new MovePitchPointCommand(Part, note, pitchPoint, deltaX, 0)); } } } @@ -544,7 +544,7 @@ public void SetNoteParams(string tag, object? obj) { if (deltaX != 0) { foreach (var note in selectedNotes) { foreach (var pitchPoint in note.pitch.data) { - DocManager.Inst.ExecuteCmd(new MovePitchPointCommand(Part, pitchPoint, deltaX, 0)); + DocManager.Inst.ExecuteCmd(new MovePitchPointCommand(Part, note, pitchPoint, deltaX, 0)); } } } diff --git a/OpenUtau/ViewModels/PianoRollViewModel.cs b/OpenUtau/ViewModels/PianoRollViewModel.cs index d0ce85699..f7fa0962e 100644 --- a/OpenUtau/ViewModels/PianoRollViewModel.cs +++ b/OpenUtau/ViewModels/PianoRollViewModel.cs @@ -128,31 +128,31 @@ public PianoRollViewModel() { PitEaseInOutCommand = ReactiveCommand.Create(info => { if (NotesViewModel.Part == null) { return; } DocManager.Inst.StartUndoGroup("command.pitch.editpoint"); - DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note.pitch.data[info.Index], PitchPointShape.io)); + DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note, info.Note.pitch.data[info.Index], PitchPointShape.io)); DocManager.Inst.EndUndoGroup(); }); PitLinearCommand = ReactiveCommand.Create(info => { if (NotesViewModel.Part == null) { return; } DocManager.Inst.StartUndoGroup("command.pitch.editpoint"); - DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note.pitch.data[info.Index], PitchPointShape.l)); + DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note, info.Note.pitch.data[info.Index], PitchPointShape.l)); DocManager.Inst.EndUndoGroup(); }); PitEaseInCommand = ReactiveCommand.Create(info => { if (NotesViewModel.Part == null) { return; } DocManager.Inst.StartUndoGroup("command.pitch.editpoint"); - DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note.pitch.data[info.Index], PitchPointShape.i)); + DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note, info.Note.pitch.data[info.Index], PitchPointShape.i)); DocManager.Inst.EndUndoGroup(); }); PitEaseOutCommand = ReactiveCommand.Create(info => { if (NotesViewModel.Part == null) { return; } DocManager.Inst.StartUndoGroup("command.pitch.editpoint"); - DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note.pitch.data[info.Index], PitchPointShape.o)); + DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note, info.Note.pitch.data[info.Index], PitchPointShape.o)); DocManager.Inst.EndUndoGroup(); }); PitSplineCommand = ReactiveCommand.Create(info => { if (NotesViewModel.Part == null) { return; } DocManager.Inst.StartUndoGroup("command.pitch.editpoint"); - DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note.pitch.data[info.Index], PitchPointShape.sp)); + DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note, info.Note.pitch.data[info.Index], PitchPointShape.sp)); DocManager.Inst.EndUndoGroup(); }); PitSnapCommand = ReactiveCommand.Create(info => { diff --git a/OpenUtau/Views/NoteEditStates.cs b/OpenUtau/Views/NoteEditStates.cs index 0752dcdb6..f10372227 100644 --- a/OpenUtau/Views/NoteEditStates.cs +++ b/OpenUtau/Views/NoteEditStates.cs @@ -643,7 +643,7 @@ public override void Update(IPointer pointer, Point point) { return; } if (notesVm.Part != null) { - DocManager.Inst.ExecuteCmd(new MovePitchPointCommand(notesVm.Part, pitchPoint, (float)deltaX, (float)deltaY)); + DocManager.Inst.ExecuteCmd(new MovePitchPointCommand(notesVm.Part, note, pitchPoint, (float)deltaX, (float)deltaY)); } valueTip.UpdateValueTip($"{pitchPoint.X:0.0}ms, {pitchPoint.Y * 10:0}cent"); } From aa6434623e688e52e543594027178a9e8289bb52 Mon Sep 17 00:00:00 2001 From: KakaruHayate Date: Tue, 2 Jun 2026 18:25:13 +0800 Subject: [PATCH 4/4] Patch DiffSinger variance from pitch cache changes --- OpenUtau.Core/Commands/ExpCommands.cs | 29 ------ OpenUtau.Core/Commands/Notifications.cs | 14 +-- .../DiffSinger/DiffSingerVariance.cs | 18 ++-- .../DiffSinger/DiffSingerVariancePatcher.cs | 83 +++++++++++----- OpenUtau.Core/DocManager.cs | 98 ++----------------- OpenUtau.Core/Editing/NoteBatchEdits.cs | 2 +- OpenUtau.Core/Render/RealCurveRefresh.cs | 3 - OpenUtau.Core/Render/RenderContext.cs | 35 ------- OpenUtau.Core/Render/RenderEngine.cs | 4 +- .../Core/Commands/PitchCommandTest.cs | 41 -------- .../DiffSingerVariancePatcherTest.cs | 38 +++++++ .../ViewModels/NotePropertiesViewModel.cs | 4 +- OpenUtau/ViewModels/NotesViewModel.cs | 87 ---------------- OpenUtau/ViewModels/PianoRollViewModel.cs | 10 +- OpenUtau/Views/NoteEditStates.cs | 2 +- 15 files changed, 123 insertions(+), 345 deletions(-) delete mode 100644 OpenUtau.Core/Render/RenderContext.cs delete mode 100644 OpenUtau.Test/Core/Commands/PitchCommandTest.cs diff --git a/OpenUtau.Core/Commands/ExpCommands.cs b/OpenUtau.Core/Commands/ExpCommands.cs index 667ddd150..e338b733a 100644 --- a/OpenUtau.Core/Commands/ExpCommands.cs +++ b/OpenUtau.Core/Commands/ExpCommands.cs @@ -140,8 +140,6 @@ public override void Unexecute() { public abstract class PitchExpCommand : ExpCommand { public PitchExpCommand(UVoicePart part) : base(part) { } - public virtual IEnumerable AffectedNotes => - Note == null ? Enumerable.Empty() : new[] { Note }; public override ValidateOptions ValidateOptions => new ValidateOptions { SkipTiming = true, Part = Part, @@ -164,22 +162,14 @@ public DeletePitchPointCommand(UVoicePart part, UNote note, int index) : base(pa } public class ChangePitchPointShapeCommand : PitchExpCommand { - UNote[] notes; public PitchPoint Point; public PitchPointShape NewShape; public PitchPointShape OldShape; public ChangePitchPointShapeCommand(UVoicePart part, PitchPoint point, PitchPointShape shape) : base(part) { - notes = Array.Empty(); this.Point = point; this.NewShape = shape; this.OldShape = point.shape; } - public ChangePitchPointShapeCommand(UVoicePart part, UNote note, PitchPoint point, PitchPointShape shape) - : this(part, point, shape) { - Note = note; - notes = new[] { note }; - } - public override IEnumerable AffectedNotes => notes; public override string ToString() { return "Change pitch point shape"; } public override void Execute() { Point.shape = NewShape; } public override void Unexecute() { Point.shape = OldShape; } @@ -198,7 +188,6 @@ public SetPitchPointShapeCommand(UVoicePart part, IEnumerable notes, Pitc .ToArray()) .ToArray(); } - public override IEnumerable AffectedNotes => Notes; public override string ToString() { return "Change pitch point shape"; } public override void Execute() { foreach (var note in Notes) { @@ -256,22 +245,14 @@ public AddPitchPointCommand(UVoicePart part, UNote note, PitchPoint point, int i } public class MovePitchPointCommand : PitchExpCommand { - UNote[] notes; readonly PitchPoint point; readonly float deltaX; readonly float deltaY; public MovePitchPointCommand(UVoicePart part, PitchPoint point, float deltaX, float deltaY) : base(part) { - notes = Array.Empty(); this.point = point; this.deltaX = deltaX; this.deltaY = deltaY; } - public MovePitchPointCommand(UVoicePart part, UNote note, PitchPoint point, float deltaX, float deltaY) - : this(part, point, deltaX, deltaY) { - Note = note; - notes = new[] { note }; - } - public override IEnumerable AffectedNotes => notes; public override string ToString() { return "Move pitch point"; } public override void Execute() { point.X += deltaX; point.Y += deltaY; } public override void Unexecute() { point.X -= deltaX; point.Y -= deltaY; } @@ -310,7 +291,6 @@ public SetPitchPointsCommand(UVoicePart part, IEnumerable notes, UPitch p oldPitch = Notes.Select(note => note.pitch).ToArray(); newPitch = pitch; } - public override IEnumerable AffectedNotes => Notes; public override string ToString() => "Set pitch points"; public override void Execute(){ lock (Part) { @@ -337,7 +317,6 @@ public class SetCurveCommand : ExpCommand { readonly int lastY; int[] oldXs; int[] oldYs; - public string Abbr => abbr; public int StartTick => Math.Min(x, lastX); public int EndTick => Math.Max(x, lastX) + 1; public override ValidateOptions ValidateOptions @@ -407,8 +386,6 @@ public class MergedSetCurveCommand : ExpCommand { readonly int[] newXs; readonly int[] newYs; readonly bool setReal; - public string Abbr => abbr; - public bool SetReal => setReal; public int StartTick => (oldXs ?? Array.Empty()) .Concat(newXs ?? Array.Empty()) .DefaultIfEmpty(0) @@ -469,9 +446,6 @@ public class PasteCurveCommand : ExpCommand { readonly int[] ys; int[]? oldXs; int[]? oldYs; - public string Abbr => abbr; - public int StartTick => xs.DefaultIfEmpty(0).Min(); - public int EndTick => xs.DefaultIfEmpty(Part.Duration).Max() + 1; public PasteCurveCommand(UProject project, UVoicePart part, string abbr, IEnumerable xs, IEnumerable ys) : base(part) { this.project = project; this.abbr = abbr; @@ -533,9 +507,6 @@ public class ClearCurveCommand : ExpCommand { readonly string abbr; readonly int[] oldXs; readonly int[] oldYs; - public string Abbr => abbr; - public int StartTick => 0; - public int EndTick => Part.Duration; public ClearCurveCommand(UVoicePart part, string abbr) : base(part) { this.abbr = abbr; var curve = Part.curves.FirstOrDefault(curve => curve.abbr == abbr); diff --git a/OpenUtau.Core/Commands/Notifications.cs b/OpenUtau.Core/Commands/Notifications.cs index a2f416193..9df5db9c1 100644 --- a/OpenUtau.Core/Commands/Notifications.cs +++ b/OpenUtau.Core/Commands/Notifications.cs @@ -222,27 +222,15 @@ public FocusNoteNotification(UPart part, UNote note) { public override string ToString() => $"Focus note {note.lyric} at {note.position}."; } - public enum PreRenderEditKind { - Generic, - Pitch, - VarianceCurve, - } - public class PreRenderPriority { public readonly UVoicePart part; public readonly int startTick; public readonly int endTick; - public readonly PreRenderEditKind editKind; - public PreRenderPriority( - UVoicePart part, - int startTick, - int endTick, - PreRenderEditKind editKind = PreRenderEditKind.Generic) { + public PreRenderPriority(UVoicePart part, int startTick, int endTick) { this.part = part; this.startTick = startTick; this.endTick = endTick; - this.editKind = editKind; } } diff --git a/OpenUtau.Core/DiffSinger/DiffSingerVariance.cs b/OpenUtau.Core/DiffSinger/DiffSingerVariance.cs index a70d90005..b005a40e1 100644 --- a/OpenUtau.Core/DiffSinger/DiffSingerVariance.cs +++ b/OpenUtau.Core/DiffSinger/DiffSingerVariance.cs @@ -269,11 +269,6 @@ public VarianceResult Process(RenderPhrase phrase){ phrase.AddCacheFile(acceptedVarianceCache?.Filename); return acceptedVariance.Result; } - var pitchPriorities = RenderContext.PreRenderPriorities - .Where(priority => - priority.editKind == PreRenderEditKind.Pitch && - RenderPriority.Overlaps(phrase.position, phrase.end, priority.startTick, priority.endTick)) - .ToArray(); var varianceCache = Preferences.Default.DiffSingerTensorCache ? new DiffSingerCache(varianceHash, varianceInputs) : null; @@ -321,11 +316,11 @@ public VarianceResult Process(RenderPhrase phrase){ return fullResult; } var result = fullResult; - if (acceptedVariance != null && pitchPriorities.Length > 0) { - result = DiffSingerVariancePatcher.Patch( - phrase, acceptedVariance.Result, fullResult, pitchPriorities); + if (acceptedVariance?.Pitch != null) { + result = DiffSingerVariancePatcher.PatchByPitchChange( + acceptedVariance.Result, fullResult, acceptedVariance.Pitch, pitch); } - SaveAcceptedVariance(acceptedVarianceCache, result, pitchHash); + SaveAcceptedVariance(acceptedVarianceCache, result, pitchHash, pitch); phrase.AddCacheFile(acceptedVarianceCache.Filename); return result; } @@ -339,6 +334,7 @@ ICollection AcceptedVarianceCacheInputs(ICollection { NamedOnnxValue.CreateFromTensor("pitch_hash", new DenseTensor(new[] { unchecked((long)pitchHash) }, new[] { 1 })), @@ -395,6 +392,7 @@ void SaveAcceptedVariance(DiffSingerCache cache, VarianceResult result, ulong pi NamedOnnxValue.CreateFromTensor("total_frames", new DenseTensor(new[] { (long)result.totalFrames }, new[] { 1 })), }; + AddOptionalFloatArray(outputs, "pitch", pitch); AddOptionalFloatArray(outputs, "energy_pred", result.energy); AddOptionalFloatArray(outputs, "breathiness_pred", result.breathiness); AddOptionalFloatArray(outputs, "voicing_pred", result.voicing); diff --git a/OpenUtau.Core/DiffSinger/DiffSingerVariancePatcher.cs b/OpenUtau.Core/DiffSinger/DiffSingerVariancePatcher.cs index 815708ef4..6a08c5921 100644 --- a/OpenUtau.Core/DiffSinger/DiffSingerVariancePatcher.cs +++ b/OpenUtau.Core/DiffSinger/DiffSingerVariancePatcher.cs @@ -2,12 +2,12 @@ using System.Collections.Generic; using System.Linq; using K4os.Hash.xxHash; -using OpenUtau.Core.Render; namespace OpenUtau.Core.DiffSinger { internal static class DiffSingerVariancePatcher { const double InfluenceMarginMs = 250; const double FadeMs = 100; + const float PitchEpsilon = 0.0001f; internal static ulong HashPitch(float[] pitch) { var bytes = new byte[pitch.Length * sizeof(float)]; @@ -30,15 +30,24 @@ static bool SameLength(float[]? a, float[]? b) { return (a == null && b == null) || (a != null && b != null && a.Length == b.Length); } - internal static VarianceResult Patch( - RenderPhrase phrase, + internal static VarianceResult PatchByPitchChange( VarianceResult oldResult, VarianceResult newResult, - IEnumerable priorities) { - if (!CanPatch(oldResult, newResult)) { + float[] oldPitch, + float[] newPitch) { + if (!CanPatch(oldResult, newResult) || + oldPitch.Length != newPitch.Length || + oldPitch.Length != newResult.totalFrames) { return newResult; } - var weights = BuildWeights(phrase, newResult, priorities); + var weights = BuildWeightsForPitchChanges(oldPitch, newPitch, newResult.frameMs); + return PatchWithWeights(oldResult, newResult, weights); + } + + static VarianceResult PatchWithWeights( + VarianceResult oldResult, + VarianceResult newResult, + float[] weights) { if (weights.All(w => w <= 0)) { return oldResult; } @@ -54,26 +63,54 @@ internal static VarianceResult Patch( }; } - internal static float[] BuildWeights( - RenderPhrase phrase, - VarianceResult result, - IEnumerable priorities) { - double startMs = phrase.positionMs - result.headFrames * result.frameMs; - var ranges = priorities - .Where(priority => - priority.editKind == PreRenderEditKind.Pitch && - RenderPriority.Overlaps(phrase.position, phrase.end, priority.startTick, priority.endTick)) - .Select(priority => ( - startMs: phrase.timeAxis.TickPosToMsPos(priority.startTick), - endMs: phrase.timeAxis.TickPosToMsPos(priority.endTick))); - return BuildWeightsForMsRanges(result.totalFrames, result.frameMs, startMs, ranges); - } - internal static float[] BuildWeightsForMsRanges( int totalFrames, float frameMs, double startMs, IEnumerable<(double startMs, double endMs)> ranges) { + return BuildWeightsForFrameRanges( + totalFrames, + frameMs, + ranges.Select(range => ( + startFrame: FrameIndex(range.startMs, startMs, frameMs, totalFrames), + endFrame: FrameIndex(range.endMs, startMs, frameMs, totalFrames)))); + } + + internal static float[] BuildWeightsForPitchChanges( + float[] oldPitch, + float[] newPitch, + float frameMs) { + if (oldPitch.Length != newPitch.Length) { + return Array.Empty(); + } + return BuildWeightsForFrameRanges( + newPitch.Length, + frameMs, + PitchChangeRanges(oldPitch, newPitch)); + } + + static IEnumerable<(int startFrame, int endFrame)> PitchChangeRanges( + float[] oldPitch, + float[] newPitch) { + int start = -1; + for (int i = 0; i < newPitch.Length; ++i) { + bool changed = Math.Abs(oldPitch[i] - newPitch[i]) > PitchEpsilon; + if (changed && start < 0) { + start = i; + } else if (!changed && start >= 0) { + yield return (start, i); + start = -1; + } + } + if (start >= 0) { + yield return (start, newPitch.Length); + } + } + + static float[] BuildWeightsForFrameRanges( + int totalFrames, + float frameMs, + IEnumerable<(int startFrame, int endFrame)> ranges) { var weights = new float[totalFrames]; if (totalFrames <= 0) { return weights; @@ -81,8 +118,8 @@ internal static float[] BuildWeightsForMsRanges( int marginFrames = Math.Max(0, (int)Math.Round(InfluenceMarginMs / frameMs)); int fadeFrames = Math.Max(1, (int)Math.Round(FadeMs / frameMs)); foreach (var range in ranges) { - int coreStart = FrameIndex(range.startMs, startMs, frameMs, totalFrames); - int coreEnd = FrameIndex(range.endMs, startMs, frameMs, totalFrames); + int coreStart = Math.Clamp(range.startFrame, 0, totalFrames); + int coreEnd = Math.Clamp(range.endFrame, 0, totalFrames); if (coreEnd <= coreStart) { coreEnd = Math.Min(totalFrames, coreStart + 1); } diff --git a/OpenUtau.Core/DocManager.cs b/OpenUtau.Core/DocManager.cs index 75365c3dd..16904f955 100644 --- a/OpenUtau.Core/DocManager.cs +++ b/OpenUtau.Core/DocManager.cs @@ -286,9 +286,9 @@ public void EndUndoGroup() { } PreRenderNotification CreatePreRenderNotification(IEnumerable commands) { - var priorityRanges = new Dictionary(); + var priorityRanges = new Dictionary(); foreach (var cmd in commands) { - if (!TryGetPreRenderRange(cmd, out var part, out var cmdStartTick, out var cmdEndTick, out var editKind) || + if (!TryGetPreRenderRange(cmd, out var part, out var cmdStartTick, out var cmdEndTick) || part == null || !Project.parts.Contains(part)) { continue; @@ -296,58 +296,23 @@ PreRenderNotification CreatePreRenderNotification(IEnumerable commands if (priorityRanges.TryGetValue(part, out var range)) { priorityRanges[part] = ( Math.Min(range.startTick, cmdStartTick), - Math.Max(range.endTick, cmdEndTick), - MergeEditKind(range.editKind, editKind)); + Math.Max(range.endTick, cmdEndTick)); } else { - priorityRanges[part] = (cmdStartTick, cmdEndTick, editKind); + priorityRanges[part] = (cmdStartTick, cmdEndTick); } } if (priorityRanges.Count > 0) { return new PreRenderNotification(priorityRanges.Select(range => - new PreRenderPriority(range.Key, range.Value.startTick, range.Value.endTick, range.Value.editKind))); + new PreRenderPriority(range.Key, range.Value.startTick, range.Value.endTick))); } return new PreRenderNotification(); } - PreRenderEditKind MergeEditKind(PreRenderEditKind a, PreRenderEditKind b) { - if (a == PreRenderEditKind.Pitch || b == PreRenderEditKind.Pitch) { - return PreRenderEditKind.Pitch; - } - if (a == PreRenderEditKind.VarianceCurve || b == PreRenderEditKind.VarianceCurve) { - return PreRenderEditKind.VarianceCurve; - } - return PreRenderEditKind.Generic; - } - - bool TryGetPreRenderRange( - UCommand cmd, - out UVoicePart? part, - out int startTick, - out int endTick, - out PreRenderEditKind editKind) { + bool TryGetPreRenderRange(UCommand cmd, out UVoicePart? part, out int startTick, out int endTick) { part = null; startTick = 0; endTick = 0; - editKind = PreRenderEditKind.Generic; switch (cmd) { - case PitchExpCommand pitchCommand when pitchCommand.Part != null: - part = pitchCommand.Part; - var pitchNotes = pitchCommand.AffectedNotes.ToArray(); - if (pitchNotes.Length > 0) { - startTick = part.position + pitchNotes.Min(note => note.position); - endTick = part.position + pitchNotes.Max(note => note.End); - } else { - startTick = part.position; - endTick = part.End; - } - editKind = PreRenderEditKind.Pitch; - return endTick > startTick; - case VibratoCommand vibratoCommand when vibratoCommand.Notes.Length > 0: - part = vibratoCommand.Part; - startTick = part.position + vibratoCommand.Notes.Min(note => note.position); - endTick = part.position + vibratoCommand.Notes.Max(note => note.End); - editKind = PreRenderEditKind.Pitch; - return endTick > startTick; case NoteCommand noteCommand when noteCommand.Notes.Length > 0: part = noteCommand.Part; startTick = part.position + noteCommand.Notes.Min(note => note.position); @@ -357,31 +322,16 @@ bool TryGetPreRenderRange( part = curveCommand.Part; startTick = part.position + curveCommand.StartTick; endTick = part.position + curveCommand.EndTick; - editKind = GetCurveEditKind(curveCommand.Abbr); return endTick > startTick; case MergedSetCurveCommand curveCommand: part = curveCommand.Part; startTick = part.position + curveCommand.StartTick; endTick = part.position + curveCommand.EndTick; - editKind = curveCommand.SetReal ? PreRenderEditKind.Generic : GetCurveEditKind(curveCommand.Abbr); - return endTick > startTick; - case PasteCurveCommand curveCommand: - part = curveCommand.Part; - startTick = part.position + curveCommand.StartTick; - endTick = part.position + curveCommand.EndTick; - editKind = GetCurveEditKind(curveCommand.Abbr); - return endTick > startTick; - case ClearCurveCommand curveCommand: - part = curveCommand.Part; - startTick = part.position + curveCommand.StartTick; - endTick = part.position + curveCommand.EndTick; - editKind = GetCurveEditKind(curveCommand.Abbr); return endTick > startTick; case ExpCommand expCommand when expCommand.Note != null: part = expCommand.Part; startTick = part.position + expCommand.Note.position; endTick = part.position + expCommand.Note.End; - editKind = GetExpressionEditKind(expCommand.Key); return endTick > startTick; case ExpCommand expCommand: part = expCommand.Part; @@ -404,42 +354,6 @@ bool TryGetPreRenderRange( } } - int GetExpCommandStartTick(ExpCommand command) { - if (command.Note != null) { - return command.Part.position + command.Note.position; - } - return command.Part.position; - } - - int GetExpCommandEndTick(ExpCommand command) { - if (command.Note != null) { - return command.Part.position + command.Note.End; - } - return command.Part.End; - } - - PreRenderEditKind GetExpressionEditKind(string? abbr) { - if (abbr == Format.Ustx.SHFT) { - return PreRenderEditKind.Pitch; - } - return PreRenderEditKind.Generic; - } - - PreRenderEditKind GetCurveEditKind(string? abbr) { - switch (abbr) { - case Format.Ustx.PITD: - case Format.Ustx.SHFC: - return PreRenderEditKind.Pitch; - case OpenUtau.Core.DiffSinger.DiffSingerUtils.ENE: - case Format.Ustx.BREC: - case Format.Ustx.VOIC: - case Format.Ustx.TENC: - return PreRenderEditKind.VarianceCurve; - default: - return PreRenderEditKind.Generic; - } - } - public void RollBackUndoGroup() { if (undoGroup == null) { Log.Error("No active undoGroup to rollback."); diff --git a/OpenUtau.Core/Editing/NoteBatchEdits.cs b/OpenUtau.Core/Editing/NoteBatchEdits.cs index 3d2985eae..817315430 100644 --- a/OpenUtau.Core/Editing/NoteBatchEdits.cs +++ b/OpenUtau.Core/Editing/NoteBatchEdits.cs @@ -750,7 +750,7 @@ public void Run(UProject project, UVoicePart part, List selectedNotes, Do docManager.ExecuteCmd(new DeletePitchPointCommand(part, note, index)); docManager.ExecuteCmd(new DeletePitchPointCommand(part, note, index)); var lastPitch = note.pitch.data[^1]; - docManager.ExecuteCmd(new MovePitchPointCommand(part, note, lastPitch, 0, -lastPitch.Y)); + docManager.ExecuteCmd(new MovePitchPointCommand(part, lastPitch, 0, -lastPitch.Y)); } } diff --git a/OpenUtau.Core/Render/RealCurveRefresh.cs b/OpenUtau.Core/Render/RealCurveRefresh.cs index a5813dca8..bee44d78f 100644 --- a/OpenUtau.Core/Render/RealCurveRefresh.cs +++ b/OpenUtau.Core/Render/RealCurveRefresh.cs @@ -18,13 +18,10 @@ public static bool CanRefresh(RenderPhrase phrase, bool allowSessionInitializati public static List LoadRenderedRealCurves( RenderPhrase phrase, - IEnumerable? priorities = null, bool allowSessionInitialization = true) { if (!CanRefresh(phrase, allowSessionInitialization)) { return new List(0); } - using var context = RenderContext.WithPreRenderPriorities( - priorities ?? Array.Empty()); return phrase.renderer.LoadRenderedRealCurves(phrase); } diff --git a/OpenUtau.Core/Render/RenderContext.cs b/OpenUtau.Core/Render/RenderContext.cs deleted file mode 100644 index 245937bc2..000000000 --- a/OpenUtau.Core/Render/RenderContext.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; - -namespace OpenUtau.Core.Render { - internal static class RenderContext { - static readonly AsyncLocal preRenderPriorities = new AsyncLocal(); - - internal static PreRenderPriority[] PreRenderPriorities => - preRenderPriorities.Value ?? Array.Empty(); - - internal static IDisposable WithPreRenderPriorities(IEnumerable priorities) { - var previous = preRenderPriorities.Value; - preRenderPriorities.Value = priorities.ToArray(); - return new Scope(() => preRenderPriorities.Value = previous); - } - - class Scope : IDisposable { - readonly Action dispose; - bool disposed; - - public Scope(Action dispose) { - this.dispose = dispose; - } - - public void Dispose() { - if (!disposed) { - dispose(); - disposed = true; - } - } - } - } -} diff --git a/OpenUtau.Core/Render/RenderEngine.cs b/OpenUtau.Core/Render/RenderEngine.cs index 7a8705f1a..a2ceb4aa2 100644 --- a/OpenUtau.Core/Render/RenderEngine.cs +++ b/OpenUtau.Core/Render/RenderEngine.cs @@ -258,8 +258,6 @@ private void RenderRequests( var phrase = tuple.phrase; var source = tuple.source; var request = tuple.request; - using var context = RenderContext.WithPreRenderPriorities( - IsDiffSinger(phrase) ? GetDiffSingerPriorityRanges(request.part) : Array.Empty()); var task = phrase.renderer.Render(phrase, progress, request.trackNo, cancellation, true); task.Wait(); if (cancellation.IsCancellationRequested) { @@ -360,7 +358,7 @@ private void RefreshRealCurves(RenderPhrase phrase, UVoicePart part) { } List results; try { - results = RealCurveRefresh.LoadRenderedRealCurves(phrase, GetDiffSingerPriorityRanges(part)); + results = RealCurveRefresh.LoadRenderedRealCurves(phrase); } catch (Exception e) { Log.Debug(e, "Failed to refresh DiffSinger real curves."); return; diff --git a/OpenUtau.Test/Core/Commands/PitchCommandTest.cs b/OpenUtau.Test/Core/Commands/PitchCommandTest.cs deleted file mode 100644 index 188f4ba0c..000000000 --- a/OpenUtau.Test/Core/Commands/PitchCommandTest.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System.Linq; -using OpenUtau.Core.Ustx; -using Xunit; - -namespace OpenUtau.Core { - public class PitchCommandTest { - [Fact] - public void MovePitchPointCommandReportsAffectedNote() { - var part = new UVoicePart(); - var note = new UNote { position = 240, duration = 480 }; - var point = new PitchPoint(0, 0, PitchPointShape.io); - - var command = new MovePitchPointCommand(part, note, point, 10, 20); - - Assert.Single(command.AffectedNotes); - Assert.Same(note, command.AffectedNotes.First()); - } - - [Fact] - public void ChangePitchPointShapeCommandReportsAffectedNote() { - var part = new UVoicePart(); - var note = new UNote { position = 240, duration = 480 }; - var point = new PitchPoint(0, 0, PitchPointShape.io); - - var command = new ChangePitchPointShapeCommand(part, note, point, PitchPointShape.l); - - Assert.Single(command.AffectedNotes); - Assert.Same(note, command.AffectedNotes.First()); - } - - [Fact] - public void LegacyPitchPointMoveConstructorHasNoAffectedNote() { - var part = new UVoicePart(); - var point = new PitchPoint(0, 0, PitchPointShape.io); - - var command = new MovePitchPointCommand(part, point, 10, 20); - - Assert.Empty(command.AffectedNotes); - } - } -} diff --git a/OpenUtau.Test/Core/DiffSinger/DiffSingerVariancePatcherTest.cs b/OpenUtau.Test/Core/DiffSinger/DiffSingerVariancePatcherTest.cs index 8b476d668..6dcad3908 100644 --- a/OpenUtau.Test/Core/DiffSinger/DiffSingerVariancePatcherTest.cs +++ b/OpenUtau.Test/Core/DiffSinger/DiffSingerVariancePatcherTest.cs @@ -35,6 +35,20 @@ public void BuildWeightsForMsRangesCrossfadesEdges() { Assert.Contains(weights, weight => weight > 0 && weight < 1); } + [Fact] + public void BuildWeightsForPitchChangesKeepsUnchangedEdgesAtZero() { + var oldPitch = Enumerable.Repeat(1f, 100).ToArray(); + var newPitch = Enumerable.Repeat(1f, 100).ToArray(); + newPitch[50] = 2f; + + var weights = DiffSingerVariancePatcher.BuildWeightsForPitchChanges( + oldPitch, newPitch, frameMs: 10); + + Assert.Equal(0, weights[0]); + Assert.Equal(0, weights[99]); + Assert.Contains(weights, weight => weight >= 1); + } + [Fact] public void PatchArrayBlendsOldAndNewByWeight() { var oldValues = Enumerable.Repeat(0f, 5).ToArray(); @@ -45,5 +59,29 @@ public void PatchArrayBlendsOldAndNewByWeight() { Assert.Equal(new[] { 0f, 2.5f, 5f, 7.5f, 10f }, result); } + + [Fact] + public void PatchByPitchChangeKeepsUnchangedFramesFromOldResult() { + var oldPitch = Enumerable.Repeat(1f, 100).ToArray(); + var newPitch = Enumerable.Repeat(1f, 100).ToArray(); + newPitch[50] = 2f; + var oldResult = new VarianceResult { + energy = Enumerable.Repeat(0f, 100).ToArray(), + frameMs = 10, + totalFrames = 100, + }; + var newResult = new VarianceResult { + energy = Enumerable.Repeat(10f, 100).ToArray(), + frameMs = 10, + totalFrames = 100, + }; + + var result = DiffSingerVariancePatcher.PatchByPitchChange( + oldResult, newResult, oldPitch, newPitch); + + Assert.Equal(0, result.energy![0]); + Assert.Equal(10, result.energy[50]); + Assert.Equal(0, result.energy[99]); + } } } diff --git a/OpenUtau/ViewModels/NotePropertiesViewModel.cs b/OpenUtau/ViewModels/NotePropertiesViewModel.cs index ec658e416..fc8749bc4 100644 --- a/OpenUtau/ViewModels/NotePropertiesViewModel.cs +++ b/OpenUtau/ViewModels/NotePropertiesViewModel.cs @@ -532,7 +532,7 @@ public void SetNoteParams(string tag, object? obj) { var newX = firstPoint.X + (pitchPoint.X - firstPoint.X) * scale; var deltaX = newX - pitchPoint.X; if (deltaX != 0) { - DocManager.Inst.ExecuteCmd(new MovePitchPointCommand(Part, note, pitchPoint, deltaX, 0)); + DocManager.Inst.ExecuteCmd(new MovePitchPointCommand(Part, pitchPoint, deltaX, 0)); } } } @@ -544,7 +544,7 @@ public void SetNoteParams(string tag, object? obj) { if (deltaX != 0) { foreach (var note in selectedNotes) { foreach (var pitchPoint in note.pitch.data) { - DocManager.Inst.ExecuteCmd(new MovePitchPointCommand(Part, note, pitchPoint, deltaX, 0)); + DocManager.Inst.ExecuteCmd(new MovePitchPointCommand(Part, pitchPoint, deltaX, 0)); } } } diff --git a/OpenUtau/ViewModels/NotesViewModel.cs b/OpenUtau/ViewModels/NotesViewModel.cs index 65c494fb5..24b324df9 100644 --- a/OpenUtau/ViewModels/NotesViewModel.cs +++ b/OpenUtau/ViewModels/NotesViewModel.cs @@ -5,7 +5,6 @@ using System.Numerics; using System.Reactive; using System.Reactive.Linq; -using System.Threading; using System.Threading.Tasks; using Avalonia; using Avalonia.Controls; @@ -15,7 +14,6 @@ using DynamicData.Binding; using OpenUtau.App.Views; using OpenUtau.Core; -using OpenUtau.Core.Render; using OpenUtau.Core.Ustx; using OpenUtau.Core.Util; using OpenUtau.ViewModels; @@ -106,8 +104,6 @@ public class NotesViewModel : ViewModelBase, ICmdSubscriber { // See the comments on TracksViewModel.playPosXToTickOffset private double playPosXToTickOffset => Bounds.Width != 0 ? ViewportTicks / Bounds.Width : 0; - private int realCurveRefreshVersion; - private readonly ObservableAsPropertyHelper viewportTicks; private readonly ObservableAsPropertyHelper viewportTracks; private readonly ObservableAsPropertyHelper smallChangeX; @@ -1104,8 +1100,6 @@ public void OnNext(UCommand cmd, bool isUndo) { } else if (cmd is PhonemizedNotification) { OnPartModified(); MessageBus.Current.SendMessage(new NotesRefreshEvent()); - } else if (notif is PreRenderNotification preRender) { - ScheduleDiffSingerRealCurveRefresh(preRender.priorities); } else if (notif is RealCurvesRenderedNotification && notif.part == Part) { MessageBus.Current.SendMessage(new NotesRefreshEvent()); } else if ((notif is PhraseRenderedNotification || notif is PartRenderedNotification) && notif.part == Part) { @@ -1172,87 +1166,6 @@ public void OnNext(UCommand cmd, bool isUndo) { } } - private void ScheduleDiffSingerRealCurveRefresh(IEnumerable priorities) { - if (Part == null) { - return; - } - var part = Part; - var project = Project; - var relevantPriorities = priorities - .Where(priority => - ReferenceEquals(priority.part, part) && - (priority.editKind == PreRenderEditKind.Pitch || - priority.editKind == PreRenderEditKind.VarianceCurve)) - .ToArray(); - if (relevantPriorities.Length == 0) { - return; - } - var phrases = part.renderPhrases - .Where(phrase => - RealCurveRefresh.CanRefresh(phrase, allowSessionInitialization: false) && - relevantPriorities.Any(priority => - phrase.end > priority.startTick && phrase.position < priority.endTick)) - .ToArray(); - if (phrases.Length == 0) { - return; - } - int version = Interlocked.Increment(ref realCurveRefreshVersion); - _ = Task.Run(() => RefreshDiffSingerRealCurves(project, part, phrases, relevantPriorities, version)); - } - - private void RefreshDiffSingerRealCurves( - UProject project, - UVoicePart part, - RenderPhrase[] phrases, - PreRenderPriority[] priorities, - int version) { - try { - var phraseResults = new List<(RenderPhrase phrase, List results)>(); - foreach (var phrase in phrases) { - if (Volatile.Read(ref realCurveRefreshVersion) != version) { - return; - } - try { - var results = RealCurveRefresh.LoadRenderedRealCurves( - phrase, priorities, allowSessionInitialization: false); - if (results.Count > 0) { - phraseResults.Add((phrase, results)); - } - } catch (Exception e) { - Log.Debug(e, "Failed to refresh DiffSinger real curves."); - } - } - if (phraseResults.Count == 0 || - Volatile.Read(ref realCurveRefreshVersion) != version) { - return; - } - void Apply() { - if (Volatile.Read(ref realCurveRefreshVersion) != version || - !ReferenceEquals(Project, project) || - !ReferenceEquals(Part, part)) { - return; - } - bool changed = false; - lock (part) { - foreach (var phraseResult in phraseResults) { - changed |= RealCurveRefresh.ApplyRealCurveResults( - project, part, phraseResult.phrase, phraseResult.results); - } - } - if (changed) { - MessageBus.Current.SendMessage(new NotesRefreshEvent()); - } - } - if (DocManager.Inst.PostOnUIThread != null) { - DocManager.Inst.PostOnUIThread(Apply); - } else { - Apply(); - } - } catch (Exception e) { - Log.Debug(e, "Failed to refresh DiffSinger real curves."); - } - } - private void MaybeAutoScroll(double positionX) { var autoScrollPreference = Convert.ToBoolean(Preferences.Default.PlaybackAutoScroll); if (autoScrollPreference) { diff --git a/OpenUtau/ViewModels/PianoRollViewModel.cs b/OpenUtau/ViewModels/PianoRollViewModel.cs index f7fa0962e..d0ce85699 100644 --- a/OpenUtau/ViewModels/PianoRollViewModel.cs +++ b/OpenUtau/ViewModels/PianoRollViewModel.cs @@ -128,31 +128,31 @@ public PianoRollViewModel() { PitEaseInOutCommand = ReactiveCommand.Create(info => { if (NotesViewModel.Part == null) { return; } DocManager.Inst.StartUndoGroup("command.pitch.editpoint"); - DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note, info.Note.pitch.data[info.Index], PitchPointShape.io)); + DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note.pitch.data[info.Index], PitchPointShape.io)); DocManager.Inst.EndUndoGroup(); }); PitLinearCommand = ReactiveCommand.Create(info => { if (NotesViewModel.Part == null) { return; } DocManager.Inst.StartUndoGroup("command.pitch.editpoint"); - DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note, info.Note.pitch.data[info.Index], PitchPointShape.l)); + DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note.pitch.data[info.Index], PitchPointShape.l)); DocManager.Inst.EndUndoGroup(); }); PitEaseInCommand = ReactiveCommand.Create(info => { if (NotesViewModel.Part == null) { return; } DocManager.Inst.StartUndoGroup("command.pitch.editpoint"); - DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note, info.Note.pitch.data[info.Index], PitchPointShape.i)); + DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note.pitch.data[info.Index], PitchPointShape.i)); DocManager.Inst.EndUndoGroup(); }); PitEaseOutCommand = ReactiveCommand.Create(info => { if (NotesViewModel.Part == null) { return; } DocManager.Inst.StartUndoGroup("command.pitch.editpoint"); - DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note, info.Note.pitch.data[info.Index], PitchPointShape.o)); + DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note.pitch.data[info.Index], PitchPointShape.o)); DocManager.Inst.EndUndoGroup(); }); PitSplineCommand = ReactiveCommand.Create(info => { if (NotesViewModel.Part == null) { return; } DocManager.Inst.StartUndoGroup("command.pitch.editpoint"); - DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note, info.Note.pitch.data[info.Index], PitchPointShape.sp)); + DocManager.Inst.ExecuteCmd(new ChangePitchPointShapeCommand(NotesViewModel.Part, info.Note.pitch.data[info.Index], PitchPointShape.sp)); DocManager.Inst.EndUndoGroup(); }); PitSnapCommand = ReactiveCommand.Create(info => { diff --git a/OpenUtau/Views/NoteEditStates.cs b/OpenUtau/Views/NoteEditStates.cs index f10372227..0752dcdb6 100644 --- a/OpenUtau/Views/NoteEditStates.cs +++ b/OpenUtau/Views/NoteEditStates.cs @@ -643,7 +643,7 @@ public override void Update(IPointer pointer, Point point) { return; } if (notesVm.Part != null) { - DocManager.Inst.ExecuteCmd(new MovePitchPointCommand(notesVm.Part, note, pitchPoint, (float)deltaX, (float)deltaY)); + DocManager.Inst.ExecuteCmd(new MovePitchPointCommand(notesVm.Part, pitchPoint, (float)deltaX, (float)deltaY)); } valueTip.UpdateValueTip($"{pitchPoint.X:0.0}ms, {pitchPoint.Y * 10:0}cent"); }