diff --git a/OpenUtau.Core/Commands/ExpCommands.cs b/OpenUtau.Core/Commands/ExpCommands.cs index 072e12c71..e338b733a 100644 --- a/OpenUtau.Core/Commands/ExpCommands.cs +++ b/OpenUtau.Core/Commands/ExpCommands.cs @@ -317,6 +317,8 @@ public class SetCurveCommand : ExpCommand { readonly int lastY; int[] oldXs; int[] oldYs; + public int StartTick => Math.Min(x, lastX); + public int EndTick => Math.Max(x, lastX) + 1; public override ValidateOptions ValidateOptions => new ValidateOptions { SkipTiming = true, @@ -384,6 +386,14 @@ public class MergedSetCurveCommand : ExpCommand { readonly int[] newXs; readonly int[] newYs; readonly bool setReal; + public int StartTick => (oldXs ?? Array.Empty()) + .Concat(newXs ?? Array.Empty()) + .DefaultIfEmpty(0) + .Min(); + public int EndTick => (oldXs ?? Array.Empty()) + .Concat(newXs ?? Array.Empty()) + .DefaultIfEmpty(Part.Duration) + .Max() + 1; public MergedSetCurveCommand(UProject project, UVoicePart part, string abbr, int[] oldXs, int[] oldYs, int[] newXs, int[] newYs, bool setReal = false) : base(part) { this.project = project; diff --git a/OpenUtau.Core/Commands/Notifications.cs b/OpenUtau.Core/Commands/Notifications.cs index 0ee6448fa..9df5db9c1 100644 --- a/OpenUtau.Core/Commands/Notifications.cs +++ b/OpenUtau.Core/Commands/Notifications.cs @@ -222,8 +222,55 @@ public FocusNoteNotification(UPart part, UNote note) { public override string ToString() => $"Focus note {note.lyric} at {note.position}."; } + public class PreRenderPriority { + public readonly UVoicePart part; + public readonly int startTick; + public readonly int endTick; + + public PreRenderPriority(UVoicePart part, int startTick, int endTick) { + this.part = part; + this.startTick = startTick; + this.endTick = endTick; + } + } + public class PreRenderNotification : UNotification { - public override string ToString() => $"Pre-render notification."; + public readonly PreRenderPriority[] priorities; + public UVoicePart? priorityPart => System.Linq.Enumerable.FirstOrDefault(priorities)?.part; + public int priorityStartTick => System.Linq.Enumerable.FirstOrDefault(priorities)?.startTick ?? -1; + public int priorityEndTick => System.Linq.Enumerable.FirstOrDefault(priorities)?.endTick ?? -1; + public bool HasPriorityRange => priorities.Length > 0; + + public PreRenderNotification() { + priorities = Array.Empty(); + } + + public PreRenderNotification(UVoicePart priorityPart, int priorityStartTick, int priorityEndTick) + : this(new[] { new PreRenderPriority(priorityPart, priorityStartTick, priorityEndTick) }) { } + + public PreRenderNotification(System.Collections.Generic.IEnumerable priorities) { + this.priorities = System.Linq.Enumerable.ToArray( + System.Linq.Enumerable.Where(priorities, priority => priority.endTick > priority.startTick)); + part = priorityPart; + } + + public override string ToString() => HasPriorityRange + ? $"Pre-render notification {priorities.Length} prioritized range(s)." + : $"Pre-render notification."; + } + + public class PhraseRenderedNotification : UNotification { + public PhraseRenderedNotification(UVoicePart part) { + this.part = part; + } + public override string ToString() => "Phrase rendered."; + } + + public class RealCurvesRenderedNotification : UNotification { + public RealCurvesRenderedNotification(UVoicePart part) { + this.part = part; + } + public override string ToString() => "Real curves rendered."; } public class PartRenderedNotification : UNotification { diff --git a/OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs b/OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs index 7844eea55..32556105a 100644 --- a/OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs +++ b/OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs @@ -81,6 +81,9 @@ public RenderResult Layout(RenderPhrase phrase) { public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { var task = Task.Run(() => { + if (cancellation.IsCancellationRequested) { + return new RenderResult(); + } lock (lockObj) { if (cancellation.IsCancellationRequested) { return new RenderResult(); diff --git a/OpenUtau.Core/DiffSinger/DiffSingerVariance.cs b/OpenUtau.Core/DiffSinger/DiffSingerVariance.cs index 5f12b9802..b005a40e1 100644 --- a/OpenUtau.Core/DiffSinger/DiffSingerVariance.cs +++ b/OpenUtau.Core/DiffSinger/DiffSingerVariance.cs @@ -24,6 +24,7 @@ public struct VarianceResult{ public int totalFrames; } public class DsVariance : IDisposable{ + const ulong AcceptedVarianceCacheSalt = 0x9E3779B97F4A7C15UL; string rootPath; DsConfig dsConfig; Dictionary languageIds = new Dictionary(); @@ -257,6 +258,17 @@ public VarianceResult Process(RenderPhrase phrase){ varianceInputs.Add(NamedOnnxValue.CreateFromTensor("spk_embed", spkEmbedTensor)); } Onnx.VerifyInputNames(varianceModel, varianceInputs); + var acceptedVarianceCache = Preferences.Default.DiffSingerTensorCache + ? new DiffSingerCache(varianceHash ^ AcceptedVarianceCacheSalt, AcceptedVarianceCacheInputs(varianceInputs)) + : null; + ulong pitchHash = DiffSingerVariancePatcher.HashPitch(pitch); + var acceptedVariance = acceptedVarianceCache == null + ? null + : LoadAcceptedVariance(acceptedVarianceCache, frameMs, headFrames, tailFrames, totalFrames); + if (acceptedVariance != null && acceptedVariance.PitchHash == pitchHash) { + phrase.AddCacheFile(acceptedVarianceCache?.Filename); + return acceptedVariance.Result; + } var varianceCache = Preferences.Default.DiffSingerTensorCache ? new DiffSingerCache(varianceHash, varianceInputs) : null; @@ -290,7 +302,7 @@ public VarianceResult Process(RenderPhrase phrase){ .First() .AsTensor() : null; - return new VarianceResult{ + var fullResult = new VarianceResult{ energy = energy_pred?.ToArray(), breathiness = breathiness_pred?.ToArray(), voicing = voicing_pred?.ToArray(), @@ -300,6 +312,111 @@ public VarianceResult Process(RenderPhrase phrase){ tailFrames = tailFrames, totalFrames = totalFrames, }; + if (acceptedVarianceCache == null) { + return fullResult; + } + var result = fullResult; + if (acceptedVariance?.Pitch != null) { + result = DiffSingerVariancePatcher.PatchByPitchChange( + acceptedVariance.Result, fullResult, acceptedVariance.Pitch, pitch); + } + SaveAcceptedVariance(acceptedVarianceCache, result, pitchHash, pitch); + phrase.AddCacheFile(acceptedVarianceCache.Filename); + return result; + } + + ICollection AcceptedVarianceCacheInputs(ICollection varianceInputs) { + return varianceInputs + .Where(input => input.Name != "pitch" && input.Name != "retake") + .ToArray(); + } + + class AcceptedVariance { + public VarianceResult Result; + public ulong PitchHash; + public float[]? Pitch; + } + + AcceptedVariance? LoadAcceptedVariance( + DiffSingerCache cache, + float expectedFrameMs, + int expectedHeadFrames, + int expectedTailFrames, + int expectedTotalFrames) { + var outputs = cache.Load(); + if (outputs == null) { + return null; + } + try { + var frameMsValue = GetRequiredFloat(outputs, "frame_ms"); + var headFramesValue = (int)GetRequiredLong(outputs, "head_frames"); + var tailFramesValue = (int)GetRequiredLong(outputs, "tail_frames"); + var totalFramesValue = (int)GetRequiredLong(outputs, "total_frames"); + if (Math.Abs(frameMsValue - expectedFrameMs) > 0.0001f || + headFramesValue != expectedHeadFrames || + tailFramesValue != expectedTailFrames || + totalFramesValue != expectedTotalFrames) { + return null; + } + return new AcceptedVariance { + PitchHash = unchecked((ulong)GetRequiredLong(outputs, "pitch_hash")), + Pitch = GetOptionalFloatArray(outputs, "pitch"), + Result = new VarianceResult { + energy = GetOptionalFloatArray(outputs, "energy_pred"), + breathiness = GetOptionalFloatArray(outputs, "breathiness_pred"), + voicing = GetOptionalFloatArray(outputs, "voicing_pred"), + tension = GetOptionalFloatArray(outputs, "tension_pred"), + frameMs = frameMsValue, + headFrames = headFramesValue, + tailFrames = tailFramesValue, + totalFrames = totalFramesValue, + } + }; + } catch (Exception e) { + Log.Error(e, "Failed to load accepted variance cache."); + cache.Delete(); + return null; + } + } + + void SaveAcceptedVariance(DiffSingerCache cache, VarianceResult result, ulong pitchHash, float[] pitch) { + var outputs = new List { + NamedOnnxValue.CreateFromTensor("pitch_hash", + new DenseTensor(new[] { unchecked((long)pitchHash) }, new[] { 1 })), + NamedOnnxValue.CreateFromTensor("frame_ms", + new DenseTensor(new[] { result.frameMs }, new[] { 1 })), + NamedOnnxValue.CreateFromTensor("head_frames", + new DenseTensor(new[] { (long)result.headFrames }, new[] { 1 })), + NamedOnnxValue.CreateFromTensor("tail_frames", + new DenseTensor(new[] { (long)result.tailFrames }, new[] { 1 })), + NamedOnnxValue.CreateFromTensor("total_frames", + new DenseTensor(new[] { (long)result.totalFrames }, new[] { 1 })), + }; + AddOptionalFloatArray(outputs, "pitch", pitch); + AddOptionalFloatArray(outputs, "energy_pred", result.energy); + AddOptionalFloatArray(outputs, "breathiness_pred", result.breathiness); + AddOptionalFloatArray(outputs, "voicing_pred", result.voicing); + AddOptionalFloatArray(outputs, "tension_pred", result.tension); + cache.Save(outputs); + } + + static void AddOptionalFloatArray(List outputs, string name, float[]? values) { + if (values != null) { + outputs.Add(NamedOnnxValue.CreateFromTensor( + name, new DenseTensor(values, new[] { values.Length }))); + } + } + + static float[]? GetOptionalFloatArray(ICollection outputs, string name) { + return outputs.FirstOrDefault(output => output.Name == name)?.AsTensor().ToArray(); + } + + static float GetRequiredFloat(ICollection outputs, string name) { + return outputs.First(output => output.Name == name).AsTensor().First(); + } + + static long GetRequiredLong(ICollection outputs, string name) { + return outputs.First(output => output.Name == name).AsTensor().First(); } private bool disposedValue; diff --git a/OpenUtau.Core/DiffSinger/DiffSingerVariancePatcher.cs b/OpenUtau.Core/DiffSinger/DiffSingerVariancePatcher.cs new file mode 100644 index 000000000..6a08c5921 --- /dev/null +++ b/OpenUtau.Core/DiffSinger/DiffSingerVariancePatcher.cs @@ -0,0 +1,166 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using K4os.Hash.xxHash; + +namespace OpenUtau.Core.DiffSinger { + internal static class DiffSingerVariancePatcher { + const double InfluenceMarginMs = 250; + const double FadeMs = 100; + const float PitchEpsilon = 0.0001f; + + internal static ulong HashPitch(float[] pitch) { + var bytes = new byte[pitch.Length * sizeof(float)]; + Buffer.BlockCopy(pitch, 0, bytes, 0, bytes.Length); + return XXH64.DigestOf(bytes); + } + + internal static bool CanPatch(VarianceResult oldResult, VarianceResult newResult) { + return oldResult.totalFrames == newResult.totalFrames && + oldResult.headFrames == newResult.headFrames && + oldResult.tailFrames == newResult.tailFrames && + Math.Abs(oldResult.frameMs - newResult.frameMs) < 0.0001f && + SameLength(oldResult.energy, newResult.energy) && + SameLength(oldResult.breathiness, newResult.breathiness) && + SameLength(oldResult.voicing, newResult.voicing) && + SameLength(oldResult.tension, newResult.tension); + } + + static bool SameLength(float[]? a, float[]? b) { + return (a == null && b == null) || (a != null && b != null && a.Length == b.Length); + } + + internal static VarianceResult PatchByPitchChange( + VarianceResult oldResult, + VarianceResult newResult, + float[] oldPitch, + float[] newPitch) { + if (!CanPatch(oldResult, newResult) || + oldPitch.Length != newPitch.Length || + oldPitch.Length != newResult.totalFrames) { + return newResult; + } + var weights = BuildWeightsForPitchChanges(oldPitch, newPitch, newResult.frameMs); + return PatchWithWeights(oldResult, newResult, weights); + } + + static VarianceResult PatchWithWeights( + VarianceResult oldResult, + VarianceResult newResult, + float[] weights) { + if (weights.All(w => w <= 0)) { + return oldResult; + } + return new VarianceResult { + energy = PatchArray(oldResult.energy, newResult.energy, weights), + breathiness = PatchArray(oldResult.breathiness, newResult.breathiness, weights), + voicing = PatchArray(oldResult.voicing, newResult.voicing, weights), + tension = PatchArray(oldResult.tension, newResult.tension, weights), + frameMs = newResult.frameMs, + headFrames = newResult.headFrames, + tailFrames = newResult.tailFrames, + totalFrames = newResult.totalFrames, + }; + } + + internal static float[] BuildWeightsForMsRanges( + int totalFrames, + float frameMs, + double startMs, + IEnumerable<(double startMs, double endMs)> ranges) { + return BuildWeightsForFrameRanges( + totalFrames, + frameMs, + ranges.Select(range => ( + startFrame: FrameIndex(range.startMs, startMs, frameMs, totalFrames), + endFrame: FrameIndex(range.endMs, startMs, frameMs, totalFrames)))); + } + + internal static float[] BuildWeightsForPitchChanges( + float[] oldPitch, + float[] newPitch, + float frameMs) { + if (oldPitch.Length != newPitch.Length) { + return Array.Empty(); + } + return BuildWeightsForFrameRanges( + newPitch.Length, + frameMs, + PitchChangeRanges(oldPitch, newPitch)); + } + + static IEnumerable<(int startFrame, int endFrame)> PitchChangeRanges( + float[] oldPitch, + float[] newPitch) { + int start = -1; + for (int i = 0; i < newPitch.Length; ++i) { + bool changed = Math.Abs(oldPitch[i] - newPitch[i]) > PitchEpsilon; + if (changed && start < 0) { + start = i; + } else if (!changed && start >= 0) { + yield return (start, i); + start = -1; + } + } + if (start >= 0) { + yield return (start, newPitch.Length); + } + } + + static float[] BuildWeightsForFrameRanges( + int totalFrames, + float frameMs, + IEnumerable<(int startFrame, int endFrame)> ranges) { + var weights = new float[totalFrames]; + if (totalFrames <= 0) { + return weights; + } + int marginFrames = Math.Max(0, (int)Math.Round(InfluenceMarginMs / frameMs)); + int fadeFrames = Math.Max(1, (int)Math.Round(FadeMs / frameMs)); + foreach (var range in ranges) { + int coreStart = Math.Clamp(range.startFrame, 0, totalFrames); + int coreEnd = Math.Clamp(range.endFrame, 0, totalFrames); + if (coreEnd <= coreStart) { + coreEnd = Math.Min(totalFrames, coreStart + 1); + } + int replaceStart = Math.Max(0, coreStart - marginFrames); + int replaceEnd = Math.Min(totalFrames, coreEnd + marginFrames); + int fadeStart = Math.Max(0, replaceStart - fadeFrames); + int fadeEnd = Math.Min(totalFrames, replaceEnd + fadeFrames); + for (int i = fadeStart; i < fadeEnd; ++i) { + float weight; + if (i < replaceStart) { + weight = SmoothStep((float)(i - fadeStart + 1) / Math.Max(1, replaceStart - fadeStart + 1)); + } else if (i < replaceEnd) { + weight = 1; + } else { + weight = 1 - SmoothStep((float)(i - replaceEnd + 1) / Math.Max(1, fadeEnd - replaceEnd + 1)); + } + weights[i] = Math.Max(weights[i], weight); + } + } + return weights; + } + + static int FrameIndex(double ms, double startMs, float frameMs, int totalFrames) { + return Math.Clamp((int)Math.Round((ms - startMs) / frameMs), 0, totalFrames); + } + + static float SmoothStep(float x) { + x = Math.Clamp(x, 0, 1); + return x * x * (3 - 2 * x); + } + + internal static float[]? PatchArray(float[]? oldValues, float[]? newValues, float[] weights) { + if (oldValues == null || newValues == null || oldValues.Length != newValues.Length) { + return newValues; + } + var result = new float[newValues.Length]; + for (int i = 0; i < result.Length; ++i) { + float weight = weights[i]; + result[i] = oldValues[i] * (1 - weight) + newValues[i] * weight; + } + return result; + } + } +} diff --git a/OpenUtau.Core/DocManager.cs b/OpenUtau.Core/DocManager.cs index 22fb2fbba..16904f955 100644 --- a/OpenUtau.Core/DocManager.cs +++ b/OpenUtau.Core/DocManager.cs @@ -278,10 +278,80 @@ public void EndUndoGroup() { if (undoGroup.DeferValidate) { Project.ValidateFull(); } + var preRenderNotification = CreatePreRenderNotification(undoGroup.Commands); undoGroup.Merge(); undoGroup = null; Log.Information("undoGroup ended"); - ExecuteCmd(new PreRenderNotification()); + ExecuteCmd(preRenderNotification); + } + + PreRenderNotification CreatePreRenderNotification(IEnumerable commands) { + var priorityRanges = new Dictionary(); + foreach (var cmd in commands) { + if (!TryGetPreRenderRange(cmd, out var part, out var cmdStartTick, out var cmdEndTick) || + part == null || + !Project.parts.Contains(part)) { + continue; + } + if (priorityRanges.TryGetValue(part, out var range)) { + priorityRanges[part] = ( + Math.Min(range.startTick, cmdStartTick), + Math.Max(range.endTick, cmdEndTick)); + } else { + priorityRanges[part] = (cmdStartTick, cmdEndTick); + } + } + if (priorityRanges.Count > 0) { + return new PreRenderNotification(priorityRanges.Select(range => + new PreRenderPriority(range.Key, range.Value.startTick, range.Value.endTick))); + } + return new PreRenderNotification(); + } + + bool TryGetPreRenderRange(UCommand cmd, out UVoicePart? part, out int startTick, out int endTick) { + part = null; + startTick = 0; + endTick = 0; + switch (cmd) { + case NoteCommand noteCommand when noteCommand.Notes.Length > 0: + part = noteCommand.Part; + startTick = part.position + noteCommand.Notes.Min(note => note.position); + endTick = part.position + noteCommand.Notes.Max(note => note.End); + return endTick > startTick; + case SetCurveCommand curveCommand: + part = curveCommand.Part; + startTick = part.position + curveCommand.StartTick; + endTick = part.position + curveCommand.EndTick; + return endTick > startTick; + case MergedSetCurveCommand curveCommand: + part = curveCommand.Part; + startTick = part.position + curveCommand.StartTick; + endTick = part.position + curveCommand.EndTick; + return endTick > startTick; + case ExpCommand expCommand when expCommand.Note != null: + part = expCommand.Part; + startTick = part.position + expCommand.Note.position; + endTick = part.position + expCommand.Note.End; + return endTick > startTick; + case ExpCommand expCommand: + part = expCommand.Part; + startTick = part.position; + endTick = part.End; + return endTick > startTick; + case PartCommand partCommand when partCommand.part is UVoicePart voicePart: + part = voicePart; + startTick = part.position; + endTick = part.End; + return endTick > startTick; + default: + if (cmd.ValidateOptions.Part is UVoicePart validatePart) { + part = validatePart; + startTick = part.position; + endTick = part.End; + return endTick > startTick; + } + return false; + } } public void RollBackUndoGroup() { @@ -314,7 +384,7 @@ public void Undo() { Publish(cmd, true); } redoQueue.AddToBack(group); - ExecuteCmd(new PreRenderNotification()); + ExecuteCmd(CreatePreRenderNotification(group.Commands)); } public void Redo() { @@ -331,7 +401,7 @@ public void Redo() { Publish(cmd); } undoQueue.AddToBack(group); - ExecuteCmd(new PreRenderNotification()); + ExecuteCmd(CreatePreRenderNotification(group.Commands)); } public bool GetUndoState(out string? key) { diff --git a/OpenUtau.Core/PlaybackManager.cs b/OpenUtau.Core/PlaybackManager.cs index 9bc67a6ed..a0183480f 100644 --- a/OpenUtau.Core/PlaybackManager.cs +++ b/OpenUtau.Core/PlaybackManager.cs @@ -373,9 +373,15 @@ private void CheckFileWritable(string filePath) { } } - void SchedulePreRender() { - Log.Information("SchedulePreRender"); - var engine = new RenderEngine(DocManager.Inst.Project); + void SchedulePreRender(PreRenderNotification? notification = null) { + Log.Information(notification?.HasPriorityRange == true + ? $"SchedulePreRender {notification.priorities.Length} prioritized range(s)" + : "SchedulePreRender"); + var engine = notification?.HasPriorityRange == true + ? new RenderEngine( + DocManager.Inst.Project, + priorityRanges: notification.priorities) + : new RenderEngine(DocManager.Inst.Project); engine.PreRenderProject(ref renderCancellation); } @@ -404,7 +410,7 @@ public void OnNext(UCommand cmd, bool isUndo) { } if (cmd is PreRenderNotification || cmd is LoadProjectNotification) { if (Util.Preferences.Default.PreRender) { - SchedulePreRender(); + SchedulePreRender(cmd as PreRenderNotification); } } } diff --git a/OpenUtau.Core/Properties/AssemblyInfo.cs b/OpenUtau.Core/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..a1f94316e --- /dev/null +++ b/OpenUtau.Core/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("OpenUtau.Test")] diff --git a/OpenUtau.Core/Render/RealCurveRefresh.cs b/OpenUtau.Core/Render/RealCurveRefresh.cs new file mode 100644 index 000000000..bee44d78f --- /dev/null +++ b/OpenUtau.Core/Render/RealCurveRefresh.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using OpenUtau.Core.DiffSinger; +using OpenUtau.Core.Ustx; +using OpenUtau.Core.Util; + +namespace OpenUtau.Core.Render { + public static class RealCurveRefresh { + public static bool CanRefresh(RenderPhrase phrase, bool allowSessionInitialization = true) { + if (phrase.renderer.SingerType != USingerType.DiffSinger || + !phrase.renderer.SupportsRealCurve || + !Preferences.Default.DiffSingerTensorCache) { + return false; + } + return true; + } + + public static List LoadRenderedRealCurves( + RenderPhrase phrase, + bool allowSessionInitialization = true) { + if (!CanRefresh(phrase, allowSessionInitialization)) { + return new List(0); + } + return phrase.renderer.LoadRenderedRealCurves(phrase); + } + + public static bool ApplyRealCurveResults( + UProject project, + UVoicePart part, + RenderPhrase phrase, + IEnumerable results) { + bool changed = false; + foreach (var result in results) { + changed |= ApplyRealCurveResult(project, part, phrase, result); + } + return changed; + } + + public static bool ApplyRealCurveResult( + UProject project, + UVoicePart part, + RenderPhrase phrase, + RenderRealCurveResult result) { + int count = Math.Min(result.ticks.Length, result.values.Length); + if (count == 0) { + return false; + } + var curve = part.curves.FirstOrDefault(curve => curve.abbr == result.abbr); + if (curve == null) { + if (!project.expressions.TryGetValue(result.abbr, out var descriptor)) { + return false; + } + curve = new UCurve(descriptor); + part.curves.Add(curve); + } + var ticks = result.ticks + .Take(count) + .Select(tick => phrase.position - part.position + (int)Math.Round(tick)) + .ToArray(); + var values = result.values + .Take(count) + .Select(value => (int)Math.Round(value * 1000.0)) + .ToArray(); + int rangeStart = ticks.First(); + int rangeEnd = ticks.Last(); + int index = curve.realXs.BinarySearch(rangeStart); + if (index < 0) { + index = ~index; + } + int removeCount = 0; + while (index + removeCount < curve.realXs.Count && curve.realXs[index + removeCount] <= rangeEnd) { + removeCount++; + } + if (removeCount > 0) { + curve.realXs.RemoveRange(index, removeCount); + curve.realYs.RemoveRange(index, removeCount); + } + var insertXs = new List(ticks.Length + 1) { rangeStart }; + var insertYs = new List(values.Length + 1) { -1 }; + insertXs.AddRange(ticks); + insertYs.AddRange(values); + curve.realXs.InsertRange(index, insertXs); + curve.realYs.InsertRange(index, insertYs); + return true; + } + } +} diff --git a/OpenUtau.Core/Render/RenderEngine.cs b/OpenUtau.Core/Render/RenderEngine.cs index b8bde5fa8..a2ceb4aa2 100644 --- a/OpenUtau.Core/Render/RenderEngine.cs +++ b/OpenUtau.Core/Render/RenderEngine.cs @@ -47,12 +47,30 @@ class RenderEngine { readonly int startTick; readonly int endTick; readonly int trackNo; + readonly PreRenderPriority[] priorityRanges; - public RenderEngine(UProject project, int startTick = 0, int endTick = -1, int trackNo = -1) { + public RenderEngine( + UProject project, + int startTick = 0, + int endTick = -1, + int trackNo = -1, + UVoicePart? priorityPart = null, + int priorityStartTick = -1, + int priorityEndTick = -1, + IEnumerable? priorityRanges = null) { this.project = project; this.startTick = startTick; this.endTick = endTick; this.trackNo = trackNo; + if (priorityRanges != null) { + this.priorityRanges = priorityRanges + .Where(priority => priority.endTick > priority.startTick) + .ToArray(); + } else if (priorityPart != null && priorityEndTick > priorityStartTick) { + this.priorityRanges = new[] { new PreRenderPriority(priorityPart, priorityStartTick, priorityEndTick) }; + } else { + this.priorityRanges = Array.Empty(); + } } // for playback or export @@ -220,35 +238,162 @@ private void RenderRequests( } var tuples = requests .SelectMany(req => req.phrases - .Zip(req.sources, (phrase, source) => Tuple.Create(phrase, source, req))) + .Zip(req.sources, (phrase, source) => (phrase, source, request: req))) .ToArray(); - if (playing) { - var orderedTuples = tuples - .Where(tuple => tuple.Item1.end > startTick) - .OrderBy(tuple => tuple.Item1.end) - .Concat(tuples.Where(tuple => tuple.Item1.end <= startTick)) - .ToArray(); - tuples = orderedTuples; + if (tuples.Length == 0) { + return; + } + if (tuples.Any(tuple => IsDiffSinger(tuple.phrase))) { + if (playing) { + tuples = OrderForPlayback(tuples); + } else if (GetDiffSingerPriorityRanges().Length > 0) { + tuples = OrderForPreRender(tuples); + } } var progress = new Progress(tuples.Sum(t => t.Item1.phones.Length)); foreach (var tuple in tuples) { - var phrase = tuple.Item1; - var source = tuple.Item2; - var request = tuple.Item3; + if (cancellation.IsCancellationRequested) { + break; + } + var phrase = tuple.phrase; + var source = tuple.source; + var request = tuple.request; var task = phrase.renderer.Render(phrase, progress, request.trackNo, cancellation, true); task.Wait(); if (cancellation.IsCancellationRequested) { break; } source.SetSamples(task.Result.samples); - if (request.sources.All(s => s.HasSamples)) { + bool partReady = request.sources.All(s => s.HasSamples); + if (IsDiffSinger(phrase)) { + RefreshRealCurves(phrase, request.part); request.part.SetMix(request.mix); + DocManager.Inst.ExecuteCmd(new PhraseRenderedNotification(request.part)); + } + if (partReady) { + if (!IsDiffSinger(phrase)) { + request.part.SetMix(request.mix); + } DocManager.Inst.ExecuteCmd(new PartRenderedNotification(request.part)); } } progress.Clear(); } + private (RenderPhrase phrase, WaveSource source, RenderPartRequest request)[] OrderForPlayback( + (RenderPhrase phrase, WaveSource source, RenderPartRequest request)[] tuples) { + double playbackStartMs = project.timeAxis.TickPosToMsPos(startTick); + return tuples + .Select((tuple, index) => (tuple, index)) + .OrderBy(item => RenderPriority.PlaybackBucket( + item.tuple.source.offsetMs, item.tuple.source.EndMs, playbackStartMs)) + .ThenBy(item => RenderPriority.PlaybackDistance( + item.tuple.source.offsetMs, item.tuple.source.EndMs, playbackStartMs)) + .ThenBy(item => item.index) + .Select(item => item.tuple) + .ToArray(); + } + + private (RenderPhrase phrase, WaveSource source, RenderPartRequest request)[] OrderForPreRender( + (RenderPhrase phrase, WaveSource source, RenderPartRequest request)[] tuples) { + var priorities = GetDiffSingerPriorityRanges(); + return tuples + .Select((tuple, index) => (tuple, index)) + .OrderBy(item => PreRenderPriorityBucket(item.tuple, priorities)) + .ThenBy(item => PreRenderPriorityIndex(item.tuple, priorities)) + .ThenBy(item => PreRenderPriorityDistance(item.tuple.phrase, priorities)) + .ThenBy(item => item.index) + .Select(item => item.tuple) + .ToArray(); + } + + private int PreRenderPriorityBucket( + (RenderPhrase phrase, WaveSource source, RenderPartRequest request) tuple, + PreRenderPriority[] priorities) { + bool isDiffSinger = IsDiffSinger(tuple.phrase); + bool isPriorityPart = priorities.Any(priority => ReferenceEquals(tuple.request.part, priority.part)); + bool overlapsPriority = priorities.Any(priority => + ReferenceEquals(tuple.request.part, priority.part) && + RenderPriority.Overlaps(tuple.phrase.position, tuple.phrase.end, priority.startTick, priority.endTick)); + int earliestPriorityStart = priorities.Min(priority => priority.startTick); + return RenderPriority.PreRenderBucket( + isDiffSinger, + isPriorityPart, + overlapsPriority, + tuple.phrase.end > earliestPriorityStart); + } + + private int PreRenderPriorityIndex( + (RenderPhrase phrase, WaveSource source, RenderPartRequest request) tuple, + PreRenderPriority[] priorities) { + for (int i = 0; i < priorities.Length; ++i) { + var priority = priorities[i]; + if (ReferenceEquals(tuple.request.part, priority.part) && + RenderPriority.Overlaps(tuple.phrase.position, tuple.phrase.end, priority.startTick, priority.endTick)) { + return i; + } + } + for (int i = 0; i < priorities.Length; ++i) { + if (ReferenceEquals(tuple.request.part, priorities[i].part)) { + return i; + } + } + return int.MaxValue; + } + + private int PreRenderPriorityDistance(RenderPhrase phrase, PreRenderPriority[] priorities) { + return priorities + .Select(priority => RenderPriority.PreRenderDistance(phrase.position, phrase.end, priority.startTick)) + .DefaultIfEmpty(0) + .Min(); + } + + private bool IsDiffSinger(RenderPhrase phrase) { + return phrase.renderer.SingerType == USingerType.DiffSinger; + } + + private void RefreshRealCurves(RenderPhrase phrase, UVoicePart part) { + if (!RealCurveRefresh.CanRefresh(phrase)) { + return; + } + List results; + try { + results = RealCurveRefresh.LoadRenderedRealCurves(phrase); + } catch (Exception e) { + Log.Debug(e, "Failed to refresh DiffSinger real curves."); + return; + } + if (results.Count == 0) { + return; + } + void Apply() { + lock (part) { + RealCurveRefresh.ApplyRealCurveResults(project, part, phrase, results); + } + DocManager.Inst.ExecuteCmd(new RealCurvesRenderedNotification(part)); + } + if (DocManager.Inst.PostOnUIThread != null) { + DocManager.Inst.PostOnUIThread(Apply); + } else { + Apply(); + } + } + + private PreRenderPriority[] GetDiffSingerPriorityRanges() { + return priorityRanges + .Where(priority => + priority.part.trackNo >= 0 && + priority.part.trackNo < project.tracks.Count && + project.tracks[priority.part.trackNo].RendererSettings.Renderer?.SingerType == USingerType.DiffSinger) + .ToArray(); + } + + private PreRenderPriority[] GetDiffSingerPriorityRanges(UVoicePart part) { + return GetDiffSingerPriorityRanges() + .Where(priority => ReferenceEquals(priority.part, part)) + .ToArray(); + } + public static void ReleaseSourceTemp() { VoicebankFiles.Inst.ReleaseSourceTemp(); } diff --git a/OpenUtau.Core/Render/RenderPriority.cs b/OpenUtau.Core/Render/RenderPriority.cs new file mode 100644 index 000000000..2f79f76ea --- /dev/null +++ b/OpenUtau.Core/Render/RenderPriority.cs @@ -0,0 +1,53 @@ +using System; + +namespace OpenUtau.Core.Render { + internal static class RenderPriority { + internal static int PlaybackBucket(double sourceStartMs, double sourceEndMs, double playbackStartMs) { + if (sourceStartMs <= playbackStartMs && sourceEndMs > playbackStartMs) { + return 0; + } + return sourceStartMs >= playbackStartMs ? 1 : 2; + } + + internal static double PlaybackDistance(double sourceStartMs, double sourceEndMs, double playbackStartMs) { + if (sourceStartMs <= playbackStartMs && sourceEndMs > playbackStartMs) { + return Math.Max(0, playbackStartMs - sourceStartMs); + } + if (sourceStartMs >= playbackStartMs) { + return sourceStartMs - playbackStartMs; + } + return playbackStartMs - sourceEndMs; + } + + internal static bool Overlaps(int startTick, int endTick, int priorityStartTick, int priorityEndTick) { + return endTick > priorityStartTick && startTick < priorityEndTick; + } + + internal static int PreRenderBucket( + bool isDiffSinger, + bool isPriorityPart, + bool overlapsPriority, + bool isAfterPriorityStart) { + if (isDiffSinger && isPriorityPart && overlapsPriority) { + return 0; + } + if (isDiffSinger && isPriorityPart) { + return 1; + } + if (isDiffSinger && isAfterPriorityStart) { + return 2; + } + return 3; + } + + internal static int PreRenderDistance(int phraseStartTick, int phraseEndTick, int priorityStartTick) { + if (phraseStartTick <= priorityStartTick && phraseEndTick > priorityStartTick) { + return 0; + } + if (phraseStartTick >= priorityStartTick) { + return phraseStartTick - priorityStartTick; + } + return priorityStartTick - phraseEndTick; + } + } +} diff --git a/OpenUtau.Test/Core/DiffSinger/DiffSingerVariancePatcherTest.cs b/OpenUtau.Test/Core/DiffSinger/DiffSingerVariancePatcherTest.cs new file mode 100644 index 000000000..6dcad3908 --- /dev/null +++ b/OpenUtau.Test/Core/DiffSinger/DiffSingerVariancePatcherTest.cs @@ -0,0 +1,87 @@ +using System.Linq; +using Xunit; + +namespace OpenUtau.Core.DiffSinger { + public class DiffSingerVariancePatcherTest { + [Fact] + public void HashPitchChangesWhenPitchChanges() { + var a = DiffSingerVariancePatcher.HashPitch(new[] { 1f, 2f, 3f }); + var b = DiffSingerVariancePatcher.HashPitch(new[] { 1f, 2.1f, 3f }); + + Assert.NotEqual(a, b); + } + + [Fact] + public void BuildWeightsForMsRangesKeepsOutsideRangeAtZero() { + var weights = DiffSingerVariancePatcher.BuildWeightsForMsRanges( + totalFrames: 100, + frameMs: 10, + startMs: 0, + ranges: new[] { (startMs: 500.0, endMs: 600.0) }); + + Assert.Equal(0, weights[0]); + Assert.Equal(0, weights[99]); + Assert.Contains(weights, weight => weight >= 1); + } + + [Fact] + public void BuildWeightsForMsRangesCrossfadesEdges() { + var weights = DiffSingerVariancePatcher.BuildWeightsForMsRanges( + totalFrames: 100, + frameMs: 10, + startMs: 0, + ranges: new[] { (startMs: 500.0, endMs: 600.0) }); + + Assert.Contains(weights, weight => weight > 0 && weight < 1); + } + + [Fact] + public void BuildWeightsForPitchChangesKeepsUnchangedEdgesAtZero() { + var oldPitch = Enumerable.Repeat(1f, 100).ToArray(); + var newPitch = Enumerable.Repeat(1f, 100).ToArray(); + newPitch[50] = 2f; + + var weights = DiffSingerVariancePatcher.BuildWeightsForPitchChanges( + oldPitch, newPitch, frameMs: 10); + + Assert.Equal(0, weights[0]); + Assert.Equal(0, weights[99]); + Assert.Contains(weights, weight => weight >= 1); + } + + [Fact] + public void PatchArrayBlendsOldAndNewByWeight() { + var oldValues = Enumerable.Repeat(0f, 5).ToArray(); + var newValues = Enumerable.Repeat(10f, 5).ToArray(); + var weights = new[] { 0f, 0.25f, 0.5f, 0.75f, 1f }; + + var result = DiffSingerVariancePatcher.PatchArray(oldValues, newValues, weights)!; + + Assert.Equal(new[] { 0f, 2.5f, 5f, 7.5f, 10f }, result); + } + + [Fact] + public void PatchByPitchChangeKeepsUnchangedFramesFromOldResult() { + var oldPitch = Enumerable.Repeat(1f, 100).ToArray(); + var newPitch = Enumerable.Repeat(1f, 100).ToArray(); + newPitch[50] = 2f; + var oldResult = new VarianceResult { + energy = Enumerable.Repeat(0f, 100).ToArray(), + frameMs = 10, + totalFrames = 100, + }; + var newResult = new VarianceResult { + energy = Enumerable.Repeat(10f, 100).ToArray(), + frameMs = 10, + totalFrames = 100, + }; + + var result = DiffSingerVariancePatcher.PatchByPitchChange( + oldResult, newResult, oldPitch, newPitch); + + Assert.Equal(0, result.energy![0]); + Assert.Equal(10, result.energy[50]); + Assert.Equal(0, result.energy[99]); + } + } +} diff --git a/OpenUtau.Test/Core/Render/RenderPriorityTest.cs b/OpenUtau.Test/Core/Render/RenderPriorityTest.cs new file mode 100644 index 000000000..55cac4b1e --- /dev/null +++ b/OpenUtau.Test/Core/Render/RenderPriorityTest.cs @@ -0,0 +1,90 @@ +using System.Linq; +using OpenUtau.Core.Ustx; +using Xunit; + +namespace OpenUtau.Core.Render { + public class RenderPriorityTest { + [Fact] + public void PlaybackPriorityPrefersSourceCoveringStart() { + var ordered = new[] { + ("future", 120.0, 160.0), + ("past", 0.0, 90.0), + ("current", 80.0, 150.0), + } + .OrderBy(item => RenderPriority.PlaybackBucket(item.Item2, item.Item3, 100)) + .ThenBy(item => RenderPriority.PlaybackDistance(item.Item2, item.Item3, 100)) + .Select(item => item.Item1) + .ToArray(); + + Assert.Equal(new[] { "current", "future", "past" }, ordered); + } + + [Fact] + public void PlaybackPriorityUsesDistanceWithinBucket() { + var ordered = new[] { + ("far", 300.0, 360.0), + ("near", 120.0, 180.0), + } + .OrderBy(item => RenderPriority.PlaybackBucket(item.Item2, item.Item3, 100)) + .ThenBy(item => RenderPriority.PlaybackDistance(item.Item2, item.Item3, 100)) + .Select(item => item.Item1) + .ToArray(); + + Assert.Equal(new[] { "near", "far" }, ordered); + } + + [Fact] + public void PreRenderPriorityBucketsEditedDiffSingerPhraseFirst() { + Assert.Equal(0, RenderPriority.PreRenderBucket( + isDiffSinger: true, + isPriorityPart: true, + overlapsPriority: true, + isAfterPriorityStart: true)); + Assert.Equal(1, RenderPriority.PreRenderBucket( + isDiffSinger: true, + isPriorityPart: true, + overlapsPriority: false, + isAfterPriorityStart: true)); + Assert.Equal(2, RenderPriority.PreRenderBucket( + isDiffSinger: true, + isPriorityPart: false, + overlapsPriority: false, + isAfterPriorityStart: true)); + Assert.Equal(3, RenderPriority.PreRenderBucket( + isDiffSinger: false, + isPriorityPart: true, + overlapsPriority: true, + isAfterPriorityStart: true)); + } + + [Fact] + public void PreRenderDistanceIsZeroWhenPhraseCoversPriorityStart() { + Assert.Equal(0, RenderPriority.PreRenderDistance(80, 160, 100)); + Assert.Equal(20, RenderPriority.PreRenderDistance(120, 160, 100)); + Assert.Equal(10, RenderPriority.PreRenderDistance(40, 90, 100)); + } + + [Fact] + public void OverlapsUsesHalfOpenTickRanges() { + Assert.True(RenderPriority.Overlaps(80, 120, 100, 160)); + Assert.False(RenderPriority.Overlaps(80, 100, 100, 160)); + Assert.False(RenderPriority.Overlaps(160, 200, 100, 160)); + } + + [Fact] + public void PreRenderNotificationKeepsMultiplePriorityRanges() { + var part1 = new UVoicePart(); + var part2 = new UVoicePart(); + var notification = new PreRenderNotification(new[] { + new PreRenderPriority(part1, 100, 200), + new PreRenderPriority(part2, 300, 400), + new PreRenderPriority(part2, 500, 500), + }); + + Assert.Equal(2, notification.priorities.Length); + Assert.Same(part1, notification.priorityPart); + Assert.Equal(100, notification.priorityStartTick); + Assert.Equal(200, notification.priorityEndTick); + } + } +} diff --git a/OpenUtau/ViewModels/NotesViewModel.cs b/OpenUtau/ViewModels/NotesViewModel.cs index 1d4d4f582..24b324df9 100644 --- a/OpenUtau/ViewModels/NotesViewModel.cs +++ b/OpenUtau/ViewModels/NotesViewModel.cs @@ -104,7 +104,6 @@ public class NotesViewModel : ViewModelBase, ICmdSubscriber { // See the comments on TracksViewModel.playPosXToTickOffset private double playPosXToTickOffset => Bounds.Width != 0 ? ViewportTicks / Bounds.Width : 0; - private readonly ObservableAsPropertyHelper viewportTicks; private readonly ObservableAsPropertyHelper viewportTracks; private readonly ObservableAsPropertyHelper smallChangeX; @@ -1101,7 +1100,9 @@ public void OnNext(UCommand cmd, bool isUndo) { } else if (cmd is PhonemizedNotification) { OnPartModified(); MessageBus.Current.SendMessage(new NotesRefreshEvent()); - } else if (notif is PartRenderedNotification && notif.part == Part) { + } else if (notif is RealCurvesRenderedNotification && notif.part == Part) { + MessageBus.Current.SendMessage(new NotesRefreshEvent()); + } else if ((notif is PhraseRenderedNotification || notif is PartRenderedNotification) && notif.part == Part) { MessageBus.Current.SendMessage(new WaveformRefreshEvent()); } } else if (cmd is PartCommand partCommand) {