From 6412a11846f3a650ed0865a87dc2eca4a320f6df Mon Sep 17 00:00:00 2001 From: Toys0125 Date: Fri, 6 Mar 2026 21:51:55 -0600 Subject: [PATCH 1/5] fix(face-tracking): make host authoritative for activity state and neutralize startup sync * Added an internal `FaceTrackingActivityRelay` channel (`HVR/Internal/FaceTrackingActive`) with a host-only inactivity timeout (0.5s). Remotes now follow host activity and do not run their own timeout logic. * Wired OSC input to activity updates so any incoming face packet marks tracking active. * Fixed remote eye application in `EyeTrackingBoneActuation` (!IsLocal && Receiver != null) and ensured remote override flags are cleared when tracking is inactive/disabled. * Gated eye and blendshape actuation behind propagated tracking activity: Active: HVR drives local/remote face data. Inactive: reset HVR-driven values to neutral and return control to Basis default behavior. * Stopped applying startup/default blendshape overrides when tracking is inactive; defaults are only pushed while tracking is active. * Changed mutualized feature initialization to neutral normalized values based on interpolation ranges (AbsoluteToRange(0) when 0 is in range), preventing signed channels from booting at low-end values (e.g. unintended TongueLeft). * Updated networking init flow to seed neutral values before initial replay, and fixed wearer/range bookkeeping issues in `HVRAvatarComms`. --- .../com.basis.eventdriver/BasisEventDriver.cs | 5 +- .../Networking/BasisRemoteFaceManagement.cs | 2 +- .../Shims/BasisCilboxBuildHook.cs | 2 +- Basis/Packages/com.cnlohr.cilbox/Cilbox.cs | 295 +++++++++++++----- .../Packages/com.cnlohr.cilbox/CilboxUtil.cs | 11 + .../com.hecomi.ulipsync/BasisUlipSync.cs | 61 ++-- .../Runtime/SteamAudioManager.cs | 24 +- .../Components/Acquisition/OSCAcquisition.cs | 5 +- .../Actuation/BlendshapeActuation.cs | 186 +++++++++-- .../Actuation/EyeTrackingBoneActuation.cs | 198 +++++++++--- .../FaceTracking/AutomaticFaceTracking.cs | 6 +- .../FaceTracking/FaceTrackingActivityRelay.cs | 1 + .../FaceTrackingActivityRelay.cs.meta | 2 + .../Components/Networking/HVRAvatarComms.cs | 23 +- .../Networking/StreamedAvatarFeature.cs | 26 +- .../Runtime/Networking/HVRCommsUtil.cs | 134 ++++++++ 16 files changed, 801 insertions(+), 180 deletions(-) create mode 100644 Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs create mode 100644 Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs.meta diff --git a/Basis/Packages/com.basis.eventdriver/BasisEventDriver.cs b/Basis/Packages/com.basis.eventdriver/BasisEventDriver.cs index 4a322bf24..15e6af9d0 100644 --- a/Basis/Packages/com.basis.eventdriver/BasisEventDriver.cs +++ b/Basis/Packages/com.basis.eventdriver/BasisEventDriver.cs @@ -230,10 +230,12 @@ public void LateUpdate() /// private void OnBeforeRender() { + // Ensure remote face jobs are completed/applied even before local player is ready. + BasisRemoteFaceManagement.Apply(); //apply blendshapes + if (BasisLocalPlayer.PlayerReady) { BasisLocalPlayer.Instance.SimulateOnRender(); - BasisRemoteFaceManagement.Apply(); //apply blendshapes BasisLocalCameraDriver.Instance.microphoneIconDriver.Simulate(DeltaTime); //update microphone icon } } @@ -275,3 +277,4 @@ public void OnDrawGizmosSelected() #endif } } + diff --git a/Basis/Packages/com.basis.framework/Networking/BasisRemoteFaceManagement.cs b/Basis/Packages/com.basis.framework/Networking/BasisRemoteFaceManagement.cs index cbc2c1285..502abf207 100644 --- a/Basis/Packages/com.basis.framework/Networking/BasisRemoteFaceManagement.cs +++ b/Basis/Packages/com.basis.framework/Networking/BasisRemoteFaceManagement.cs @@ -90,7 +90,7 @@ public static void Simulate(double t,float dt) blinkOut = blinkOut, }; - handle = job.Schedule(count, BatchSize); + handle = job.Schedule(count, BatchSize, handle); } public static void Apply() diff --git a/Basis/Packages/com.basis.shim/Shims/BasisCilboxBuildHook.cs b/Basis/Packages/com.basis.shim/Shims/BasisCilboxBuildHook.cs index 8d1920c20..ca9c81f20 100644 --- a/Basis/Packages/com.basis.shim/Shims/BasisCilboxBuildHook.cs +++ b/Basis/Packages/com.basis.shim/Shims/BasisCilboxBuildHook.cs @@ -82,7 +82,7 @@ private static Task HandlePreBuildBundle(BasisContentBase basisContentBase, List return Task.CompletedTask; } - CilboxScenePostprocessor.OnPostprocessScene(); + CilboxScenePostprocessor.OnPostprocessScene(temporaryScene); EnsureTemporarySceneHasAssemblyData(temporarySceneCilbox, cilboxAssemblySnapshot); RebindProxiesToTemporarySceneCilbox(contentRoot, temporarySceneCilbox); RestoreExternalCilboxAssemblyData(cilboxAssemblySnapshot, temporaryScene); diff --git a/Basis/Packages/com.cnlohr.cilbox/Cilbox.cs b/Basis/Packages/com.cnlohr.cilbox/Cilbox.cs index 244454a30..b2d58a81e 100644 --- a/Basis/Packages/com.cnlohr.cilbox/Cilbox.cs +++ b/Basis/Packages/com.cnlohr.cilbox/Cilbox.cs @@ -38,6 +38,13 @@ public class CilboxExceptionHandlingClause public string? CatchTypeName; } + public class CilboxHeapInstance + { + public string className; + public CilboxClass cls; + public StackElement[] fields; + } + public class CilboxMethod { public CilboxClass parentClass; @@ -46,6 +53,7 @@ public class CilboxMethod public String fullSignature; public String[] methodLocals; public bool isStatic; + public bool isConstructor; public Type[] typeLocals; public byte[] byteCode; public bool isVoid; @@ -83,6 +91,15 @@ public void Load( CilboxClass cclass, String name, Serializee payload ) isVoid = Convert.ToInt32((methodProps["isVoid"].AsString())) != 0; isStatic = Convert.ToInt32((methodProps["isStatic"].AsString())) != 0; fullSignature = methodProps["fullSignature"].AsString(); + if( methodProps.TryGetValue("isCtor", out Serializee isCtorSer) ) + { + isConstructor = Convert.ToInt32(isCtorSer.AsString()) != 0; + } + else + { + // Backward compatibility for payloads generated before isCtor existed. + isConstructor = methodName == ".ctor" || methodName == ".cctor" || fullSignature.StartsWith("Void .ctor(") || fullSignature.StartsWith("Void .cctor("); + } Serializee [] od = methodProps["parameters"].AsArray(); signatureParameters = new String[od.Length]; @@ -317,11 +334,14 @@ private StackElement InterpretInner( ArraySegment stackBufferIn, A case 0x73: //newobj case 0x6F: //callvirt { + int currentInstruction = pc - 1; uint bc = (b == 0x29) ? stackBuffer[sp--].u : BytecodeAsU32( ref pc ); object iko = null; // Returned value. CilMetadataTokenInfo dt = box.metadatas[bc]; bool isVoid = false; MethodBase st; + bool isNewObj = b == 0x73; + bool isJmp = b == 0x27; if( !dt.isValid ) { @@ -347,7 +367,6 @@ private StackElement InterpretInner( ArraySegment stackBufferIn, A stackBuffer[++sp] = dt.shim( dt, stackBufferIn.Slice( nextStackHead ), stackBufferIn.Slice( nextParameterStart, numParams + staticOffset ) ); else dt.shim( dt, stackBufferIn.Slice( nextStackHead ), stackBufferIn.Slice( nextParameterStart, numParams + staticOffset ) ); - } else { @@ -355,31 +374,90 @@ private StackElement InterpretInner( ArraySegment stackBufferIn, A // interpretiveMethodClass CilboxClass targetClass = box.classesList[dt.interpretiveMethodClass]; CilboxMethod targetMethod = targetClass.methods[dt.interpretiveMethod]; - isVoid = targetMethod.isVoid; if( targetMethod == null ) throw new CilboxInterpreterRuntimeException($"Function {dt.Name} not found", parentClass.className, methodName, pc); + isVoid = targetMethod.isVoid; int staticOffset = (targetMethod.isStatic?0:1); int numParams = targetMethod.signatureParameters.Length; - int nextParameterStart = stackContinues; int nextStackHead = nextParameterStart + numParams + staticOffset; for( int i = numParams - 1; i >= 0; i-- ) stackBuffer[nextParameterStart+i+staticOffset] = stackBuffer[sp--]; - if( !targetMethod.isStatic ) - stackBuffer[nextParameterStart] = stackBuffer[sp--]; + bool ctorAsNewObj = targetMethod.isConstructor && isNewObj; + bool ctorAsCall = targetMethod.isConstructor && !isNewObj; - if( !isVoid ) - stackBuffer[++sp] = targetMethod.InterpretInner( stackBufferIn.Slice( nextStackHead ), stackBufferIn.Slice( nextParameterStart, numParams + staticOffset ) ); + if( ctorAsNewObj ) + { + CilboxHeapInstance newObj = new CilboxHeapInstance(); + newObj.className = targetClass.className; + newObj.cls = targetClass; + newObj.fields = new StackElement[targetClass.instanceFieldNames.Length]; + for( int i = 0; i < targetClass.instanceFieldNames.Length; i++ ) + { + Type fieldType = targetClass.instanceFieldTypes[i]; + if( fieldType == null ) + { + newObj.fields[i].LoadObject( null ); + continue; + } + + StackType fieldStackType = StackElement.StackTypeFromType( fieldType ); + if( fieldStackType < StackType.Object ) + { + newObj.fields[i].type = fieldStackType; + } + else if( fieldType.IsValueType ) + { + try + { + newObj.fields[i].LoadObject( Activator.CreateInstance( fieldType ) ); + } + catch + { + newObj.fields[i].LoadObject( null ); + } + } + else + { + newObj.fields[i].LoadObject( null ); + } + } + stackBuffer[nextParameterStart].LoadObject( newObj ); + try + { + targetMethod.InterpretInner(stackBufferIn.Slice(nextStackHead), stackBufferIn.Slice(nextParameterStart, numParams + staticOffset)); + } + catch (CilboxUnhandledInterpretedException e) + { + interpretedThrow(currentInstruction, e.Throwee); + } + stackBuffer[++sp].LoadObject( newObj ); + } else - targetMethod.InterpretInner( stackBufferIn.Slice( nextStackHead ), stackBufferIn.Slice( nextParameterStart, numParams + staticOffset ) ); + { + if( !targetMethod.isStatic ) + stackBuffer[nextParameterStart] = stackBuffer[sp--]; - if( b == 0x27 ) + try + { + if (!isVoid && !ctorAsCall) + stackBuffer[++sp] = targetMethod.InterpretInner(stackBufferIn.Slice(nextStackHead), stackBufferIn.Slice(nextParameterStart, numParams + staticOffset)); + else + targetMethod.InterpretInner(stackBufferIn.Slice(nextStackHead), stackBufferIn.Slice(nextParameterStart, numParams + staticOffset)); + } + catch (CilboxUnhandledInterpretedException e) + { + interpretedThrow(currentInstruction, e.Throwee); + } + } + + if( isJmp ) { // This is returning from a jump, so immediately abort. - if( isVoid ) stackBuffer[++sp] = StackElement.nil; /// ?? Please check me! If wrong, fix above, too. + if( isVoid || ctorAsCall ) stackBuffer[++sp] = StackElement.nil; /// ?? Please check me! If wrong, fix above, too. cont = false; } } @@ -394,7 +472,6 @@ private StackElement InterpretInner( ArraySegment stackBufferIn, A int numFields = pa.Length; object callthis = null; object [] callpar = new object[numFields]; - StackElement callthis_se = new StackElement{}; StackElement [] callpar_se = new StackElement[numFields]; int ik; for( ik = 0; ik < numFields; ik++ ) @@ -429,11 +506,33 @@ private StackElement InterpretInner( ArraySegment stackBufferIn, A } if( st.IsConstructor ) { - // TRICKY: This generally can only be arrived at when scripts run their own constructors. - if( st.DeclaringType == typeof( MonoBehaviour ) ) - iko = this; - else // Otherwise it's normal. - iko = ((ConstructorInfo)st).Invoke( callpar ); + ConstructorInfo ctor = (ConstructorInfo)st; + if( isNewObj ) + { + iko = ctor.Invoke( callpar ); + isVoid = false; // newobj always pushes a reference/value. + } + else + { + StackElement ctorThisSe = stackBuffer[sp--]; + object ctorThis = ctorThisSe.AsObject(box); + if (ctorThis == null) + { + interpretedThrow(pc - 1, new NullReferenceException()); + break; + } + + Type ctorDeclaringType = ctor.DeclaringType; + if( ctorDeclaringType == null || ( ctorDeclaringType != typeof(object) && ctorDeclaringType != typeof(MonoBehaviour) ) ) + { + throw new CilboxInterpreterRuntimeException( + $"Unsupported native constructor call on existing instance: {ctor.DeclaringType?.FullName}", + parentClass.className, methodName, pc); + } + + // Base constructors for Object/MonoBehaviour are no-ops in interpreter mode. + isVoid = true; + } } else if( !st.IsStatic ) { @@ -503,7 +602,7 @@ private StackElement InterpretInner( ArraySegment stackBufferIn, A { stackBuffer[++sp].Load( iko ); } - if( b == 0x27 ) + if( isJmp ) { // This is returning from a jump, so immediately abort. if( isVoid ) stackBuffer[++sp] = StackElement.nil; /// ?? Please check me! If wrong, fix above, too. @@ -870,20 +969,19 @@ private StackElement InterpretInner( ArraySegment stackBufferIn, A object oRet = null; if( ti.nativeTypeIsCilboxProxy ) { - if( se.o is CilboxProxy ) + if( TryGetInternalObjectData( se.o, out string seClassName, out _ ) ) { - // Both are proxies. Check name. - if( ((CilboxProxy)(se.o)).className == ti.Name ) + if( seClassName == ti.Name ) oRet = se.o; } } else if( ti.nativeTypeIsStackType ) { - if( ti.nativeTypeStackType == StackElement.TypeToStackType[ti.Name] ) - stackBuffer[++sp] = se; + if( StackElement.TypeToStackType.TryGetValue( ti.Name, out StackType stackType ) && ti.nativeTypeStackType == stackType ) + oRet = se.AsObject(); } - else if( se.o.GetType() == ti.nativeType ) - stackBuffer[++sp].LoadObject( se.o ); + else if( se.o != null && se.o.GetType() == ti.nativeType ) + oRet = se.o; stackBuffer[++sp].LoadObject( oRet ); @@ -912,9 +1010,9 @@ private StackElement InterpretInner( ArraySegment stackBufferIn, A break; } - if( opths is CilboxProxy proxy ) + if( TryGetInternalObjectData( opths, out _, out StackElement[] internalFields ) ) { - stackBuffer[++sp] = proxy.fields[box.metadatas[bc].fieldIndex]; + stackBuffer[++sp] = internalFields[box.metadatas[bc].fieldIndex]; break; } @@ -943,9 +1041,9 @@ private StackElement InterpretInner( ArraySegment stackBufferIn, A break; } - if( opths is CilboxProxy proxy ) + if( TryGetInternalObjectData( opths, out _, out StackElement[] internalFields ) ) { - stackBuffer[++sp] = StackElement.CreateAddressReference((Array)(proxy.fields), (uint)box.metadatas[bc].fieldIndex); + stackBuffer[++sp] = StackElement.CreateAddressReference((Array)(internalFields), (uint)box.metadatas[bc].fieldIndex); break; } @@ -963,9 +1061,9 @@ private StackElement InterpretInner( ArraySegment stackBufferIn, A break; } - if( opths is CilboxProxy proxy ) + if( TryGetInternalObjectData( opths, out _, out StackElement[] internalFields ) ) { - proxy.fields[box.metadatas[bc].fieldIndex] = se; + internalFields[box.metadatas[bc].fieldIndex] = se; //Debug.Log( "Type: " + ((CilboxProxy)opths).fields[box.metadatas[bc].fieldIndex].type ); break; } @@ -1435,6 +1533,11 @@ private StackElement InterpretInner( ArraySegment stackBufferIn, A } while( cont ); } + catch (CilboxUnhandledInterpretedException) + { + // don't break program flow for interpreted exceptions; we want to pass flow back to the outer call. + throw; + } catch( Exception e ) { string fullError = $"Breakwarn: {e.ToString()} Class: {parentClass.className}, Function: {methodName}, Bytecode: {pc}"; @@ -1464,8 +1567,7 @@ void interpretedThrow(int currentInstruction, object thrownObj) exceptionRegister = new StackElement() { type = StackType.Object, o = thrownObj }; if (!hasExceptionClauses) { - // todo: figure out how to re-throw to outer interpreter. - throw new CilboxInterpreterRuntimeException("Exception thrown with no handlers: " + thrownObj.ToString(), parentClass.className, methodName, currentInstruction); + throw new CilboxUnhandledInterpretedException("Exception thrown with no handlers: " + thrownObj.ToString(), thrownObj, parentClass.className, methodName, currentInstruction); } CilboxExceptionHandlingClause found = null; @@ -1504,7 +1606,7 @@ void interpretedThrow(int currentInstruction, object thrownObj) { // Cilboxable type match // todo: it isn't actually possible to throw a Cilboxable type (yet?) - if (!(thrownObj is CilboxProxy && ((CilboxProxy)thrownObj).className == c.CatchTypeName)) + if (!IsInternalObjectInstanceOf(thrownObj, c.CatchTypeName)) { continue; } @@ -1520,8 +1622,7 @@ void interpretedThrow(int currentInstruction, object thrownObj) if (found == null) { - // how do I handle this? - throw new CilboxInterpreterRuntimeException("No handlers matched exception: " + thrownObj.ToString(), parentClass.className, methodName, currentInstruction); + throw new CilboxUnhandledInterpretedException("No handlers matched exception: " + thrownObj.ToString(), thrownObj, parentClass.className, methodName, currentInstruction); } leaveRegionEnqueueFinallys(currentInstruction, found.HandlerOffset, true); @@ -1593,6 +1694,32 @@ void jumpToNextHandlerDestination() } } + private static bool TryGetInternalObjectData( object candidate, out string className, out StackElement[] fields ) + { + if( candidate is CilboxProxy proxy ) + { + className = proxy.className; + fields = proxy.fields; + return true; + } + if( candidate is CilboxHeapInstance heap ) + { + className = heap.className; + fields = heap.fields; + return true; + } + + className = string.Empty; + fields = Array.Empty(); + return false; + } + + private static bool IsInternalObjectInstanceOf( object candidate, string className ) + { + return TryGetInternalObjectData( candidate, out string candidateClassName, out _ ) && candidateClassName == className; + } + + uint BytecodeAs16( ref int i ) { return (uint)CilboxUtil.BytecodePullLiteral( byteCode, ref i, 2 ); @@ -2226,7 +2353,7 @@ class CilboxCustomBuildProcessor : IProcessSceneWithReport public void OnProcessScene( UnityEngine.SceneManagement.Scene scene, UnityEditor.Build.Reporting.BuildReport report) { //Debug.Log( "IProcessSceneWithReport" ); - CilboxScenePostprocessor.OnPostprocessScene(); + CilboxScenePostprocessor.OnPostprocessScene(scene); } } @@ -2235,6 +2362,7 @@ class CilboxCustomBuildProcessor2 : IPreprocessBuildWithReport public int callbackOrder { get { return 0; } } public void OnPreprocessBuild(UnityEditor.Build.Reporting.BuildReport report) { + UnityEngine.SceneManagement.Scene activeScene = UnityEngine.SceneManagement.SceneManager.GetActiveScene(); MonoBehaviour [] allBehavioursThatNeedCilboxing = CilboxUtil.GetAllBehavioursThatNeedCilboxing(); if( allBehavioursThatNeedCilboxing.Length == 0 ) @@ -2249,13 +2377,13 @@ public void OnPreprocessBuild(UnityEditor.Build.Reporting.BuildReport report) dirtier.hideFlags = HideFlags.HideInHierarchy; dirtier.transform.position = new Vector3(UnityEngine.Random.Range(-100,100),UnityEngine.Random.Range(-100,100),UnityEngine.Random.Range(-100,100)); UnityEditor.SceneManagement.EditorSceneManager.MarkSceneDirty( - UnityEngine.SceneManagement.SceneManager.GetActiveScene() ); - UnityEditor.SceneManagement.EditorSceneManager.SaveScene( UnityEngine.SceneManagement.SceneManager.GetActiveScene() ); + activeScene ); + UnityEditor.SceneManagement.EditorSceneManager.SaveScene( activeScene ); } } public class CilboxScenePostprocessor { //[PostProcessSceneAttribute (2)] This is actually called by IProcessSceneWithReport - public static void OnPostprocessScene() { + public static void OnPostprocessScene(UnityEngine.SceneManagement.Scene scene) { ProfilerMarker perf = new ProfilerMarker("Initial Setup"); perf.Begin(); @@ -2267,7 +2395,9 @@ public static void OnPostprocessScene() { Dictionary< String, Serializee > assemblyMetadata = new Dictionary< String, Serializee >(); Dictionary< uint, String > originalMetaToFriendlyName = new Dictionary< uint, String >(); - Dictionary< int, uint> assemblyMetadataReverseOriginal = new Dictionary< int, uint >(); + // This is used for remapping tokens in the bytecode to point to our own metadata. + // Since tokens are per-assembly, we need prevent collisions by keying per assembly as well. + Dictionary< (System.Reflection.Assembly, int), uint> assemblyMetadataReverseOriginal = new Dictionary< (System.Reflection.Assembly, int), uint >(); uint mdcount = 1; // token 0 is invalid. int bytecodeLength = 0; @@ -2276,15 +2406,16 @@ public static void OnPostprocessScene() { perf.End(); perf = new ProfilerMarker( "Main Getting Types" ); perf.Begin(); - // Make sure the cilbox script is in use in the scene. - HashSet TypesInUseInScene = null; + // Make sure the cilbox script is in use in the scene or we have no scene loaded. + List TypesInUseInSceneList = new List(); + HashSet TypesInUseInScene = new HashSet();; - UnityEngine.SceneManagement.Scene activeScene = UnityEngine.SceneManagement.SceneManager.GetActiveScene(); - if( activeScene != null ) + System.Reflection.Assembly [] assys = AppDomain.CurrentDomain.GetAssemblies(); + + if( scene != null ) { - TypesInUseInScene = new HashSet(); - GameObject[] rootObjects = activeScene.GetRootGameObjects(); + GameObject[] rootObjects = scene.GetRootGameObjects(); foreach (GameObject root in rootObjects) { MonoBehaviour[] components = root.GetComponentsInChildren(true); @@ -2296,7 +2427,8 @@ public static void OnPostprocessScene() { Type t = component.GetType(); if( !TypesInUseInScene.Contains( t ) ) { - TypesInUseInScene.Add( t); + TypesInUseInScene.Add( t ); + TypesInUseInSceneList.Add( t ); } } } @@ -2304,24 +2436,32 @@ public static void OnPostprocessScene() { } else { - Debug.LogWarning( "No scene loaded. Converting ALL Cilboxable scripts." ); + // Collect ALL cilboxable classes if no scene active. + + foreach( System.Reflection.Assembly proxyAssembly in assys ) + { + foreach (Type t in proxyAssembly.GetTypes()) + { + if( CilboxUtil.HasCilboxableAttribute( t ) && !TypesInUseInScene.Contains( t ) ) + { + TypesInUseInScene.Add( t ); + TypesInUseInSceneList.Add( t ); + } + } + } } - System.Reflection.Assembly [] assys = AppDomain.CurrentDomain.GetAssemblies(); - foreach( System.Reflection.Assembly proxyAssembly in assys ) { - assemblyMetadataReverseOriginal.Clear(); - - foreach (Type type in proxyAssembly.GetTypes()) + for( int typeIndex = 0; typeIndex < TypesInUseInSceneList.Count; typeIndex++ ) { + Type type = TypesInUseInSceneList[typeIndex]; + System.Reflection.Assembly proxyAssembly = type.Assembly; + if( !CilboxUtil.HasCilboxableAttribute( type ) ) continue; if( type.IsEnum ) continue; - // Cilbox is not in use... But do ALL cilboxes if no scene is loaded. - if( TypesInUseInScene != null && !TypesInUseInScene.Contains( type ) ) continue; - ProfilerMarker perfType = new ProfilerMarker(type.ToString()); perfType.Begin(); Dictionary< String, Serializee > methods = new Dictionary< String, Serializee >(); @@ -2408,7 +2548,7 @@ public static void OnPostprocessScene() { switch( operand>>24 ) { case 0x04: // Special case handling for constant initializers. - if( !assemblyMetadataReverseOriginal.TryGetValue( (int)operand, out writebackToken ) ) + if( !assemblyMetadataReverseOriginal.TryGetValue( (proxyAssembly, (int)operand), out writebackToken ) ) { writebackToken = mdcount; // Special +__StaticArrayInitTypeSize=24 instance. @@ -2428,7 +2568,7 @@ public static void OnPostprocessScene() { break; /* case 0x02: // Inline Token for Type (typically used with typeof()) - if( !assemblyMetadataReverseOriginal.TryGetValue( (int)operand, out writebackToken ) ) + if( !assemblyMetadataReverseOriginal.TryGetValue( (proxyAssembly, (int)operand), out writebackToken ) ) { // TODO: Actually investigate this. See if we really need it. writebackToken = mdcount; @@ -2452,7 +2592,7 @@ public static void OnPostprocessScene() { } else if( ot == CilboxUtil.OpCodes.OperandType.InlineString ) { - if( !assemblyMetadataReverseOriginal.TryGetValue( (int)operand, out writebackToken ) ) + if( !assemblyMetadataReverseOriginal.TryGetValue( (proxyAssembly, (int)operand), out writebackToken ) ) { writebackToken = mdcount; Dictionary< String, String > thisMeta = new Dictionary< String, String >(); @@ -2465,7 +2605,7 @@ public static void OnPostprocessScene() { } else if( ot == CilboxUtil.OpCodes.OperandType.InlineMethod ) { - if( !assemblyMetadataReverseOriginal.TryGetValue( (int)operand, out writebackToken ) ) + if( !assemblyMetadataReverseOriginal.TryGetValue( (proxyAssembly, (int)operand), out writebackToken ) ) { writebackToken = mdcount; MethodBase tmb = proxyAssembly.ManifestModule.ResolveMethod( (int)operand ); @@ -2485,6 +2625,14 @@ public static void OnPostprocessScene() { } } + // If we are using another type here, make sure it gets in our list. + // We only need to do this here, because either the script is in-use or we are creating it. + if( !TypesInUseInScene.Contains( tmb.DeclaringType ) ) + { + TypesInUseInScene.Add( tmb.DeclaringType ); + TypesInUseInSceneList.Add( tmb.DeclaringType ); + } + methodProps["dt"] = CilboxUtil.GetSerializeeFromNativeType( tmb.DeclaringType ); methodProps["name"] = new Serializee( tmb.Name ); @@ -2509,7 +2657,7 @@ public static void OnPostprocessScene() { } else if( ot == CilboxUtil.OpCodes.OperandType.InlineField ) { - if( !assemblyMetadataReverseOriginal.TryGetValue( (int)operand, out writebackToken ) ) + if( !assemblyMetadataReverseOriginal.TryGetValue( (proxyAssembly, (int)operand), out writebackToken ) ) { writebackToken = mdcount; FieldInfo rf = proxyAssembly.ManifestModule.ResolveField( (int)operand ); @@ -2526,11 +2674,10 @@ public static void OnPostprocessScene() { } else if( ot == CilboxUtil.OpCodes.OperandType.InlineType ) { - if( !assemblyMetadataReverseOriginal.TryGetValue( (int)operand, out writebackToken ) ) + if( !assemblyMetadataReverseOriginal.TryGetValue( (proxyAssembly, (int)operand), out writebackToken ) ) { writebackToken = mdcount; Type ty = proxyAssembly.ManifestModule.ResolveType( (int)operand ); - Dictionary fieldProps = new Dictionary(); fieldProps["mt"] = new Serializee( ((int)MetaTokenType.mtType).ToString() ); fieldProps["dt"] = CilboxUtil.GetSerializeeFromNativeType( ty ); @@ -2544,7 +2691,7 @@ public static void OnPostprocessScene() { if( changeOperand ) { i = backupi; - assemblyMetadataReverseOriginal[(int)operand] = writebackToken; + assemblyMetadataReverseOriginal[(proxyAssembly, (int)operand)] = writebackToken; CilboxUtil.BytecodeReplaceLiteral( ref byteCode, ref i, opLen, writebackToken ); } if( i >= byteCode.Length ) break; @@ -2606,7 +2753,9 @@ public static void OnPostprocessScene() { } MethodProps["parameters"] = new Serializee( parameterList ); MethodProps["maxStack"] = new Serializee( mb.MaxStackSize.ToString() ); - MethodProps["isVoid"] = new Serializee( (m is MethodInfo)?(((MethodInfo)m).ReturnType == typeof(void) ? "1" : "0" ): "0" ); + bool isCtor = m.IsConstructor; + MethodProps["isVoid"] = new Serializee( (m is MethodInfo)?(((MethodInfo)m).ReturnType == typeof(void) ? "1" : "0" ): (isCtor ? "1" : "0") ); + MethodProps["isCtor"] = new Serializee( isCtor ? "1" : "0" ); MethodProps["isStatic"] = new Serializee( m.IsStatic ? "1" : "0" ); MethodProps["fullSignature"] = new Serializee( m.ToString() ); @@ -2618,14 +2767,19 @@ public static void OnPostprocessScene() { allClassMethods[type.FullName] = new Serializee( methods ); perfType.End(); } + } - perf.End(); perf = new ProfilerMarker( "Secondary Getting Types" ); perf.Begin(); - // Now that we've iterated through all classes, and collected all possible uses of field IDs, - // go through the classes again, collecting the fields themselves. + perf.End(); perf = new ProfilerMarker( "Secondary Getting Types" ); perf.Begin(); - foreach (Type type in proxyAssembly.GetTypes()) + // Now that we've iterated through all classes, and collected all possible uses of field IDs, + // go through the classes again, collecting the fields themselves. + + { + for( int typeIndex = 0; typeIndex < TypesInUseInSceneList.Count; typeIndex++ ) { + Type type = TypesInUseInSceneList[typeIndex]; + if( !CilboxUtil.HasCilboxableAttribute( type ) ) continue; if( type.IsEnum ) @@ -2656,7 +2810,7 @@ public static void OnPostprocessScene() { // Fill in our metadata with a class-specific field ID, if this field ID was used in code anywhere. uint mdid; - if( assemblyMetadataReverseOriginal.TryGetValue(f.MetadataToken, out mdid) ) + if( assemblyMetadataReverseOriginal.TryGetValue((type.Assembly, f.MetadataToken), out mdid) ) { Serializee sOpen = assemblyMetadata[mdid.ToString()]; Dictionary< String, Serializee > m = sOpen.AsMap(); @@ -2879,4 +3033,3 @@ public enum ImportFunctionID OnCollisionExit } } - diff --git a/Basis/Packages/com.cnlohr.cilbox/CilboxUtil.cs b/Basis/Packages/com.cnlohr.cilbox/CilboxUtil.cs index 87612ac5d..4ff086944 100644 --- a/Basis/Packages/com.cnlohr.cilbox/CilboxUtil.cs +++ b/Basis/Packages/com.cnlohr.cilbox/CilboxUtil.cs @@ -420,6 +420,17 @@ public CilboxInterpreterTimeoutException(string msg, string className, string me } } + public class CilboxUnhandledInterpretedException : CilboxInterpreterRuntimeException + { + public readonly object Throwee; + + public CilboxUnhandledInterpretedException(string msg, object throwee, string className, string methodName, int pc) : + base(msg, className, methodName, pc) + { + Throwee = throwee; + } + } + /////////////////////////////////////////////////////////////////////////// // SERIALIZATION / DESERIALIZATION ////////////////////////////////////// /////////////////////////////////////////////////////////////////////////// diff --git a/Basis/Packages/com.hecomi.ulipsync/BasisUlipSync.cs b/Basis/Packages/com.hecomi.ulipsync/BasisUlipSync.cs index d866195f4..1d4ece47e 100644 --- a/Basis/Packages/com.hecomi.ulipsync/BasisUlipSync.cs +++ b/Basis/Packages/com.hecomi.ulipsync/BasisUlipSync.cs @@ -52,6 +52,7 @@ public unsafe class BasisUlipSync BasisDctPlan _dctPlan; float[] _lastApplied; + readonly object _bufferSwapLock = new object(); public struct BlendMap { @@ -142,6 +143,13 @@ public void Execute() public void Simulate(float DeltaTime) { + if (HasJob) + { + _jobHandle.Complete(); + _jobHandle = default; + HasJob = false; + } + if (Interlocked.Exchange(ref _isDataReceived, 0) != 1) return; if (!_allocated) return; @@ -149,16 +157,23 @@ public void Simulate(float DeltaTime) if (!BasisUlipSyncDriver.IsInitialized) return; - int oldActive = _activeInputBuffer; - int newActive = oldActive ^ 1; + int oldActive; + int frozenStartIndex; + NativeArray frozenInput; + + lock (_bufferSwapLock) + { + oldActive = _activeInputBuffer; + int newActive = oldActive ^ 1; - Volatile.Write(ref _activeInputBuffer, newActive); + Volatile.Write(ref _activeInputBuffer, newActive); - int frozenStartIndex = oldActive == 0 - ? Volatile.Read(ref _writeIndexA) - : Volatile.Read(ref _writeIndexB); + frozenStartIndex = oldActive == 0 + ? Volatile.Read(ref _writeIndexA) + : Volatile.Read(ref _writeIndexB); - NativeArray frozenInput = oldActive == 0 ? _inputA : _inputB; + frozenInput = oldActive == 0 ? _inputA : _inputB; + } byte normalizeScores = (byte)0; @@ -236,6 +251,7 @@ public void Apply() HasJob = false; _jobHandle.Complete(); + _jobHandle = default; if (_mfccForOther.IsCreated && _mfcc.IsCreated) { @@ -441,24 +457,27 @@ public void OnDataReceived(float[] input, int channels, int length) int ch = math.max(channels, 1); - int buf = Volatile.Read(ref _activeInputBuffer); - NativeArray dstArr = (buf == 0) ? _inputA : _inputB; - - float* dst = (float*)NativeArrayUnsafeUtility.GetUnsafePtr(dstArr); - - fixed (float* src = input) + lock (_bufferSwapLock) { - int w = (buf == 0) ? Volatile.Read(ref _writeIndexA) : Volatile.Read(ref _writeIndexB); + int buf = Volatile.Read(ref _activeInputBuffer); + NativeArray dstArr = (buf == 0) ? _inputA : _inputB; + + float* dst = (float*)NativeArrayUnsafeUtility.GetUnsafePtr(dstArr); - for (int s = 0; s < length; s += ch) + fixed (float* src = input) { - dst[w] = src[s]; - w++; - if (w == cap) w = 0; - } + int w = (buf == 0) ? Volatile.Read(ref _writeIndexA) : Volatile.Read(ref _writeIndexB); + + for (int s = 0; s < length; s += ch) + { + dst[w] = src[s]; + w++; + if (w == cap) w = 0; + } - if (buf == 0) Volatile.Write(ref _writeIndexA, w); - else Volatile.Write(ref _writeIndexB, w); + if (buf == 0) Volatile.Write(ref _writeIndexA, w); + else Volatile.Write(ref _writeIndexB, w); + } } Interlocked.Exchange(ref _isDataReceived, 1); diff --git a/Basis/Packages/com.steam.steamaudio/Runtime/SteamAudioManager.cs b/Basis/Packages/com.steam.steamaudio/Runtime/SteamAudioManager.cs index 30274baff..47432bea8 100644 --- a/Basis/Packages/com.steam.steamaudio/Runtime/SteamAudioManager.cs +++ b/Basis/Packages/com.steam.steamaudio/Runtime/SteamAudioManager.cs @@ -549,8 +549,9 @@ private void ScheduleInstance() EnsureSourceCapacity(CurrentArraySource); EnsureListenerCapacity(CurrentArrayListener); - JobHandle sourcesHandle = default; - JobHandle listenersHandle = default; + JobHandle priorHandle = combined; + JobHandle sourcesHandle = priorHandle; + JobHandle listenersHandle = priorHandle; if (CurrentArraySource > 0 && mSourceTransforms.isCreated) { @@ -558,7 +559,7 @@ private void ScheduleInstance() { PoseData = mSourceGathers, }; - sourcesHandle = job.Schedule(mSourceTransforms); + sourcesHandle = job.Schedule(mSourceTransforms, priorHandle); } if (CurrentArrayListener > 0 && mListenerTransforms.isCreated) @@ -567,13 +568,17 @@ private void ScheduleInstance() { PoseData = mListenerGathers, }; - listenersHandle = job.Schedule(mListenerTransforms); + listenersHandle = job.Schedule(mListenerTransforms, priorHandle); } + combined = JobHandle.CombineDependencies(sourcesHandle, listenersHandle); } public JobHandle combined; private void ApplyInstance() { + // Always complete pose gather jobs even when later early-returns skip simulation work. + combined.Complete(); + if (mAudioEngineState == null) return; @@ -621,10 +626,6 @@ private void ApplyInstance() mSimulator.SetSharedInputs(SimulationFlags.Direct, sharedInputs); - - // Complete before calling into Steam Audio (main-thread plugin calls) - combined.Complete(); - // --- Direct inputs from cached pose arrays --- for (int i = 0; i < CurrentArraySource; i++) { @@ -1011,7 +1012,8 @@ private void EnsureSourceCapacity(int required) int newCap = (mSourceCapacity <= 0) ? 8 : mSourceCapacity * 2; if (newCap < required) newCap = required; - // Dispose old arrays if created + // Dispose old arrays if created. Ensure no gather jobs are still using them. + combined.Complete(); if (mSourceGathers.IsCreated) mSourceGathers.Dispose(); mSourceGathers = new NativeArray(newCap, Allocator.Persistent); @@ -1027,6 +1029,7 @@ private void EnsureListenerCapacity(int required) int newCap = (mListenerCapacity <= 0) ? 4 : mListenerCapacity * 2; if (newCap < required) newCap = required; + combined.Complete(); if (mListenerGathers.IsCreated) mListenerGathers.Dispose(); mListenerGathers = new NativeArray(newCap, Allocator.Persistent); @@ -1049,8 +1052,9 @@ private void DisposeTransformAndPoseBuffers() if (mSourceTransforms.isCreated) mSourceTransforms.Dispose(); if (mListenerTransforms.isCreated) mListenerTransforms.Dispose(); - if (mSourceGathers.IsCreated) mSourceGathers.Dispose(); + combined.Complete(); + if (mSourceGathers.IsCreated) mSourceGathers.Dispose(); if (mListenerGathers.IsCreated) mListenerGathers.Dispose(); mSourceCapacity = 0; diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs index a53149cf2..ab972378b 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs @@ -1,4 +1,4 @@ -using Basis.Scripts.BasisSdk; +using Basis.Scripts.BasisSdk; using UnityEngine; namespace HVR.Basis.Comms @@ -13,11 +13,13 @@ public class OSCAcquisition : MonoBehaviour private OSCAcquisitionServer _acquisitionServer; private bool _alreadyInitialized; + private FaceTrackingActivityRelay _activityRelay; private void Awake() { if (avatar == null) avatar = HVRCommsUtil.GetAvatar(this); if (acquisitionService == null) acquisitionService = AcquisitionService.SceneInstance; + _activityRelay = FaceTrackingActivityRelay.GetOrCreate(avatar); avatar.OnAvatarReady -= OnAvatarReady; avatar.OnAvatarReady += OnAvatarReady; @@ -55,6 +57,7 @@ private void OnAddressUpdated(string address, float value) { if (!isActiveAndEnabled) return; + _activityRelay?.NotifySourceSample(); acquisitionService.Submit(HVRAddress.AddressToId(address), value); } } diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs index da04f2fe9..fbd472a63 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs @@ -22,9 +22,14 @@ public class BlendshapeActuation : MonoBehaviour, IHVRInitializable [HideInInspector] [SerializeField] private AcquisitionService acquisition; private Dictionary _addessIdToBaseIndex = new(); + private readonly Dictionary _latestAbsoluteByAddress = new(); private ComputedActuator[] _computedActuators; private ComputedActuator[][] _addressBaseIndexToActuators; private Dictionary _addressToStreamedLowerUpper; + private AddressOverride[] _defaultOverrides = Array.Empty(); + private FaceTrackingActivityRelay _activityRelay; + private bool _isWearer; + private bool _trackingActive; #region NetworkingFields // Can be null due to: @@ -56,6 +61,7 @@ private void Awake() acquisition = AcquisitionService.SceneInstance; } + _activityRelay = FaceTrackingActivityRelay.GetOrCreate(avatar); renderers = HVRCommsUtil.SlowSanitizeEndUserProvidedObjectArray(renderers); definitionFiles = HVRCommsUtil.SlowSanitizeEndUserProvidedObjectArray(definitionFiles); definitions = HVRCommsUtil.SlowSanitizeEndUserProvidedStructArray(definitions); @@ -63,26 +69,16 @@ private void Awake() private void OnAddressUpdated(int address, float inRange) { - if (!_addessIdToBaseIndex.TryGetValue(address, out var baseIndex)) return; - - // TODO: Might need to queue and delay this change so that it executes on the Update loop. - - var actuatorsForThisAddress = _addressBaseIndexToActuators[baseIndex]; - if (actuatorsForThisAddress == null) return; // There may be no actuator for an address when it does not exist in the renderers. - - foreach (var actuator in actuatorsForThisAddress) - { - Actuate(actuator, inRange); - } - - if (featureInterpolator != null) - { - featureInterpolator.SubmitAbsolute(baseIndex, inRange); - } + ApplyAddressValue(address, inRange, forwardToNetwork: _isWearer); } private void OnInterpolatedDataChanged(float[] current) { + if (!_trackingActive || current == null) + { + return; + } + foreach (var actuator in _computedActuators) { var absolute = current[actuator.AddressIndex]; @@ -112,6 +108,10 @@ private static void Actuate(ComputedActuator actuator, float inRange) public void OnHVRAvatarReady(bool isWearer) { + _isWearer = isWearer; + acquisition.RegisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); + _trackingActive = _activityRelay != null && _activityRelay.IsTrackingActive; + var allDefinitions = definitions .Concat(definitionFiles.SelectMany(file => file.definitions)) .ToArray(); @@ -128,7 +128,7 @@ public void OnHVRAvatarReady(bool isWearer) var inValuesForThisAddress = grouping // Reminder that InStart may be greater than InEnd. // We want the lower bound, not the minimum of InStart. - .SelectMany(definition => new [] { definition.inStart, definition.inEnd }) + .SelectMany(definition => new[] { definition.inStart, definition.inEnd }) .ToArray(); return (inValuesForThisAddress.Min(), inValuesForThisAddress.Max()); }); @@ -192,6 +192,12 @@ public void OnHVRAvatarReady(bool isWearer) _addressBaseIndexToActuators[computedActuator.Key] = computedActuator.ToArray(); } + _defaultOverrides = definitionFiles + .SelectMany(file => file.addressOverrides) + .Concat(addressOverrides) + .Where(it => it.overrideDefaultValue) + .ToArray(); + if (isWearer) { acquisition.RegisterAddresses(_addessIdToBaseIndex.Keys.ToArray(), OnAddressUpdated); @@ -215,20 +221,21 @@ public static Dictionary> ResolveSmrToBlendsha public void OnHVRReadyBothAvatarAndNetwork(bool isLocallyOwned) { HVRLogging.ProtocolDebug("OnReadyBothAvatarAndNetwork called on BlendshapeActuation."); + _isWearer = isLocallyOwned; // FIXME: We should be using the computed actuators instead of the address base, assuming that // the list of blendshapes is the same local and remote (no local-only or remote-only blendshapes). featureInterpolator = CommsNetworking.UsingMutualizedInterpolator(avatar, MakeMutualized(), OnInterpolatedDataChanged); - var overrides = definitionFiles - .SelectMany(file => file.addressOverrides) - .Concat(addressOverrides) - .Where(it => it.overrideDefaultValue) - .ToArray(); - foreach (var addressOverride in overrides) + if (_isWearer) { - if (_addessIdToBaseIndex.TryGetValue(HVRAddress.AddressToId(addressOverride.address), out var key)) + if (_trackingActive) { - featureInterpolator.SubmitAbsolute(key, addressOverride.defaultValue); + SubmitDefaultOverridesToNetwork(); + ReplayLatestTrackedValuesToNetwork(); + } + else + { + SubmitNeutralValuesToNetwork(); } } } @@ -272,13 +279,138 @@ private void OnDisable() private void OnDestroy() { - avatar.OnAvatarReady -= OnHVRAvatarReady; + if (avatar != null) + { + avatar.OnAvatarReady -= OnHVRAvatarReady; + } + + if (acquisition != null) + { + acquisition.UnregisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); + if (_isWearer && _addessIdToBaseIndex.Count > 0) + { + acquisition.UnregisterAddresses(_addessIdToBaseIndex.Keys.ToArray(), OnAddressUpdated); + } + } + } + + private void OnTrackingActivityUpdated(int address, float value) + { + if (address != FaceTrackingActivityRelay.ActivityAddressId) + { + return; + } + + bool isTrackingActive = value >= 0.5f; + if (_trackingActive == isTrackingActive) + { + return; + } + + _trackingActive = isTrackingActive; + if (_trackingActive) + { + if (_isWearer) + { + ApplyDefaultOverrides(); + ReplayLatestTrackedValuesToNetwork(); + } + return; + } - acquisition.UnregisterAddresses(_addessIdToBaseIndex.Keys.ToArray(), OnAddressUpdated); + ResetAllBlendshapesToZero(); + _latestAbsoluteByAddress.Clear(); + if (_isWearer) + { + SubmitNeutralValuesToNetwork(); + } + } + + private void ApplyAddressValue(int address, float inRange, bool forwardToNetwork) + { + if (!_trackingActive || !_addessIdToBaseIndex.TryGetValue(address, out var baseIndex)) + { + return; + } + + var actuatorsForThisAddress = _addressBaseIndexToActuators[baseIndex]; + if (actuatorsForThisAddress == null) + { + return; + } + + _latestAbsoluteByAddress[address] = inRange; + foreach (var actuator in actuatorsForThisAddress) + { + Actuate(actuator, inRange); + } + + if (forwardToNetwork && _isWearer && featureInterpolator != null) + { + featureInterpolator.SubmitAbsolute(baseIndex, inRange); + } + } + + private void ApplyDefaultOverrides() + { + foreach (var addressOverride in _defaultOverrides) + { + ApplyAddressValue(HVRAddress.AddressToId(addressOverride.address), addressOverride.defaultValue, forwardToNetwork: true); + } + } + + private void ReplayLatestTrackedValuesToNetwork() + { + if (!_isWearer || featureInterpolator == null) + { + return; + } + + foreach (var pair in _latestAbsoluteByAddress) + { + if (_addessIdToBaseIndex.TryGetValue(pair.Key, out var baseIndex)) + { + featureInterpolator.SubmitAbsolute(baseIndex, pair.Value); + } + } + } + + private void SubmitDefaultOverridesToNetwork() + { + if (!_isWearer || featureInterpolator == null) + { + return; + } + + foreach (var addressOverride in _defaultOverrides) + { + if (_addessIdToBaseIndex.TryGetValue(HVRAddress.AddressToId(addressOverride.address), out var baseIndex)) + { + featureInterpolator.SubmitAbsolute(baseIndex, addressOverride.defaultValue); + } + } + } + + private void SubmitNeutralValuesToNetwork() + { + if (!_isWearer || featureInterpolator == null) + { + return; + } + + foreach (var baseIndex in _addessIdToBaseIndex.Values) + { + featureInterpolator.SubmitAbsolute(baseIndex, 0f); + } } private void ResetAllBlendshapesToZero() { + if (_computedActuators == null) + { + return; + } + foreach (var computedActuator in _computedActuators) { foreach (var target in computedActuator.Targets) diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs index da12f3ff3..61751f262 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs @@ -39,8 +39,14 @@ public EyeTrackingBoneActuation() public float _fEyeLeftX; public float _fEyeRightX; public float _fEyeY; - public bool _anyAddressUpdated; public bool IsLocal; + public BasisNetworkReceiver Receiver = null; + + private bool _eyeFollowDriverApplicable; + private bool _trackingActive; + private bool _registeredSourceAddresses; + private FaceTrackingActivityRelay _activityRelay; + #region NetworkingFields // Can be null due to: // - Application with no network, or @@ -49,21 +55,24 @@ public EyeTrackingBoneActuation() // - Becomes non-null after HVRAvatarComms.OnAvatarNetworkReady is successfully invoked [NonSerialized] internal MutualizedFeatureInterpolator featureInterpolator; #endregion - public BasisNetworkReceiver Receiver = null; - private bool _eyeFollowDriverApplicable; private void Awake() { if (avatar == null) avatar = HVRCommsUtil.GetAvatar(this); if (acquisition == null) acquisition = AcquisitionService.SceneInstance; + _activityRelay = FaceTrackingActivityRelay.GetOrCreate(avatar); } public void OnHVRAvatarReady(bool isWearer) { + _registeredSourceAddresses = isWearer; + _eyeFollowDriverApplicable = isWearer; + _trackingActive = _activityRelay != null && _activityRelay.IsTrackingActive; + + acquisition.RegisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); if (isWearer) { acquisition.RegisterAddresses(OurAddresses, OnAddressUpdated); - _eyeFollowDriverApplicable = true; } } @@ -84,11 +93,27 @@ public void OnHVRReadyBothAvatarAndNetwork(bool isWearer) upper = 1f, }).ToList(); featureInterpolator = CommsNetworking.UsingMutualizedInterpolator(avatar, mutualizedInterpolationRanges, OnInterpolatedDataChanged); + + if (IsLocal) + { + SetBuiltInEyeFollowDriverOverriden(_trackingActive); + if (_trackingActive) + { + SubmitCurrentEyeStateToNetwork(); + } + else + { + SubmitNeutralEyesToNetwork(); + } + } + else if (!_trackingActive) + { + ClearRemoteOverrides(); + } } private void OnEnable() { - SetBuiltInEyeFollowDriverOverriden(true); BasisNetworkTransmitter.AfterAvatarChanges += ForceUpdate; } @@ -96,51 +121,98 @@ private void OnDisable() { SetBuiltInEyeFollowDriverOverriden(false); BasisNetworkTransmitter.AfterAvatarChanges -= ForceUpdate; + ClearRemoteOverrides(); } private void OnDestroy() { - if (IsLocal) + if (acquisition != null) { - acquisition.UnregisterAddresses(OurAddresses, OnAddressUpdated); - - if (IsLocal && Receiver != null) + acquisition.UnregisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); + if (_registeredSourceAddresses) { - Receiver.RemotePlayer.RemoteFaceDriver.OverrideEye = false; - Receiver.RemotePlayer.RemoteFaceDriver.OverrideBlinking = false; + acquisition.UnregisterAddresses(OurAddresses, OnAddressUpdated); } } + + ClearRemoteOverrides(); + SetBuiltInEyeFollowDriverOverriden(false); } private void OnAddressUpdated(int address, float value) { - // FIXME: Temp fix, we'll need to hook to NetworkReady instead. - // This is a quick fix so that we don't need to reupload the avatar. - _anyAddressUpdated = _anyAddressUpdated || value != 0f; - if (_anyAddressUpdated) + if (!_trackingActive) { - BasisLocalEyeDriver.IsEnabled = false; + return; } if (address == EyeLeftXAddress) { _fEyeLeftX = value; - if (featureInterpolator != null) featureInterpolator.SubmitAbsolute(0, value); + if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(0, value); } else if (address == EyeRightXAddress) { _fEyeRightX = value; - if (featureInterpolator != null) featureInterpolator.SubmitAbsolute(1, value); + if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(1, value); } else if (address == EyeYAddress) { _fEyeY = value; - if (featureInterpolator != null) featureInterpolator.SubmitAbsolute(2, value); + if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(2, value); + } + } + + private void OnTrackingActivityUpdated(int address, float value) + { + if (address != FaceTrackingActivityRelay.ActivityAddressId) + { + return; + } + + bool isTrackingActive = value >= 0.5f; + if (_trackingActive == isTrackingActive) + { + return; + } + + _trackingActive = isTrackingActive; + if (IsLocal) + { + SetBuiltInEyeFollowDriverOverriden(_trackingActive); + } + + if (_trackingActive) + { + if (IsLocal) + { + SubmitCurrentEyeStateToNetwork(); + } + return; + } + + _fEyeLeftX = 0f; + _fEyeRightX = 0f; + _fEyeY = 0f; + + if (IsLocal) + { + SubmitNeutralEyesToNetwork(); + } + else + { + SetNeutralRemoteEyes(); + ClearRemoteOverrides(); } } private void ForceUpdate() { + if (!_trackingActive) + { + return; + } + SetEyeRotation(_fEyeLeftX, _fEyeY, EyeSide.Left); SetEyeRotation(_fEyeRightX, _fEyeY, EyeSide.Right); } @@ -166,29 +238,22 @@ private void SetEyeRotation(float x, float y, EyeSide side) throw new ArgumentOutOfRangeException(nameof(side), side, null); } } - else + else if (!IsLocal && Receiver != null) { - if (IsLocal && Receiver != null) + Receiver.RemotePlayer.RemoteFaceDriver.OverrideEye = true; + Receiver.RemotePlayer.RemoteFaceDriver.OverrideBlinking = true; + switch (side) { - Receiver.RemotePlayer.RemoteFaceDriver.OverrideEye = true; - Receiver.RemotePlayer.RemoteFaceDriver.OverrideBlinking = true; - switch (side) - { - case EyeSide.Left: - float result0 = (y + 1) / 2; - float result1 = (x + 1) / 2; - Receiver.EyesAndMouth[0] = result0; - Receiver.EyesAndMouth[1] = result1; - break; - case EyeSide.Right: - result0 = (y + 1) / 2; - result1 = (x + 1) / 2; - Receiver.EyesAndMouth[2] = result0; - Receiver.EyesAndMouth[3] = result1; - break; - default: - throw new ArgumentOutOfRangeException(nameof(side), side, null); - } + case EyeSide.Left: + Receiver.EyesAndMouth[0] = (y + 1) / 2; + Receiver.EyesAndMouth[1] = (x + 1) / 2; + break; + case EyeSide.Right: + Receiver.EyesAndMouth[2] = (y + 1) / 2; + Receiver.EyesAndMouth[3] = (x + 1) / 2; + break; + default: + throw new ArgumentOutOfRangeException(nameof(side), side, null); } } } @@ -198,6 +263,54 @@ private void SetBuiltInEyeFollowDriverOverriden(bool value) BasisLocalEyeDriver.Override = value; } + private void SubmitCurrentEyeStateToNetwork() + { + if (!IsLocal || featureInterpolator == null) + { + return; + } + + featureInterpolator.SubmitAbsolute(0, _fEyeLeftX); + featureInterpolator.SubmitAbsolute(1, _fEyeRightX); + featureInterpolator.SubmitAbsolute(2, _fEyeY); + } + + private void SubmitNeutralEyesToNetwork() + { + if (!IsLocal || featureInterpolator == null) + { + return; + } + + featureInterpolator.SubmitAbsolute(0, 0f); + featureInterpolator.SubmitAbsolute(1, 0f); + featureInterpolator.SubmitAbsolute(2, 0f); + } + + private void SetNeutralRemoteEyes() + { + if (Receiver == null) + { + return; + } + + Receiver.EyesAndMouth[0] = 0.5f; + Receiver.EyesAndMouth[1] = 0.5f; + Receiver.EyesAndMouth[2] = 0.5f; + Receiver.EyesAndMouth[3] = 0.5f; + } + + private void ClearRemoteOverrides() + { + if (IsLocal || Receiver == null) + { + return; + } + + Receiver.RemotePlayer.RemoteFaceDriver.OverrideEye = false; + Receiver.RemotePlayer.RemoteFaceDriver.OverrideBlinking = false; + } + private enum EyeSide { Left, Right @@ -206,6 +319,11 @@ private enum EyeSide #region NetworkingMethods private void OnInterpolatedDataChanged(float[] current) { + if (!_trackingActive || current == null || current.Length < 3) + { + return; + } + _fEyeLeftX = current[0]; _fEyeRightX = current[1]; _fEyeY = current[2]; diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/AutomaticFaceTracking.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/AutomaticFaceTracking.cs index 4639a8c33..2b2bf8edf 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/AutomaticFaceTracking.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/AutomaticFaceTracking.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; @@ -35,6 +35,7 @@ public class AutomaticFaceTracking : MonoBehaviour, IHVRInitializable [NonSerialized] internal OSCAcquisition oscAcquisition; [NonSerialized] internal BlendshapeActuation blendshapeActuation; [NonSerialized] internal EyeTrackingBoneActuation eyeTrackingBoneActuation; + [NonSerialized] internal FaceTrackingActivityRelay faceTrackingActivityRelay; private bool _isWearer; @@ -119,6 +120,9 @@ private void Failed() private void SetupFaceTracking(BlendshapeActuationDefinitionFile[] definitionFiles, List smrs) { renderers = smrs; + faceTrackingActivityRelay = FaceTrackingActivityRelay.GetOrCreate(_avatar); + faceTrackingActivityRelay.OnHVRAvatarReady(_isWearer); + if (_isWearer) { oscAcquisition = CreateOSCAcquisitionIfNotExists(); diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs new file mode 100644 index 000000000..2d8e2b74e --- /dev/null +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs @@ -0,0 +1 @@ +// The FaceTrackingActivityRelay implementation lives in Runtime/Networking/HVRCommsUtil.cs. diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs.meta b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs.meta new file mode 100644 index 000000000..eb7f2e13b --- /dev/null +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 8317c8e0fc6d3eb418345948d6c69a1a \ No newline at end of file diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/HVRAvatarComms.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/HVRAvatarComms.cs index dac972ba7..4e95f3917 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/HVRAvatarComms.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/HVRAvatarComms.cs @@ -52,7 +52,7 @@ private void Awake() private void OnAvatarReady(bool isWearer) { - _isWearer = true; + _isWearer = isWearer; var allInitializables = avatar.GetComponentsInChildren(true); foreach (var initializable in allInitializables) @@ -103,14 +103,16 @@ private void DeclareMutualizedInterpolator(bool isWearer, HVRNetworkingCarrier c _streamedLateInit.transmitter = carrier; _streamedLateInit.isWearer = isWearer; _streamedLateInit.localIdentifier = 0; - _toStoreLater.Clear(); + var pendingStores = _toStoreLater.ToArray(); holder.SetActive(true); + _streamedLateInit.InitializeNormalizedValues(BuildNeutralNormalizedValues()); // StreamedAvatarFeature only gets the ability to store data AFTER Awake() runs, so order matters here. - foreach (var toStoreLater in _toStoreLater) + foreach (var toStoreLater in pendingStores) { var mutualizedIndex = toStoreLater.mutualizedIndex; _streamedLateInit.Store(mutualizedIndex, _ranges[mutualizedIndex].AbsoluteToRange(toStoreLater.absolute)); } + _toStoreLater.Clear(); _streamedLateInit.OnInterpolatedDataChanged += mutualizedData => { @@ -169,6 +171,7 @@ public MutualizedFeatureInterpolator NeedsMutualizedInterpolator(List= 0f + ? Mathf.Clamp01(range.AbsoluteToRange(0f)) + : 0f; + } + + return normalized; + } + private class HVRRedirectToStreamed : IFeatureReceiver { private readonly StreamedAvatarFeature streamed; diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/StreamedAvatarFeature.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/StreamedAvatarFeature.cs index cdc04cffc..4123288fc 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/StreamedAvatarFeature.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/StreamedAvatarFeature.cs @@ -40,9 +40,7 @@ public class StreamedAvatarFeature : MonoBehaviour private void Awake() { - previous ??= new float[valueArraySize]; - target ??= new float[valueArraySize]; - current ??= new float[valueArraySize]; + EnsureBuffers(); } private void OnDisable() @@ -52,6 +50,7 @@ private void OnDisable() public void Store(int index, float value) { + EnsureBuffers(); current[index] = value; if (PrioritizeLargeChanges && isWearer) { @@ -66,6 +65,20 @@ public void Store(int index, float value) } } + public void InitializeNormalizedValues(IReadOnlyList normalizedValues) + { + EnsureBuffers(); + + int count = Mathf.Min(valueArraySize, normalizedValues?.Count ?? 0); + for (int i = 0; i < count; i++) + { + float clamped = Mathf.Clamp01(normalizedValues[i]); + current[i] = clamped; + previous[i] = clamped; + target[i] = clamped; + } + } + /// Exposed for testing purposes. public void QueueEvent(StreamedAvatarFeaturePayload message) { @@ -261,6 +274,13 @@ public void OnResyncRequested(ushort[] whoAsked) FloatValues = current }, whoAsked); } + + private void EnsureBuffers() + { + previous ??= new float[valueArraySize]; + target ??= new float[valueArraySize]; + current ??= new float[valueArraySize]; + } } public class StreamedAvatarFeaturePayload { diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Networking/HVRCommsUtil.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Networking/HVRCommsUtil.cs index f4cb03bcd..dec9693c0 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Networking/HVRCommsUtil.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Networking/HVRCommsUtil.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using Basis.Scripts.BasisSdk; using UnityEngine; @@ -43,4 +44,137 @@ public static T[] SlowSanitizeEndUserProvidedStructArray(T[] structuresNullab return structuresNullable; } } + + [AddComponentMenu("HVR.Basis/Comms/Internal/Face Tracking Activity Relay")] + public class FaceTrackingActivityRelay : MonoBehaviour, IHVRInitializable + { + public const string ActivityAddress = "HVR/Internal/FaceTrackingActive"; + public static readonly int ActivityAddressId = HVRAddress.AddressToId(ActivityAddress); + public const float InactivityTimeoutSeconds = 0.5f; + + [HideInInspector] [SerializeField] private BasisAvatar avatar; + [HideInInspector] [SerializeField] private AcquisitionService acquisition; + + [NonSerialized] internal MutualizedFeatureInterpolator featureInterpolator; + + private bool _isWearer; + private bool _isTrackingActive; + private float _lastActivityTime = float.NegativeInfinity; + + public bool IsTrackingActive => _isTrackingActive; + + public static FaceTrackingActivityRelay GetOrCreate(BasisAvatar avatar) + { + if (avatar == null) + { + return null; + } + + var relay = avatar.GetComponentInChildren(true); + if (relay != null) + { + return relay; + } + + var relayRoot = new GameObject("Generated__FaceTrackingActivityRelay") + { + transform = + { + parent = avatar.transform, + } + }; + return relayRoot.AddComponent(); + } + + private void Awake() + { + if (avatar == null) + { + avatar = HVRCommsUtil.GetAvatar(this); + } + + if (acquisition == null) + { + acquisition = AcquisitionService.SceneInstance; + } + } + + public void OnHVRAvatarReady(bool isWearer) + { + _isWearer = isWearer; + ApplyTrackingState(false, submitToNetwork: false); + } + + public void OnHVRReadyBothAvatarAndNetwork(bool isWearer) + { + _isWearer = isWearer; + featureInterpolator = CommsNetworking.UsingMutualizedInterpolator(avatar, new List + { + new MutualizedInterpolationRange + { + address = ActivityAddressId, + lower = 0f, + upper = 1f, + } + }, OnInterpolatedDataChanged); + + if (_isWearer && featureInterpolator != null) + { + featureInterpolator.SubmitAbsolute(0, _isTrackingActive ? 1f : 0f); + } + } + + private void Update() + { + if (!_isWearer || !_isTrackingActive) + { + return; + } + + if (Time.unscaledTime - _lastActivityTime > InactivityTimeoutSeconds) + { + ApplyTrackingState(false, submitToNetwork: true); + } + } + + public void NotifySourceSample() + { + if (!_isWearer) + { + return; + } + + _lastActivityTime = Time.unscaledTime; + if (!_isTrackingActive) + { + ApplyTrackingState(true, submitToNetwork: true); + } + } + + private void OnInterpolatedDataChanged(float[] current) + { + if (_isWearer || current == null || current.Length == 0) + { + return; + } + + ApplyTrackingState(current[0] >= 0.5f, submitToNetwork: false); + } + + private void ApplyTrackingState(bool isTrackingActive, bool submitToNetwork) + { + bool stateChanged = _isTrackingActive != isTrackingActive; + _isTrackingActive = isTrackingActive; + + if (acquisition != null && (stateChanged || submitToNetwork)) + { + acquisition.Submit(ActivityAddressId, isTrackingActive ? 1f : 0f); + } + + if (submitToNetwork && _isWearer && featureInterpolator != null) + { + featureInterpolator.SubmitAbsolute(0, isTrackingActive ? 1f : 0f); + } + } + } } From ad6c3847f6cdfce4780de45e7217170d218306b4 Mon Sep 17 00:00:00 2001 From: Toys0125 Date: Fri, 6 Mar 2026 21:51:55 -0600 Subject: [PATCH 2/5] fix(face-tracking): make host authoritative for activity state and neutralize startup sync * Added an internal `FaceTrackingActivityRelay` channel (`HVR/Internal/FaceTrackingActive`) with a host-only inactivity timeout (0.5s). Remotes now follow host activity and do not run their own timeout logic. * Wired OSC input to activity updates so any incoming face packet marks tracking active. * Fixed remote eye application in `EyeTrackingBoneActuation` (!IsLocal && Receiver != null) and ensured remote override flags are cleared when tracking is inactive/disabled. * Gated eye and blendshape actuation behind propagated tracking activity: Active: HVR drives local/remote face data. Inactive: reset HVR-driven values to neutral and return control to Basis default behavior. * Stopped applying startup/default blendshape overrides when tracking is inactive; defaults are only pushed while tracking is active. * Changed mutualized feature initialization to neutral normalized values based on interpolation ranges (AbsoluteToRange(0) when 0 is in range), preventing signed channels from booting at low-end values (e.g. unintended TongueLeft). * Updated networking init flow to seed neutral values before initial replay, and fixed wearer/range bookkeeping issues in `HVRAvatarComms`. --- .../com.basis.eventdriver/BasisEventDriver.cs | 5 +- .../Networking/BasisRemoteFaceManagement.cs | 2 +- .../com.hecomi.ulipsync/BasisUlipSync.cs | 61 ++++-- .../Runtime/SteamAudioManager.cs | 24 ++- .../Components/Acquisition/OSCAcquisition.cs | 5 +- .../Actuation/BlendshapeActuation.cs | 186 +++++++++++++--- .../Actuation/EyeTrackingBoneActuation.cs | 198 ++++++++++++++---- .../FaceTracking/AutomaticFaceTracking.cs | 6 +- .../FaceTracking/FaceTrackingActivityRelay.cs | 1 + .../FaceTrackingActivityRelay.cs.meta | 2 + .../Components/Networking/HVRAvatarComms.cs | 23 +- .../Networking/StreamedAvatarFeature.cs | 26 ++- .../Runtime/Networking/HVRCommsUtil.cs | 134 ++++++++++++ 13 files changed, 565 insertions(+), 108 deletions(-) create mode 100644 Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs create mode 100644 Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs.meta diff --git a/Basis/Packages/com.basis.eventdriver/BasisEventDriver.cs b/Basis/Packages/com.basis.eventdriver/BasisEventDriver.cs index 4a322bf24..15e6af9d0 100644 --- a/Basis/Packages/com.basis.eventdriver/BasisEventDriver.cs +++ b/Basis/Packages/com.basis.eventdriver/BasisEventDriver.cs @@ -230,10 +230,12 @@ public void LateUpdate() /// private void OnBeforeRender() { + // Ensure remote face jobs are completed/applied even before local player is ready. + BasisRemoteFaceManagement.Apply(); //apply blendshapes + if (BasisLocalPlayer.PlayerReady) { BasisLocalPlayer.Instance.SimulateOnRender(); - BasisRemoteFaceManagement.Apply(); //apply blendshapes BasisLocalCameraDriver.Instance.microphoneIconDriver.Simulate(DeltaTime); //update microphone icon } } @@ -275,3 +277,4 @@ public void OnDrawGizmosSelected() #endif } } + diff --git a/Basis/Packages/com.basis.framework/Networking/BasisRemoteFaceManagement.cs b/Basis/Packages/com.basis.framework/Networking/BasisRemoteFaceManagement.cs index cbc2c1285..502abf207 100644 --- a/Basis/Packages/com.basis.framework/Networking/BasisRemoteFaceManagement.cs +++ b/Basis/Packages/com.basis.framework/Networking/BasisRemoteFaceManagement.cs @@ -90,7 +90,7 @@ public static void Simulate(double t,float dt) blinkOut = blinkOut, }; - handle = job.Schedule(count, BatchSize); + handle = job.Schedule(count, BatchSize, handle); } public static void Apply() diff --git a/Basis/Packages/com.hecomi.ulipsync/BasisUlipSync.cs b/Basis/Packages/com.hecomi.ulipsync/BasisUlipSync.cs index d866195f4..1d4ece47e 100644 --- a/Basis/Packages/com.hecomi.ulipsync/BasisUlipSync.cs +++ b/Basis/Packages/com.hecomi.ulipsync/BasisUlipSync.cs @@ -52,6 +52,7 @@ public unsafe class BasisUlipSync BasisDctPlan _dctPlan; float[] _lastApplied; + readonly object _bufferSwapLock = new object(); public struct BlendMap { @@ -142,6 +143,13 @@ public void Execute() public void Simulate(float DeltaTime) { + if (HasJob) + { + _jobHandle.Complete(); + _jobHandle = default; + HasJob = false; + } + if (Interlocked.Exchange(ref _isDataReceived, 0) != 1) return; if (!_allocated) return; @@ -149,16 +157,23 @@ public void Simulate(float DeltaTime) if (!BasisUlipSyncDriver.IsInitialized) return; - int oldActive = _activeInputBuffer; - int newActive = oldActive ^ 1; + int oldActive; + int frozenStartIndex; + NativeArray frozenInput; + + lock (_bufferSwapLock) + { + oldActive = _activeInputBuffer; + int newActive = oldActive ^ 1; - Volatile.Write(ref _activeInputBuffer, newActive); + Volatile.Write(ref _activeInputBuffer, newActive); - int frozenStartIndex = oldActive == 0 - ? Volatile.Read(ref _writeIndexA) - : Volatile.Read(ref _writeIndexB); + frozenStartIndex = oldActive == 0 + ? Volatile.Read(ref _writeIndexA) + : Volatile.Read(ref _writeIndexB); - NativeArray frozenInput = oldActive == 0 ? _inputA : _inputB; + frozenInput = oldActive == 0 ? _inputA : _inputB; + } byte normalizeScores = (byte)0; @@ -236,6 +251,7 @@ public void Apply() HasJob = false; _jobHandle.Complete(); + _jobHandle = default; if (_mfccForOther.IsCreated && _mfcc.IsCreated) { @@ -441,24 +457,27 @@ public void OnDataReceived(float[] input, int channels, int length) int ch = math.max(channels, 1); - int buf = Volatile.Read(ref _activeInputBuffer); - NativeArray dstArr = (buf == 0) ? _inputA : _inputB; - - float* dst = (float*)NativeArrayUnsafeUtility.GetUnsafePtr(dstArr); - - fixed (float* src = input) + lock (_bufferSwapLock) { - int w = (buf == 0) ? Volatile.Read(ref _writeIndexA) : Volatile.Read(ref _writeIndexB); + int buf = Volatile.Read(ref _activeInputBuffer); + NativeArray dstArr = (buf == 0) ? _inputA : _inputB; + + float* dst = (float*)NativeArrayUnsafeUtility.GetUnsafePtr(dstArr); - for (int s = 0; s < length; s += ch) + fixed (float* src = input) { - dst[w] = src[s]; - w++; - if (w == cap) w = 0; - } + int w = (buf == 0) ? Volatile.Read(ref _writeIndexA) : Volatile.Read(ref _writeIndexB); + + for (int s = 0; s < length; s += ch) + { + dst[w] = src[s]; + w++; + if (w == cap) w = 0; + } - if (buf == 0) Volatile.Write(ref _writeIndexA, w); - else Volatile.Write(ref _writeIndexB, w); + if (buf == 0) Volatile.Write(ref _writeIndexA, w); + else Volatile.Write(ref _writeIndexB, w); + } } Interlocked.Exchange(ref _isDataReceived, 1); diff --git a/Basis/Packages/com.steam.steamaudio/Runtime/SteamAudioManager.cs b/Basis/Packages/com.steam.steamaudio/Runtime/SteamAudioManager.cs index 30274baff..47432bea8 100644 --- a/Basis/Packages/com.steam.steamaudio/Runtime/SteamAudioManager.cs +++ b/Basis/Packages/com.steam.steamaudio/Runtime/SteamAudioManager.cs @@ -549,8 +549,9 @@ private void ScheduleInstance() EnsureSourceCapacity(CurrentArraySource); EnsureListenerCapacity(CurrentArrayListener); - JobHandle sourcesHandle = default; - JobHandle listenersHandle = default; + JobHandle priorHandle = combined; + JobHandle sourcesHandle = priorHandle; + JobHandle listenersHandle = priorHandle; if (CurrentArraySource > 0 && mSourceTransforms.isCreated) { @@ -558,7 +559,7 @@ private void ScheduleInstance() { PoseData = mSourceGathers, }; - sourcesHandle = job.Schedule(mSourceTransforms); + sourcesHandle = job.Schedule(mSourceTransforms, priorHandle); } if (CurrentArrayListener > 0 && mListenerTransforms.isCreated) @@ -567,13 +568,17 @@ private void ScheduleInstance() { PoseData = mListenerGathers, }; - listenersHandle = job.Schedule(mListenerTransforms); + listenersHandle = job.Schedule(mListenerTransforms, priorHandle); } + combined = JobHandle.CombineDependencies(sourcesHandle, listenersHandle); } public JobHandle combined; private void ApplyInstance() { + // Always complete pose gather jobs even when later early-returns skip simulation work. + combined.Complete(); + if (mAudioEngineState == null) return; @@ -621,10 +626,6 @@ private void ApplyInstance() mSimulator.SetSharedInputs(SimulationFlags.Direct, sharedInputs); - - // Complete before calling into Steam Audio (main-thread plugin calls) - combined.Complete(); - // --- Direct inputs from cached pose arrays --- for (int i = 0; i < CurrentArraySource; i++) { @@ -1011,7 +1012,8 @@ private void EnsureSourceCapacity(int required) int newCap = (mSourceCapacity <= 0) ? 8 : mSourceCapacity * 2; if (newCap < required) newCap = required; - // Dispose old arrays if created + // Dispose old arrays if created. Ensure no gather jobs are still using them. + combined.Complete(); if (mSourceGathers.IsCreated) mSourceGathers.Dispose(); mSourceGathers = new NativeArray(newCap, Allocator.Persistent); @@ -1027,6 +1029,7 @@ private void EnsureListenerCapacity(int required) int newCap = (mListenerCapacity <= 0) ? 4 : mListenerCapacity * 2; if (newCap < required) newCap = required; + combined.Complete(); if (mListenerGathers.IsCreated) mListenerGathers.Dispose(); mListenerGathers = new NativeArray(newCap, Allocator.Persistent); @@ -1049,8 +1052,9 @@ private void DisposeTransformAndPoseBuffers() if (mSourceTransforms.isCreated) mSourceTransforms.Dispose(); if (mListenerTransforms.isCreated) mListenerTransforms.Dispose(); - if (mSourceGathers.IsCreated) mSourceGathers.Dispose(); + combined.Complete(); + if (mSourceGathers.IsCreated) mSourceGathers.Dispose(); if (mListenerGathers.IsCreated) mListenerGathers.Dispose(); mSourceCapacity = 0; diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs index a53149cf2..ab972378b 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs @@ -1,4 +1,4 @@ -using Basis.Scripts.BasisSdk; +using Basis.Scripts.BasisSdk; using UnityEngine; namespace HVR.Basis.Comms @@ -13,11 +13,13 @@ public class OSCAcquisition : MonoBehaviour private OSCAcquisitionServer _acquisitionServer; private bool _alreadyInitialized; + private FaceTrackingActivityRelay _activityRelay; private void Awake() { if (avatar == null) avatar = HVRCommsUtil.GetAvatar(this); if (acquisitionService == null) acquisitionService = AcquisitionService.SceneInstance; + _activityRelay = FaceTrackingActivityRelay.GetOrCreate(avatar); avatar.OnAvatarReady -= OnAvatarReady; avatar.OnAvatarReady += OnAvatarReady; @@ -55,6 +57,7 @@ private void OnAddressUpdated(string address, float value) { if (!isActiveAndEnabled) return; + _activityRelay?.NotifySourceSample(); acquisitionService.Submit(HVRAddress.AddressToId(address), value); } } diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs index da04f2fe9..fbd472a63 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs @@ -22,9 +22,14 @@ public class BlendshapeActuation : MonoBehaviour, IHVRInitializable [HideInInspector] [SerializeField] private AcquisitionService acquisition; private Dictionary _addessIdToBaseIndex = new(); + private readonly Dictionary _latestAbsoluteByAddress = new(); private ComputedActuator[] _computedActuators; private ComputedActuator[][] _addressBaseIndexToActuators; private Dictionary _addressToStreamedLowerUpper; + private AddressOverride[] _defaultOverrides = Array.Empty(); + private FaceTrackingActivityRelay _activityRelay; + private bool _isWearer; + private bool _trackingActive; #region NetworkingFields // Can be null due to: @@ -56,6 +61,7 @@ private void Awake() acquisition = AcquisitionService.SceneInstance; } + _activityRelay = FaceTrackingActivityRelay.GetOrCreate(avatar); renderers = HVRCommsUtil.SlowSanitizeEndUserProvidedObjectArray(renderers); definitionFiles = HVRCommsUtil.SlowSanitizeEndUserProvidedObjectArray(definitionFiles); definitions = HVRCommsUtil.SlowSanitizeEndUserProvidedStructArray(definitions); @@ -63,26 +69,16 @@ private void Awake() private void OnAddressUpdated(int address, float inRange) { - if (!_addessIdToBaseIndex.TryGetValue(address, out var baseIndex)) return; - - // TODO: Might need to queue and delay this change so that it executes on the Update loop. - - var actuatorsForThisAddress = _addressBaseIndexToActuators[baseIndex]; - if (actuatorsForThisAddress == null) return; // There may be no actuator for an address when it does not exist in the renderers. - - foreach (var actuator in actuatorsForThisAddress) - { - Actuate(actuator, inRange); - } - - if (featureInterpolator != null) - { - featureInterpolator.SubmitAbsolute(baseIndex, inRange); - } + ApplyAddressValue(address, inRange, forwardToNetwork: _isWearer); } private void OnInterpolatedDataChanged(float[] current) { + if (!_trackingActive || current == null) + { + return; + } + foreach (var actuator in _computedActuators) { var absolute = current[actuator.AddressIndex]; @@ -112,6 +108,10 @@ private static void Actuate(ComputedActuator actuator, float inRange) public void OnHVRAvatarReady(bool isWearer) { + _isWearer = isWearer; + acquisition.RegisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); + _trackingActive = _activityRelay != null && _activityRelay.IsTrackingActive; + var allDefinitions = definitions .Concat(definitionFiles.SelectMany(file => file.definitions)) .ToArray(); @@ -128,7 +128,7 @@ public void OnHVRAvatarReady(bool isWearer) var inValuesForThisAddress = grouping // Reminder that InStart may be greater than InEnd. // We want the lower bound, not the minimum of InStart. - .SelectMany(definition => new [] { definition.inStart, definition.inEnd }) + .SelectMany(definition => new[] { definition.inStart, definition.inEnd }) .ToArray(); return (inValuesForThisAddress.Min(), inValuesForThisAddress.Max()); }); @@ -192,6 +192,12 @@ public void OnHVRAvatarReady(bool isWearer) _addressBaseIndexToActuators[computedActuator.Key] = computedActuator.ToArray(); } + _defaultOverrides = definitionFiles + .SelectMany(file => file.addressOverrides) + .Concat(addressOverrides) + .Where(it => it.overrideDefaultValue) + .ToArray(); + if (isWearer) { acquisition.RegisterAddresses(_addessIdToBaseIndex.Keys.ToArray(), OnAddressUpdated); @@ -215,20 +221,21 @@ public static Dictionary> ResolveSmrToBlendsha public void OnHVRReadyBothAvatarAndNetwork(bool isLocallyOwned) { HVRLogging.ProtocolDebug("OnReadyBothAvatarAndNetwork called on BlendshapeActuation."); + _isWearer = isLocallyOwned; // FIXME: We should be using the computed actuators instead of the address base, assuming that // the list of blendshapes is the same local and remote (no local-only or remote-only blendshapes). featureInterpolator = CommsNetworking.UsingMutualizedInterpolator(avatar, MakeMutualized(), OnInterpolatedDataChanged); - var overrides = definitionFiles - .SelectMany(file => file.addressOverrides) - .Concat(addressOverrides) - .Where(it => it.overrideDefaultValue) - .ToArray(); - foreach (var addressOverride in overrides) + if (_isWearer) { - if (_addessIdToBaseIndex.TryGetValue(HVRAddress.AddressToId(addressOverride.address), out var key)) + if (_trackingActive) { - featureInterpolator.SubmitAbsolute(key, addressOverride.defaultValue); + SubmitDefaultOverridesToNetwork(); + ReplayLatestTrackedValuesToNetwork(); + } + else + { + SubmitNeutralValuesToNetwork(); } } } @@ -272,13 +279,138 @@ private void OnDisable() private void OnDestroy() { - avatar.OnAvatarReady -= OnHVRAvatarReady; + if (avatar != null) + { + avatar.OnAvatarReady -= OnHVRAvatarReady; + } + + if (acquisition != null) + { + acquisition.UnregisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); + if (_isWearer && _addessIdToBaseIndex.Count > 0) + { + acquisition.UnregisterAddresses(_addessIdToBaseIndex.Keys.ToArray(), OnAddressUpdated); + } + } + } + + private void OnTrackingActivityUpdated(int address, float value) + { + if (address != FaceTrackingActivityRelay.ActivityAddressId) + { + return; + } + + bool isTrackingActive = value >= 0.5f; + if (_trackingActive == isTrackingActive) + { + return; + } + + _trackingActive = isTrackingActive; + if (_trackingActive) + { + if (_isWearer) + { + ApplyDefaultOverrides(); + ReplayLatestTrackedValuesToNetwork(); + } + return; + } - acquisition.UnregisterAddresses(_addessIdToBaseIndex.Keys.ToArray(), OnAddressUpdated); + ResetAllBlendshapesToZero(); + _latestAbsoluteByAddress.Clear(); + if (_isWearer) + { + SubmitNeutralValuesToNetwork(); + } + } + + private void ApplyAddressValue(int address, float inRange, bool forwardToNetwork) + { + if (!_trackingActive || !_addessIdToBaseIndex.TryGetValue(address, out var baseIndex)) + { + return; + } + + var actuatorsForThisAddress = _addressBaseIndexToActuators[baseIndex]; + if (actuatorsForThisAddress == null) + { + return; + } + + _latestAbsoluteByAddress[address] = inRange; + foreach (var actuator in actuatorsForThisAddress) + { + Actuate(actuator, inRange); + } + + if (forwardToNetwork && _isWearer && featureInterpolator != null) + { + featureInterpolator.SubmitAbsolute(baseIndex, inRange); + } + } + + private void ApplyDefaultOverrides() + { + foreach (var addressOverride in _defaultOverrides) + { + ApplyAddressValue(HVRAddress.AddressToId(addressOverride.address), addressOverride.defaultValue, forwardToNetwork: true); + } + } + + private void ReplayLatestTrackedValuesToNetwork() + { + if (!_isWearer || featureInterpolator == null) + { + return; + } + + foreach (var pair in _latestAbsoluteByAddress) + { + if (_addessIdToBaseIndex.TryGetValue(pair.Key, out var baseIndex)) + { + featureInterpolator.SubmitAbsolute(baseIndex, pair.Value); + } + } + } + + private void SubmitDefaultOverridesToNetwork() + { + if (!_isWearer || featureInterpolator == null) + { + return; + } + + foreach (var addressOverride in _defaultOverrides) + { + if (_addessIdToBaseIndex.TryGetValue(HVRAddress.AddressToId(addressOverride.address), out var baseIndex)) + { + featureInterpolator.SubmitAbsolute(baseIndex, addressOverride.defaultValue); + } + } + } + + private void SubmitNeutralValuesToNetwork() + { + if (!_isWearer || featureInterpolator == null) + { + return; + } + + foreach (var baseIndex in _addessIdToBaseIndex.Values) + { + featureInterpolator.SubmitAbsolute(baseIndex, 0f); + } } private void ResetAllBlendshapesToZero() { + if (_computedActuators == null) + { + return; + } + foreach (var computedActuator in _computedActuators) { foreach (var target in computedActuator.Targets) diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs index da12f3ff3..61751f262 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs @@ -39,8 +39,14 @@ public EyeTrackingBoneActuation() public float _fEyeLeftX; public float _fEyeRightX; public float _fEyeY; - public bool _anyAddressUpdated; public bool IsLocal; + public BasisNetworkReceiver Receiver = null; + + private bool _eyeFollowDriverApplicable; + private bool _trackingActive; + private bool _registeredSourceAddresses; + private FaceTrackingActivityRelay _activityRelay; + #region NetworkingFields // Can be null due to: // - Application with no network, or @@ -49,21 +55,24 @@ public EyeTrackingBoneActuation() // - Becomes non-null after HVRAvatarComms.OnAvatarNetworkReady is successfully invoked [NonSerialized] internal MutualizedFeatureInterpolator featureInterpolator; #endregion - public BasisNetworkReceiver Receiver = null; - private bool _eyeFollowDriverApplicable; private void Awake() { if (avatar == null) avatar = HVRCommsUtil.GetAvatar(this); if (acquisition == null) acquisition = AcquisitionService.SceneInstance; + _activityRelay = FaceTrackingActivityRelay.GetOrCreate(avatar); } public void OnHVRAvatarReady(bool isWearer) { + _registeredSourceAddresses = isWearer; + _eyeFollowDriverApplicable = isWearer; + _trackingActive = _activityRelay != null && _activityRelay.IsTrackingActive; + + acquisition.RegisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); if (isWearer) { acquisition.RegisterAddresses(OurAddresses, OnAddressUpdated); - _eyeFollowDriverApplicable = true; } } @@ -84,11 +93,27 @@ public void OnHVRReadyBothAvatarAndNetwork(bool isWearer) upper = 1f, }).ToList(); featureInterpolator = CommsNetworking.UsingMutualizedInterpolator(avatar, mutualizedInterpolationRanges, OnInterpolatedDataChanged); + + if (IsLocal) + { + SetBuiltInEyeFollowDriverOverriden(_trackingActive); + if (_trackingActive) + { + SubmitCurrentEyeStateToNetwork(); + } + else + { + SubmitNeutralEyesToNetwork(); + } + } + else if (!_trackingActive) + { + ClearRemoteOverrides(); + } } private void OnEnable() { - SetBuiltInEyeFollowDriverOverriden(true); BasisNetworkTransmitter.AfterAvatarChanges += ForceUpdate; } @@ -96,51 +121,98 @@ private void OnDisable() { SetBuiltInEyeFollowDriverOverriden(false); BasisNetworkTransmitter.AfterAvatarChanges -= ForceUpdate; + ClearRemoteOverrides(); } private void OnDestroy() { - if (IsLocal) + if (acquisition != null) { - acquisition.UnregisterAddresses(OurAddresses, OnAddressUpdated); - - if (IsLocal && Receiver != null) + acquisition.UnregisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); + if (_registeredSourceAddresses) { - Receiver.RemotePlayer.RemoteFaceDriver.OverrideEye = false; - Receiver.RemotePlayer.RemoteFaceDriver.OverrideBlinking = false; + acquisition.UnregisterAddresses(OurAddresses, OnAddressUpdated); } } + + ClearRemoteOverrides(); + SetBuiltInEyeFollowDriverOverriden(false); } private void OnAddressUpdated(int address, float value) { - // FIXME: Temp fix, we'll need to hook to NetworkReady instead. - // This is a quick fix so that we don't need to reupload the avatar. - _anyAddressUpdated = _anyAddressUpdated || value != 0f; - if (_anyAddressUpdated) + if (!_trackingActive) { - BasisLocalEyeDriver.IsEnabled = false; + return; } if (address == EyeLeftXAddress) { _fEyeLeftX = value; - if (featureInterpolator != null) featureInterpolator.SubmitAbsolute(0, value); + if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(0, value); } else if (address == EyeRightXAddress) { _fEyeRightX = value; - if (featureInterpolator != null) featureInterpolator.SubmitAbsolute(1, value); + if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(1, value); } else if (address == EyeYAddress) { _fEyeY = value; - if (featureInterpolator != null) featureInterpolator.SubmitAbsolute(2, value); + if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(2, value); + } + } + + private void OnTrackingActivityUpdated(int address, float value) + { + if (address != FaceTrackingActivityRelay.ActivityAddressId) + { + return; + } + + bool isTrackingActive = value >= 0.5f; + if (_trackingActive == isTrackingActive) + { + return; + } + + _trackingActive = isTrackingActive; + if (IsLocal) + { + SetBuiltInEyeFollowDriverOverriden(_trackingActive); + } + + if (_trackingActive) + { + if (IsLocal) + { + SubmitCurrentEyeStateToNetwork(); + } + return; + } + + _fEyeLeftX = 0f; + _fEyeRightX = 0f; + _fEyeY = 0f; + + if (IsLocal) + { + SubmitNeutralEyesToNetwork(); + } + else + { + SetNeutralRemoteEyes(); + ClearRemoteOverrides(); } } private void ForceUpdate() { + if (!_trackingActive) + { + return; + } + SetEyeRotation(_fEyeLeftX, _fEyeY, EyeSide.Left); SetEyeRotation(_fEyeRightX, _fEyeY, EyeSide.Right); } @@ -166,29 +238,22 @@ private void SetEyeRotation(float x, float y, EyeSide side) throw new ArgumentOutOfRangeException(nameof(side), side, null); } } - else + else if (!IsLocal && Receiver != null) { - if (IsLocal && Receiver != null) + Receiver.RemotePlayer.RemoteFaceDriver.OverrideEye = true; + Receiver.RemotePlayer.RemoteFaceDriver.OverrideBlinking = true; + switch (side) { - Receiver.RemotePlayer.RemoteFaceDriver.OverrideEye = true; - Receiver.RemotePlayer.RemoteFaceDriver.OverrideBlinking = true; - switch (side) - { - case EyeSide.Left: - float result0 = (y + 1) / 2; - float result1 = (x + 1) / 2; - Receiver.EyesAndMouth[0] = result0; - Receiver.EyesAndMouth[1] = result1; - break; - case EyeSide.Right: - result0 = (y + 1) / 2; - result1 = (x + 1) / 2; - Receiver.EyesAndMouth[2] = result0; - Receiver.EyesAndMouth[3] = result1; - break; - default: - throw new ArgumentOutOfRangeException(nameof(side), side, null); - } + case EyeSide.Left: + Receiver.EyesAndMouth[0] = (y + 1) / 2; + Receiver.EyesAndMouth[1] = (x + 1) / 2; + break; + case EyeSide.Right: + Receiver.EyesAndMouth[2] = (y + 1) / 2; + Receiver.EyesAndMouth[3] = (x + 1) / 2; + break; + default: + throw new ArgumentOutOfRangeException(nameof(side), side, null); } } } @@ -198,6 +263,54 @@ private void SetBuiltInEyeFollowDriverOverriden(bool value) BasisLocalEyeDriver.Override = value; } + private void SubmitCurrentEyeStateToNetwork() + { + if (!IsLocal || featureInterpolator == null) + { + return; + } + + featureInterpolator.SubmitAbsolute(0, _fEyeLeftX); + featureInterpolator.SubmitAbsolute(1, _fEyeRightX); + featureInterpolator.SubmitAbsolute(2, _fEyeY); + } + + private void SubmitNeutralEyesToNetwork() + { + if (!IsLocal || featureInterpolator == null) + { + return; + } + + featureInterpolator.SubmitAbsolute(0, 0f); + featureInterpolator.SubmitAbsolute(1, 0f); + featureInterpolator.SubmitAbsolute(2, 0f); + } + + private void SetNeutralRemoteEyes() + { + if (Receiver == null) + { + return; + } + + Receiver.EyesAndMouth[0] = 0.5f; + Receiver.EyesAndMouth[1] = 0.5f; + Receiver.EyesAndMouth[2] = 0.5f; + Receiver.EyesAndMouth[3] = 0.5f; + } + + private void ClearRemoteOverrides() + { + if (IsLocal || Receiver == null) + { + return; + } + + Receiver.RemotePlayer.RemoteFaceDriver.OverrideEye = false; + Receiver.RemotePlayer.RemoteFaceDriver.OverrideBlinking = false; + } + private enum EyeSide { Left, Right @@ -206,6 +319,11 @@ private enum EyeSide #region NetworkingMethods private void OnInterpolatedDataChanged(float[] current) { + if (!_trackingActive || current == null || current.Length < 3) + { + return; + } + _fEyeLeftX = current[0]; _fEyeRightX = current[1]; _fEyeY = current[2]; diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/AutomaticFaceTracking.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/AutomaticFaceTracking.cs index 4639a8c33..2b2bf8edf 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/AutomaticFaceTracking.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/AutomaticFaceTracking.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; @@ -35,6 +35,7 @@ public class AutomaticFaceTracking : MonoBehaviour, IHVRInitializable [NonSerialized] internal OSCAcquisition oscAcquisition; [NonSerialized] internal BlendshapeActuation blendshapeActuation; [NonSerialized] internal EyeTrackingBoneActuation eyeTrackingBoneActuation; + [NonSerialized] internal FaceTrackingActivityRelay faceTrackingActivityRelay; private bool _isWearer; @@ -119,6 +120,9 @@ private void Failed() private void SetupFaceTracking(BlendshapeActuationDefinitionFile[] definitionFiles, List smrs) { renderers = smrs; + faceTrackingActivityRelay = FaceTrackingActivityRelay.GetOrCreate(_avatar); + faceTrackingActivityRelay.OnHVRAvatarReady(_isWearer); + if (_isWearer) { oscAcquisition = CreateOSCAcquisitionIfNotExists(); diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs new file mode 100644 index 000000000..2d8e2b74e --- /dev/null +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs @@ -0,0 +1 @@ +// The FaceTrackingActivityRelay implementation lives in Runtime/Networking/HVRCommsUtil.cs. diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs.meta b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs.meta new file mode 100644 index 000000000..eb7f2e13b --- /dev/null +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 8317c8e0fc6d3eb418345948d6c69a1a \ No newline at end of file diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/HVRAvatarComms.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/HVRAvatarComms.cs index dac972ba7..4e95f3917 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/HVRAvatarComms.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/HVRAvatarComms.cs @@ -52,7 +52,7 @@ private void Awake() private void OnAvatarReady(bool isWearer) { - _isWearer = true; + _isWearer = isWearer; var allInitializables = avatar.GetComponentsInChildren(true); foreach (var initializable in allInitializables) @@ -103,14 +103,16 @@ private void DeclareMutualizedInterpolator(bool isWearer, HVRNetworkingCarrier c _streamedLateInit.transmitter = carrier; _streamedLateInit.isWearer = isWearer; _streamedLateInit.localIdentifier = 0; - _toStoreLater.Clear(); + var pendingStores = _toStoreLater.ToArray(); holder.SetActive(true); + _streamedLateInit.InitializeNormalizedValues(BuildNeutralNormalizedValues()); // StreamedAvatarFeature only gets the ability to store data AFTER Awake() runs, so order matters here. - foreach (var toStoreLater in _toStoreLater) + foreach (var toStoreLater in pendingStores) { var mutualizedIndex = toStoreLater.mutualizedIndex; _streamedLateInit.Store(mutualizedIndex, _ranges[mutualizedIndex].AbsoluteToRange(toStoreLater.absolute)); } + _toStoreLater.Clear(); _streamedLateInit.OnInterpolatedDataChanged += mutualizedData => { @@ -169,6 +171,7 @@ public MutualizedFeatureInterpolator NeedsMutualizedInterpolator(List= 0f + ? Mathf.Clamp01(range.AbsoluteToRange(0f)) + : 0f; + } + + return normalized; + } + private class HVRRedirectToStreamed : IFeatureReceiver { private readonly StreamedAvatarFeature streamed; diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/StreamedAvatarFeature.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/StreamedAvatarFeature.cs index cdc04cffc..4123288fc 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/StreamedAvatarFeature.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/StreamedAvatarFeature.cs @@ -40,9 +40,7 @@ public class StreamedAvatarFeature : MonoBehaviour private void Awake() { - previous ??= new float[valueArraySize]; - target ??= new float[valueArraySize]; - current ??= new float[valueArraySize]; + EnsureBuffers(); } private void OnDisable() @@ -52,6 +50,7 @@ private void OnDisable() public void Store(int index, float value) { + EnsureBuffers(); current[index] = value; if (PrioritizeLargeChanges && isWearer) { @@ -66,6 +65,20 @@ public void Store(int index, float value) } } + public void InitializeNormalizedValues(IReadOnlyList normalizedValues) + { + EnsureBuffers(); + + int count = Mathf.Min(valueArraySize, normalizedValues?.Count ?? 0); + for (int i = 0; i < count; i++) + { + float clamped = Mathf.Clamp01(normalizedValues[i]); + current[i] = clamped; + previous[i] = clamped; + target[i] = clamped; + } + } + /// Exposed for testing purposes. public void QueueEvent(StreamedAvatarFeaturePayload message) { @@ -261,6 +274,13 @@ public void OnResyncRequested(ushort[] whoAsked) FloatValues = current }, whoAsked); } + + private void EnsureBuffers() + { + previous ??= new float[valueArraySize]; + target ??= new float[valueArraySize]; + current ??= new float[valueArraySize]; + } } public class StreamedAvatarFeaturePayload { diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Networking/HVRCommsUtil.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Networking/HVRCommsUtil.cs index f4cb03bcd..dec9693c0 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Networking/HVRCommsUtil.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Networking/HVRCommsUtil.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using Basis.Scripts.BasisSdk; using UnityEngine; @@ -43,4 +44,137 @@ public static T[] SlowSanitizeEndUserProvidedStructArray(T[] structuresNullab return structuresNullable; } } + + [AddComponentMenu("HVR.Basis/Comms/Internal/Face Tracking Activity Relay")] + public class FaceTrackingActivityRelay : MonoBehaviour, IHVRInitializable + { + public const string ActivityAddress = "HVR/Internal/FaceTrackingActive"; + public static readonly int ActivityAddressId = HVRAddress.AddressToId(ActivityAddress); + public const float InactivityTimeoutSeconds = 0.5f; + + [HideInInspector] [SerializeField] private BasisAvatar avatar; + [HideInInspector] [SerializeField] private AcquisitionService acquisition; + + [NonSerialized] internal MutualizedFeatureInterpolator featureInterpolator; + + private bool _isWearer; + private bool _isTrackingActive; + private float _lastActivityTime = float.NegativeInfinity; + + public bool IsTrackingActive => _isTrackingActive; + + public static FaceTrackingActivityRelay GetOrCreate(BasisAvatar avatar) + { + if (avatar == null) + { + return null; + } + + var relay = avatar.GetComponentInChildren(true); + if (relay != null) + { + return relay; + } + + var relayRoot = new GameObject("Generated__FaceTrackingActivityRelay") + { + transform = + { + parent = avatar.transform, + } + }; + return relayRoot.AddComponent(); + } + + private void Awake() + { + if (avatar == null) + { + avatar = HVRCommsUtil.GetAvatar(this); + } + + if (acquisition == null) + { + acquisition = AcquisitionService.SceneInstance; + } + } + + public void OnHVRAvatarReady(bool isWearer) + { + _isWearer = isWearer; + ApplyTrackingState(false, submitToNetwork: false); + } + + public void OnHVRReadyBothAvatarAndNetwork(bool isWearer) + { + _isWearer = isWearer; + featureInterpolator = CommsNetworking.UsingMutualizedInterpolator(avatar, new List + { + new MutualizedInterpolationRange + { + address = ActivityAddressId, + lower = 0f, + upper = 1f, + } + }, OnInterpolatedDataChanged); + + if (_isWearer && featureInterpolator != null) + { + featureInterpolator.SubmitAbsolute(0, _isTrackingActive ? 1f : 0f); + } + } + + private void Update() + { + if (!_isWearer || !_isTrackingActive) + { + return; + } + + if (Time.unscaledTime - _lastActivityTime > InactivityTimeoutSeconds) + { + ApplyTrackingState(false, submitToNetwork: true); + } + } + + public void NotifySourceSample() + { + if (!_isWearer) + { + return; + } + + _lastActivityTime = Time.unscaledTime; + if (!_isTrackingActive) + { + ApplyTrackingState(true, submitToNetwork: true); + } + } + + private void OnInterpolatedDataChanged(float[] current) + { + if (_isWearer || current == null || current.Length == 0) + { + return; + } + + ApplyTrackingState(current[0] >= 0.5f, submitToNetwork: false); + } + + private void ApplyTrackingState(bool isTrackingActive, bool submitToNetwork) + { + bool stateChanged = _isTrackingActive != isTrackingActive; + _isTrackingActive = isTrackingActive; + + if (acquisition != null && (stateChanged || submitToNetwork)) + { + acquisition.Submit(ActivityAddressId, isTrackingActive ? 1f : 0f); + } + + if (submitToNetwork && _isWearer && featureInterpolator != null) + { + featureInterpolator.SubmitAbsolute(0, isTrackingActive ? 1f : 0f); + } + } + } } From a3d76a2df1175e441b3c2818f1774c3079c23572 Mon Sep 17 00:00:00 2001 From: Toys0125 Date: Fri, 6 Mar 2026 22:38:10 -0600 Subject: [PATCH 3/5] Fix potential bug, in which the override might not be called again when enabled. --- .../Runtime/Components/Actuation/EyeTrackingBoneActuation.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs index 61751f262..e1f0ab042 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs @@ -114,6 +114,10 @@ public void OnHVRReadyBothAvatarAndNetwork(bool isWearer) private void OnEnable() { + if (_trackingActive && _eyeFollowDriverApplicable) + { + SetBuiltInEyeFollowDriverOverriden(true); + } BasisNetworkTransmitter.AfterAvatarChanges += ForceUpdate; } From bbe6deaf601d6e767636b998f8b039c3b178e9b0 Mon Sep 17 00:00:00 2001 From: Toys0125 Date: Sat, 7 Mar 2026 14:16:29 -0600 Subject: [PATCH 4/5] Reset changes not related to face tracking back to developer --- .../com.basis.eventdriver/BasisEventDriver.cs | 5 +- .../Networking/BasisRemoteFaceManagement.cs | 2 +- .../com.hecomi.ulipsync/BasisUlipSync.cs | 61 +++++++------------ .../Runtime/SteamAudioManager.cs | 24 +++----- 4 files changed, 33 insertions(+), 59 deletions(-) diff --git a/Basis/Packages/com.basis.eventdriver/BasisEventDriver.cs b/Basis/Packages/com.basis.eventdriver/BasisEventDriver.cs index 15e6af9d0..4a322bf24 100644 --- a/Basis/Packages/com.basis.eventdriver/BasisEventDriver.cs +++ b/Basis/Packages/com.basis.eventdriver/BasisEventDriver.cs @@ -230,12 +230,10 @@ public void LateUpdate() /// private void OnBeforeRender() { - // Ensure remote face jobs are completed/applied even before local player is ready. - BasisRemoteFaceManagement.Apply(); //apply blendshapes - if (BasisLocalPlayer.PlayerReady) { BasisLocalPlayer.Instance.SimulateOnRender(); + BasisRemoteFaceManagement.Apply(); //apply blendshapes BasisLocalCameraDriver.Instance.microphoneIconDriver.Simulate(DeltaTime); //update microphone icon } } @@ -277,4 +275,3 @@ public void OnDrawGizmosSelected() #endif } } - diff --git a/Basis/Packages/com.basis.framework/Networking/BasisRemoteFaceManagement.cs b/Basis/Packages/com.basis.framework/Networking/BasisRemoteFaceManagement.cs index 502abf207..cbc2c1285 100644 --- a/Basis/Packages/com.basis.framework/Networking/BasisRemoteFaceManagement.cs +++ b/Basis/Packages/com.basis.framework/Networking/BasisRemoteFaceManagement.cs @@ -90,7 +90,7 @@ public static void Simulate(double t,float dt) blinkOut = blinkOut, }; - handle = job.Schedule(count, BatchSize, handle); + handle = job.Schedule(count, BatchSize); } public static void Apply() diff --git a/Basis/Packages/com.hecomi.ulipsync/BasisUlipSync.cs b/Basis/Packages/com.hecomi.ulipsync/BasisUlipSync.cs index 1d4ece47e..d866195f4 100644 --- a/Basis/Packages/com.hecomi.ulipsync/BasisUlipSync.cs +++ b/Basis/Packages/com.hecomi.ulipsync/BasisUlipSync.cs @@ -52,7 +52,6 @@ public unsafe class BasisUlipSync BasisDctPlan _dctPlan; float[] _lastApplied; - readonly object _bufferSwapLock = new object(); public struct BlendMap { @@ -143,13 +142,6 @@ public void Execute() public void Simulate(float DeltaTime) { - if (HasJob) - { - _jobHandle.Complete(); - _jobHandle = default; - HasJob = false; - } - if (Interlocked.Exchange(ref _isDataReceived, 0) != 1) return; if (!_allocated) return; @@ -157,23 +149,16 @@ public void Simulate(float DeltaTime) if (!BasisUlipSyncDriver.IsInitialized) return; - int oldActive; - int frozenStartIndex; - NativeArray frozenInput; - - lock (_bufferSwapLock) - { - oldActive = _activeInputBuffer; - int newActive = oldActive ^ 1; + int oldActive = _activeInputBuffer; + int newActive = oldActive ^ 1; - Volatile.Write(ref _activeInputBuffer, newActive); + Volatile.Write(ref _activeInputBuffer, newActive); - frozenStartIndex = oldActive == 0 - ? Volatile.Read(ref _writeIndexA) - : Volatile.Read(ref _writeIndexB); + int frozenStartIndex = oldActive == 0 + ? Volatile.Read(ref _writeIndexA) + : Volatile.Read(ref _writeIndexB); - frozenInput = oldActive == 0 ? _inputA : _inputB; - } + NativeArray frozenInput = oldActive == 0 ? _inputA : _inputB; byte normalizeScores = (byte)0; @@ -251,7 +236,6 @@ public void Apply() HasJob = false; _jobHandle.Complete(); - _jobHandle = default; if (_mfccForOther.IsCreated && _mfcc.IsCreated) { @@ -457,27 +441,24 @@ public void OnDataReceived(float[] input, int channels, int length) int ch = math.max(channels, 1); - lock (_bufferSwapLock) - { - int buf = Volatile.Read(ref _activeInputBuffer); - NativeArray dstArr = (buf == 0) ? _inputA : _inputB; - - float* dst = (float*)NativeArrayUnsafeUtility.GetUnsafePtr(dstArr); + int buf = Volatile.Read(ref _activeInputBuffer); + NativeArray dstArr = (buf == 0) ? _inputA : _inputB; - fixed (float* src = input) - { - int w = (buf == 0) ? Volatile.Read(ref _writeIndexA) : Volatile.Read(ref _writeIndexB); + float* dst = (float*)NativeArrayUnsafeUtility.GetUnsafePtr(dstArr); - for (int s = 0; s < length; s += ch) - { - dst[w] = src[s]; - w++; - if (w == cap) w = 0; - } + fixed (float* src = input) + { + int w = (buf == 0) ? Volatile.Read(ref _writeIndexA) : Volatile.Read(ref _writeIndexB); - if (buf == 0) Volatile.Write(ref _writeIndexA, w); - else Volatile.Write(ref _writeIndexB, w); + for (int s = 0; s < length; s += ch) + { + dst[w] = src[s]; + w++; + if (w == cap) w = 0; } + + if (buf == 0) Volatile.Write(ref _writeIndexA, w); + else Volatile.Write(ref _writeIndexB, w); } Interlocked.Exchange(ref _isDataReceived, 1); diff --git a/Basis/Packages/com.steam.steamaudio/Runtime/SteamAudioManager.cs b/Basis/Packages/com.steam.steamaudio/Runtime/SteamAudioManager.cs index 47432bea8..30274baff 100644 --- a/Basis/Packages/com.steam.steamaudio/Runtime/SteamAudioManager.cs +++ b/Basis/Packages/com.steam.steamaudio/Runtime/SteamAudioManager.cs @@ -549,9 +549,8 @@ private void ScheduleInstance() EnsureSourceCapacity(CurrentArraySource); EnsureListenerCapacity(CurrentArrayListener); - JobHandle priorHandle = combined; - JobHandle sourcesHandle = priorHandle; - JobHandle listenersHandle = priorHandle; + JobHandle sourcesHandle = default; + JobHandle listenersHandle = default; if (CurrentArraySource > 0 && mSourceTransforms.isCreated) { @@ -559,7 +558,7 @@ private void ScheduleInstance() { PoseData = mSourceGathers, }; - sourcesHandle = job.Schedule(mSourceTransforms, priorHandle); + sourcesHandle = job.Schedule(mSourceTransforms); } if (CurrentArrayListener > 0 && mListenerTransforms.isCreated) @@ -568,17 +567,13 @@ private void ScheduleInstance() { PoseData = mListenerGathers, }; - listenersHandle = job.Schedule(mListenerTransforms, priorHandle); + listenersHandle = job.Schedule(mListenerTransforms); } - combined = JobHandle.CombineDependencies(sourcesHandle, listenersHandle); } public JobHandle combined; private void ApplyInstance() { - // Always complete pose gather jobs even when later early-returns skip simulation work. - combined.Complete(); - if (mAudioEngineState == null) return; @@ -626,6 +621,10 @@ private void ApplyInstance() mSimulator.SetSharedInputs(SimulationFlags.Direct, sharedInputs); + + // Complete before calling into Steam Audio (main-thread plugin calls) + combined.Complete(); + // --- Direct inputs from cached pose arrays --- for (int i = 0; i < CurrentArraySource; i++) { @@ -1012,8 +1011,7 @@ private void EnsureSourceCapacity(int required) int newCap = (mSourceCapacity <= 0) ? 8 : mSourceCapacity * 2; if (newCap < required) newCap = required; - // Dispose old arrays if created. Ensure no gather jobs are still using them. - combined.Complete(); + // Dispose old arrays if created if (mSourceGathers.IsCreated) mSourceGathers.Dispose(); mSourceGathers = new NativeArray(newCap, Allocator.Persistent); @@ -1029,7 +1027,6 @@ private void EnsureListenerCapacity(int required) int newCap = (mListenerCapacity <= 0) ? 4 : mListenerCapacity * 2; if (newCap < required) newCap = required; - combined.Complete(); if (mListenerGathers.IsCreated) mListenerGathers.Dispose(); mListenerGathers = new NativeArray(newCap, Allocator.Persistent); @@ -1052,9 +1049,8 @@ private void DisposeTransformAndPoseBuffers() if (mSourceTransforms.isCreated) mSourceTransforms.Dispose(); if (mListenerTransforms.isCreated) mListenerTransforms.Dispose(); - combined.Complete(); - if (mSourceGathers.IsCreated) mSourceGathers.Dispose(); + if (mListenerGathers.IsCreated) mListenerGathers.Dispose(); mSourceCapacity = 0; From d99b68e98f076a0daf79f5a97e5e22aa8f7ca0ed Mon Sep 17 00:00:00 2001 From: Toys0125 Date: Sat, 7 Mar 2026 19:10:49 -0600 Subject: [PATCH 5/5] Fix(face-tracking): fallback to default Basis eye driver when eye params are missing --- .../Actuation/EyeTrackingBoneActuation.cs | 232 ++++++++++++++---- 1 file changed, 179 insertions(+), 53 deletions(-) diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs index e1f0ab042..5f5230edc 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs @@ -5,7 +5,7 @@ using Basis.Scripts.Networking.Transmitters; using HVR.Basis.Comms.HVRUtility; using System; -using System.Linq; +using System.Collections.Generic; using Unity.Mathematics; using UnityEngine; @@ -18,18 +18,19 @@ public class EyeTrackingBoneActuation : BasisAvatarMonoBehaviour, IHVRInitializa private const string EyeLeftX = "FT/v2/EyeLeftX"; private const string EyeRightX = "FT/v2/EyeRightX"; private const string EyeY = "FT/v2/EyeY"; - private readonly int EyeLeftXAddress; - private readonly int EyeRightXAddress; - private readonly int EyeYAddress; - private int[] OurAddresses; + private const string EyeTrackingActive = "HVR/Internal/EyeTrackingActive"; + private const float EyeParameterInactivityTimeoutSeconds = 0.5f; - public EyeTrackingBoneActuation() - { - EyeLeftXAddress = HVRAddress.AddressToId(EyeLeftX); - EyeRightXAddress = HVRAddress.AddressToId(EyeRightX); - EyeYAddress = HVRAddress.AddressToId(EyeY); - OurAddresses = new[] { EyeLeftXAddress, EyeRightXAddress, EyeYAddress }; - } + private const int LeftEyeFeatureIndex = 0; + private const int RightEyeFeatureIndex = 1; + private const int EyeYFeatureIndex = 2; + private const int EyeTrackingActiveFeatureIndex = 3; + + private readonly int _eyeLeftXAddress; + private readonly int _eyeRightXAddress; + private readonly int _eyeYAddress; + private readonly int _eyeTrackingActiveAddress; + private readonly int[] _sourceEyeAddresses; [HideInInspector] [SerializeField] private BasisAvatar avatar; [HideInInspector] [SerializeField] private AcquisitionService acquisition; @@ -44,17 +45,28 @@ public EyeTrackingBoneActuation() private bool _eyeFollowDriverApplicable; private bool _trackingActive; + private bool _eyeTrackingParametersActive; + private float _lastEyeParameterSampleTime = float.NegativeInfinity; private bool _registeredSourceAddresses; private FaceTrackingActivityRelay _activityRelay; - #region NetworkingFields +#region NetworkingFields // Can be null due to: // - Application with no network, or // - Network late initialization. // Nullability is needed for local tests without initialization scene. // - Becomes non-null after HVRAvatarComms.OnAvatarNetworkReady is successfully invoked [NonSerialized] internal MutualizedFeatureInterpolator featureInterpolator; - #endregion +#endregion + + public EyeTrackingBoneActuation() + { + _eyeLeftXAddress = HVRAddress.AddressToId(EyeLeftX); + _eyeRightXAddress = HVRAddress.AddressToId(EyeRightX); + _eyeYAddress = HVRAddress.AddressToId(EyeY); + _eyeTrackingActiveAddress = HVRAddress.AddressToId(EyeTrackingActive); + _sourceEyeAddresses = new[] { _eyeLeftXAddress, _eyeRightXAddress, _eyeYAddress }; + } private void Awake() { @@ -68,11 +80,13 @@ public void OnHVRAvatarReady(bool isWearer) _registeredSourceAddresses = isWearer; _eyeFollowDriverApplicable = isWearer; _trackingActive = _activityRelay != null && _activityRelay.IsTrackingActive; + _eyeTrackingParametersActive = false; + _lastEyeParameterSampleTime = float.NegativeInfinity; acquisition.RegisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); if (isWearer) { - acquisition.RegisterAddresses(OurAddresses, OnAddressUpdated); + acquisition.RegisterAddresses(_sourceEyeAddresses, OnAddressUpdated); } } @@ -86,18 +100,20 @@ public void OnHVRReadyBothAvatarAndNetwork(bool isWearer) Receiver = NetworkedPlayer as BasisNetworkReceiver; } - var mutualizedInterpolationRanges = OurAddresses.Select(address => new MutualizedInterpolationRange + var mutualizedInterpolationRanges = new List { - address = address, - lower = -1f, - upper = 1f, - }).ToList(); + new MutualizedInterpolationRange { address = _eyeLeftXAddress, lower = -1f, upper = 1f }, + new MutualizedInterpolationRange { address = _eyeRightXAddress, lower = -1f, upper = 1f }, + new MutualizedInterpolationRange { address = _eyeYAddress, lower = -1f, upper = 1f }, + new MutualizedInterpolationRange { address = _eyeTrackingActiveAddress, lower = 0f, upper = 1f } + }; featureInterpolator = CommsNetworking.UsingMutualizedInterpolator(avatar, mutualizedInterpolationRanges, OnInterpolatedDataChanged); - + bool shouldApply = ShouldApplyEyeTracking(); if (IsLocal) { - SetBuiltInEyeFollowDriverOverriden(_trackingActive); - if (_trackingActive) + SubmitEyeTrackingParameterStateToNetwork(); + SetBuiltInEyeFollowDriverOverriden(shouldApply); + if (shouldApply) { SubmitCurrentEyeStateToNetwork(); } @@ -106,7 +122,7 @@ public void OnHVRReadyBothAvatarAndNetwork(bool isWearer) SubmitNeutralEyesToNetwork(); } } - else if (!_trackingActive) + else if (!shouldApply) { ClearRemoteOverrides(); } @@ -114,7 +130,7 @@ public void OnHVRReadyBothAvatarAndNetwork(bool isWearer) private void OnEnable() { - if (_trackingActive && _eyeFollowDriverApplicable) + if (ShouldApplyEyeTracking() && _eyeFollowDriverApplicable) { SetBuiltInEyeFollowDriverOverriden(true); } @@ -135,7 +151,7 @@ private void OnDestroy() acquisition.UnregisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); if (_registeredSourceAddresses) { - acquisition.UnregisterAddresses(OurAddresses, OnAddressUpdated); + acquisition.UnregisterAddresses(_sourceEyeAddresses, OnAddressUpdated); } } @@ -143,27 +159,55 @@ private void OnDestroy() SetBuiltInEyeFollowDriverOverriden(false); } - private void OnAddressUpdated(int address, float value) + private void Update() { - if (!_trackingActive) + if (!IsLocal || !_trackingActive || !_eyeTrackingParametersActive) { return; } - if (address == EyeLeftXAddress) + if (Time.unscaledTime - _lastEyeParameterSampleTime > EyeParameterInactivityTimeoutSeconds) + { + SetLocalEyeParameterState(false); + SetBuiltInEyeFollowDriverOverriden(false); + SubmitNeutralEyesToNetwork(); + } + } + + private void OnAddressUpdated(int address, float value) + { + if (!_trackingActive) { - _fEyeLeftX = value; - if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(0, value); + return; } - else if (address == EyeRightXAddress) + + float sanitizedValue = SanitizeAndClampEyeValue(value); + switch (address) { - _fEyeRightX = value; - if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(1, value); + case var _ when address == _eyeLeftXAddress: + _fEyeLeftX = sanitizedValue; + if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(LeftEyeFeatureIndex, sanitizedValue); + break; + case var _ when address == _eyeRightXAddress: + _fEyeRightX = sanitizedValue; + if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(RightEyeFeatureIndex, sanitizedValue); + break; + case var _ when address == _eyeYAddress: + _fEyeY = sanitizedValue; + if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(EyeYFeatureIndex, sanitizedValue); + break; + default: + return; } - else if (address == EyeYAddress) + + if (IsLocal) { - _fEyeY = value; - if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(2, value); + _lastEyeParameterSampleTime = Time.unscaledTime; + if (!_eyeTrackingParametersActive) + { + SetLocalEyeParameterState(true); + SetBuiltInEyeFollowDriverOverriden(true); + } } } @@ -181,23 +225,39 @@ private void OnTrackingActivityUpdated(int address, float value) } _trackingActive = isTrackingActive; + if (IsLocal && !_trackingActive) + { + SetLocalEyeParameterState(false); + } + + bool shouldApplyEyeTracking = ShouldApplyEyeTracking(); if (IsLocal) { - SetBuiltInEyeFollowDriverOverriden(_trackingActive); + SetBuiltInEyeFollowDriverOverriden(shouldApplyEyeTracking); } if (_trackingActive) { if (IsLocal) { - SubmitCurrentEyeStateToNetwork(); + if (shouldApplyEyeTracking) + { + SubmitCurrentEyeStateToNetwork(); + } + else + { + SubmitNeutralEyesToNetwork(); + } + } + else if (!shouldApplyEyeTracking) + { + ClearRemoteOverrides(); } return; } - _fEyeLeftX = 0f; - _fEyeRightX = 0f; - _fEyeY = 0f; + ResetEyeValuesToZero(); + _eyeTrackingParametersActive = false; if (IsLocal) { @@ -212,7 +272,7 @@ private void OnTrackingActivityUpdated(int address, float value) private void ForceUpdate() { - if (!_trackingActive) + if (!ShouldApplyEyeTracking()) { return; } @@ -223,6 +283,9 @@ private void ForceUpdate() private void SetEyeRotation(float x, float y, EyeSide side) { + x = SanitizeAndClampEyeValue(x); + y = SanitizeAndClampEyeValue(y); + if (_eyeFollowDriverApplicable) { var xDeg = Mathf.Asin(x) * Mathf.Rad2Deg * multiplyX; @@ -274,9 +337,9 @@ private void SubmitCurrentEyeStateToNetwork() return; } - featureInterpolator.SubmitAbsolute(0, _fEyeLeftX); - featureInterpolator.SubmitAbsolute(1, _fEyeRightX); - featureInterpolator.SubmitAbsolute(2, _fEyeY); + featureInterpolator.SubmitAbsolute(LeftEyeFeatureIndex, SanitizeAndClampEyeValue(_fEyeLeftX)); + featureInterpolator.SubmitAbsolute(RightEyeFeatureIndex, SanitizeAndClampEyeValue(_fEyeRightX)); + featureInterpolator.SubmitAbsolute(EyeYFeatureIndex, SanitizeAndClampEyeValue(_fEyeY)); } private void SubmitNeutralEyesToNetwork() @@ -286,9 +349,48 @@ private void SubmitNeutralEyesToNetwork() return; } - featureInterpolator.SubmitAbsolute(0, 0f); - featureInterpolator.SubmitAbsolute(1, 0f); - featureInterpolator.SubmitAbsolute(2, 0f); + featureInterpolator.SubmitAbsolute(LeftEyeFeatureIndex, 0f); + featureInterpolator.SubmitAbsolute(RightEyeFeatureIndex, 0f); + featureInterpolator.SubmitAbsolute(EyeYFeatureIndex, 0f); + } + + private void SetLocalEyeParameterState(bool isActive) + { + _eyeTrackingParametersActive = isActive; + _lastEyeParameterSampleTime = isActive ? Time.unscaledTime : float.NegativeInfinity; + SubmitEyeTrackingParameterStateToNetwork(); + } + + private void SubmitEyeTrackingParameterStateToNetwork() + { + if (!IsLocal || featureInterpolator == null) + { + return; + } + + featureInterpolator.SubmitAbsolute(EyeTrackingActiveFeatureIndex, _eyeTrackingParametersActive ? 1f : 0f); + } + + private bool ShouldApplyEyeTracking() + { + return _trackingActive && _eyeTrackingParametersActive; + } + + private static float SanitizeAndClampEyeValue(float value) + { + if (float.IsNaN(value) || float.IsInfinity(value)) + { + return 0f; + } + + return Mathf.Clamp(value, -1f, 1f); + } + + private void ResetEyeValuesToZero() + { + _fEyeLeftX = 0f; + _fEyeRightX = 0f; + _fEyeY = 0f; } private void SetNeutralRemoteEyes() @@ -323,15 +425,39 @@ private enum EyeSide #region NetworkingMethods private void OnInterpolatedDataChanged(float[] current) { - if (!_trackingActive || current == null || current.Length < 3) + if (current == null) { return; } - _fEyeLeftX = current[0]; - _fEyeRightX = current[1]; - _fEyeY = current[2]; + if (!IsLocal) + { + if (current.Length > EyeTrackingActiveFeatureIndex) + { + _eyeTrackingParametersActive = current[EyeTrackingActiveFeatureIndex] >= 0.5f; + } + else + { + // Legacy compatibility with senders that only stream 3 values. + _eyeTrackingParametersActive = true; + } + } + + bool shouldApply = ShouldApplyEyeTracking(); + if (!shouldApply || current.Length < 3) + { + if (!IsLocal && !shouldApply) + { + ClearRemoteOverrides(); + } + return; + } + + _fEyeLeftX = SanitizeAndClampEyeValue(current[LeftEyeFeatureIndex]); + _fEyeRightX = SanitizeAndClampEyeValue(current[RightEyeFeatureIndex]); + _fEyeY = SanitizeAndClampEyeValue(current[EyeYFeatureIndex]); } #endregion } } +