diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs index a53149cf2..ab972378b 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs @@ -1,4 +1,4 @@ -using Basis.Scripts.BasisSdk; +using Basis.Scripts.BasisSdk; using UnityEngine; namespace HVR.Basis.Comms @@ -13,11 +13,13 @@ public class OSCAcquisition : MonoBehaviour private OSCAcquisitionServer _acquisitionServer; private bool _alreadyInitialized; + private FaceTrackingActivityRelay _activityRelay; private void Awake() { if (avatar == null) avatar = HVRCommsUtil.GetAvatar(this); if (acquisitionService == null) acquisitionService = AcquisitionService.SceneInstance; + _activityRelay = FaceTrackingActivityRelay.GetOrCreate(avatar); avatar.OnAvatarReady -= OnAvatarReady; avatar.OnAvatarReady += OnAvatarReady; @@ -55,6 +57,7 @@ private void OnAddressUpdated(string address, float value) { if (!isActiveAndEnabled) return; + _activityRelay?.NotifySourceSample(); acquisitionService.Submit(HVRAddress.AddressToId(address), value); } } diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs index da04f2fe9..fbd472a63 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs @@ -22,9 +22,14 @@ public class BlendshapeActuation : MonoBehaviour, IHVRInitializable [HideInInspector] [SerializeField] private AcquisitionService acquisition; private Dictionary _addessIdToBaseIndex = new(); + private readonly Dictionary _latestAbsoluteByAddress = new(); private ComputedActuator[] _computedActuators; private ComputedActuator[][] _addressBaseIndexToActuators; private Dictionary _addressToStreamedLowerUpper; + private AddressOverride[] _defaultOverrides = Array.Empty(); + private FaceTrackingActivityRelay _activityRelay; + private bool _isWearer; + private bool _trackingActive; #region NetworkingFields // Can be null due to: @@ -56,6 +61,7 @@ private void Awake() acquisition = AcquisitionService.SceneInstance; } + _activityRelay = FaceTrackingActivityRelay.GetOrCreate(avatar); renderers = HVRCommsUtil.SlowSanitizeEndUserProvidedObjectArray(renderers); definitionFiles = HVRCommsUtil.SlowSanitizeEndUserProvidedObjectArray(definitionFiles); definitions = HVRCommsUtil.SlowSanitizeEndUserProvidedStructArray(definitions); @@ -63,26 +69,16 @@ private void Awake() private void OnAddressUpdated(int address, float inRange) { - if (!_addessIdToBaseIndex.TryGetValue(address, out var baseIndex)) return; - - // TODO: Might need to queue and delay this change so that it executes on the Update loop. - - var actuatorsForThisAddress = _addressBaseIndexToActuators[baseIndex]; - if (actuatorsForThisAddress == null) return; // There may be no actuator for an address when it does not exist in the renderers. - - foreach (var actuator in actuatorsForThisAddress) - { - Actuate(actuator, inRange); - } - - if (featureInterpolator != null) - { - featureInterpolator.SubmitAbsolute(baseIndex, inRange); - } + ApplyAddressValue(address, inRange, forwardToNetwork: _isWearer); } private void OnInterpolatedDataChanged(float[] current) { + if (!_trackingActive || current == null) + { + return; + } + foreach (var actuator in _computedActuators) { var absolute = current[actuator.AddressIndex]; @@ -112,6 +108,10 @@ private static void Actuate(ComputedActuator actuator, float inRange) public void OnHVRAvatarReady(bool isWearer) { + _isWearer = isWearer; + acquisition.RegisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); + _trackingActive = _activityRelay != null && _activityRelay.IsTrackingActive; + var allDefinitions = definitions .Concat(definitionFiles.SelectMany(file => file.definitions)) .ToArray(); @@ -128,7 +128,7 @@ public void OnHVRAvatarReady(bool isWearer) var inValuesForThisAddress = grouping // Reminder that InStart may be greater than InEnd. // We want the lower bound, not the minimum of InStart. - .SelectMany(definition => new [] { definition.inStart, definition.inEnd }) + .SelectMany(definition => new[] { definition.inStart, definition.inEnd }) .ToArray(); return (inValuesForThisAddress.Min(), inValuesForThisAddress.Max()); }); @@ -192,6 +192,12 @@ public void OnHVRAvatarReady(bool isWearer) _addressBaseIndexToActuators[computedActuator.Key] = computedActuator.ToArray(); } + _defaultOverrides = definitionFiles + .SelectMany(file => file.addressOverrides) + .Concat(addressOverrides) + .Where(it => it.overrideDefaultValue) + .ToArray(); + if (isWearer) { acquisition.RegisterAddresses(_addessIdToBaseIndex.Keys.ToArray(), OnAddressUpdated); @@ -215,20 +221,21 @@ public static Dictionary> ResolveSmrToBlendsha public void OnHVRReadyBothAvatarAndNetwork(bool isLocallyOwned) { HVRLogging.ProtocolDebug("OnReadyBothAvatarAndNetwork called on BlendshapeActuation."); + _isWearer = isLocallyOwned; // FIXME: We should be using the computed actuators instead of the address base, assuming that // the list of blendshapes is the same local and remote (no local-only or remote-only blendshapes). featureInterpolator = CommsNetworking.UsingMutualizedInterpolator(avatar, MakeMutualized(), OnInterpolatedDataChanged); - var overrides = definitionFiles - .SelectMany(file => file.addressOverrides) - .Concat(addressOverrides) - .Where(it => it.overrideDefaultValue) - .ToArray(); - foreach (var addressOverride in overrides) + if (_isWearer) { - if (_addessIdToBaseIndex.TryGetValue(HVRAddress.AddressToId(addressOverride.address), out var key)) + if (_trackingActive) { - featureInterpolator.SubmitAbsolute(key, addressOverride.defaultValue); + SubmitDefaultOverridesToNetwork(); + ReplayLatestTrackedValuesToNetwork(); + } + else + { + SubmitNeutralValuesToNetwork(); } } } @@ -272,13 +279,138 @@ private void OnDisable() private void OnDestroy() { - avatar.OnAvatarReady -= OnHVRAvatarReady; + if (avatar != null) + { + avatar.OnAvatarReady -= OnHVRAvatarReady; + } + + if (acquisition != null) + { + acquisition.UnregisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); + if (_isWearer && _addessIdToBaseIndex.Count > 0) + { + acquisition.UnregisterAddresses(_addessIdToBaseIndex.Keys.ToArray(), OnAddressUpdated); + } + } + } + + private void OnTrackingActivityUpdated(int address, float value) + { + if (address != FaceTrackingActivityRelay.ActivityAddressId) + { + return; + } + + bool isTrackingActive = value >= 0.5f; + if (_trackingActive == isTrackingActive) + { + return; + } + + _trackingActive = isTrackingActive; + if (_trackingActive) + { + if (_isWearer) + { + ApplyDefaultOverrides(); + ReplayLatestTrackedValuesToNetwork(); + } + return; + } - acquisition.UnregisterAddresses(_addessIdToBaseIndex.Keys.ToArray(), OnAddressUpdated); + ResetAllBlendshapesToZero(); + _latestAbsoluteByAddress.Clear(); + if (_isWearer) + { + SubmitNeutralValuesToNetwork(); + } + } + + private void ApplyAddressValue(int address, float inRange, bool forwardToNetwork) + { + if (!_trackingActive || !_addessIdToBaseIndex.TryGetValue(address, out var baseIndex)) + { + return; + } + + var actuatorsForThisAddress = _addressBaseIndexToActuators[baseIndex]; + if (actuatorsForThisAddress == null) + { + return; + } + + _latestAbsoluteByAddress[address] = inRange; + foreach (var actuator in actuatorsForThisAddress) + { + Actuate(actuator, inRange); + } + + if (forwardToNetwork && _isWearer && featureInterpolator != null) + { + featureInterpolator.SubmitAbsolute(baseIndex, inRange); + } + } + + private void ApplyDefaultOverrides() + { + foreach (var addressOverride in _defaultOverrides) + { + ApplyAddressValue(HVRAddress.AddressToId(addressOverride.address), addressOverride.defaultValue, forwardToNetwork: true); + } + } + + private void ReplayLatestTrackedValuesToNetwork() + { + if (!_isWearer || featureInterpolator == null) + { + return; + } + + foreach (var pair in _latestAbsoluteByAddress) + { + if (_addessIdToBaseIndex.TryGetValue(pair.Key, out var baseIndex)) + { + featureInterpolator.SubmitAbsolute(baseIndex, pair.Value); + } + } + } + + private void SubmitDefaultOverridesToNetwork() + { + if (!_isWearer || featureInterpolator == null) + { + return; + } + + foreach (var addressOverride in _defaultOverrides) + { + if (_addessIdToBaseIndex.TryGetValue(HVRAddress.AddressToId(addressOverride.address), out var baseIndex)) + { + featureInterpolator.SubmitAbsolute(baseIndex, addressOverride.defaultValue); + } + } + } + + private void SubmitNeutralValuesToNetwork() + { + if (!_isWearer || featureInterpolator == null) + { + return; + } + + foreach (var baseIndex in _addessIdToBaseIndex.Values) + { + featureInterpolator.SubmitAbsolute(baseIndex, 0f); + } } private void ResetAllBlendshapesToZero() { + if (_computedActuators == null) + { + return; + } + foreach (var computedActuator in _computedActuators) { foreach (var target in computedActuator.Targets) diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs index da12f3ff3..5f5230edc 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/EyeTrackingBoneActuation.cs @@ -5,7 +5,7 @@ using Basis.Scripts.Networking.Transmitters; using HVR.Basis.Comms.HVRUtility; using System; -using System.Linq; +using System.Collections.Generic; using Unity.Mathematics; using UnityEngine; @@ -18,18 +18,19 @@ public class EyeTrackingBoneActuation : BasisAvatarMonoBehaviour, IHVRInitializa private const string EyeLeftX = "FT/v2/EyeLeftX"; private const string EyeRightX = "FT/v2/EyeRightX"; private const string EyeY = "FT/v2/EyeY"; - private readonly int EyeLeftXAddress; - private readonly int EyeRightXAddress; - private readonly int EyeYAddress; - private int[] OurAddresses; + private const string EyeTrackingActive = "HVR/Internal/EyeTrackingActive"; + private const float EyeParameterInactivityTimeoutSeconds = 0.5f; - public EyeTrackingBoneActuation() - { - EyeLeftXAddress = HVRAddress.AddressToId(EyeLeftX); - EyeRightXAddress = HVRAddress.AddressToId(EyeRightX); - EyeYAddress = HVRAddress.AddressToId(EyeY); - OurAddresses = new[] { EyeLeftXAddress, EyeRightXAddress, EyeYAddress }; - } + private const int LeftEyeFeatureIndex = 0; + private const int RightEyeFeatureIndex = 1; + private const int EyeYFeatureIndex = 2; + private const int EyeTrackingActiveFeatureIndex = 3; + + private readonly int _eyeLeftXAddress; + private readonly int _eyeRightXAddress; + private readonly int _eyeYAddress; + private readonly int _eyeTrackingActiveAddress; + private readonly int[] _sourceEyeAddresses; [HideInInspector] [SerializeField] private BasisAvatar avatar; [HideInInspector] [SerializeField] private AcquisitionService acquisition; @@ -39,31 +40,53 @@ public EyeTrackingBoneActuation() public float _fEyeLeftX; public float _fEyeRightX; public float _fEyeY; - public bool _anyAddressUpdated; public bool IsLocal; - #region NetworkingFields + public BasisNetworkReceiver Receiver = null; + + private bool _eyeFollowDriverApplicable; + private bool _trackingActive; + private bool _eyeTrackingParametersActive; + private float _lastEyeParameterSampleTime = float.NegativeInfinity; + private bool _registeredSourceAddresses; + private FaceTrackingActivityRelay _activityRelay; + +#region NetworkingFields // Can be null due to: // - Application with no network, or // - Network late initialization. // Nullability is needed for local tests without initialization scene. // - Becomes non-null after HVRAvatarComms.OnAvatarNetworkReady is successfully invoked [NonSerialized] internal MutualizedFeatureInterpolator featureInterpolator; - #endregion - public BasisNetworkReceiver Receiver = null; - private bool _eyeFollowDriverApplicable; +#endregion + + public EyeTrackingBoneActuation() + { + _eyeLeftXAddress = HVRAddress.AddressToId(EyeLeftX); + _eyeRightXAddress = HVRAddress.AddressToId(EyeRightX); + _eyeYAddress = HVRAddress.AddressToId(EyeY); + _eyeTrackingActiveAddress = HVRAddress.AddressToId(EyeTrackingActive); + _sourceEyeAddresses = new[] { _eyeLeftXAddress, _eyeRightXAddress, _eyeYAddress }; + } private void Awake() { if (avatar == null) avatar = HVRCommsUtil.GetAvatar(this); if (acquisition == null) acquisition = AcquisitionService.SceneInstance; + _activityRelay = FaceTrackingActivityRelay.GetOrCreate(avatar); } public void OnHVRAvatarReady(bool isWearer) { + _registeredSourceAddresses = isWearer; + _eyeFollowDriverApplicable = isWearer; + _trackingActive = _activityRelay != null && _activityRelay.IsTrackingActive; + _eyeTrackingParametersActive = false; + _lastEyeParameterSampleTime = float.NegativeInfinity; + + acquisition.RegisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); if (isWearer) { - acquisition.RegisterAddresses(OurAddresses, OnAddressUpdated); - _eyeFollowDriverApplicable = true; + acquisition.RegisterAddresses(_sourceEyeAddresses, OnAddressUpdated); } } @@ -77,18 +100,40 @@ public void OnHVRReadyBothAvatarAndNetwork(bool isWearer) Receiver = NetworkedPlayer as BasisNetworkReceiver; } - var mutualizedInterpolationRanges = OurAddresses.Select(address => new MutualizedInterpolationRange + var mutualizedInterpolationRanges = new List { - address = address, - lower = -1f, - upper = 1f, - }).ToList(); + new MutualizedInterpolationRange { address = _eyeLeftXAddress, lower = -1f, upper = 1f }, + new MutualizedInterpolationRange { address = _eyeRightXAddress, lower = -1f, upper = 1f }, + new MutualizedInterpolationRange { address = _eyeYAddress, lower = -1f, upper = 1f }, + new MutualizedInterpolationRange { address = _eyeTrackingActiveAddress, lower = 0f, upper = 1f } + }; featureInterpolator = CommsNetworking.UsingMutualizedInterpolator(avatar, mutualizedInterpolationRanges, OnInterpolatedDataChanged); + bool shouldApply = ShouldApplyEyeTracking(); + if (IsLocal) + { + SubmitEyeTrackingParameterStateToNetwork(); + SetBuiltInEyeFollowDriverOverriden(shouldApply); + if (shouldApply) + { + SubmitCurrentEyeStateToNetwork(); + } + else + { + SubmitNeutralEyesToNetwork(); + } + } + else if (!shouldApply) + { + ClearRemoteOverrides(); + } } private void OnEnable() { - SetBuiltInEyeFollowDriverOverriden(true); + if (ShouldApplyEyeTracking() && _eyeFollowDriverApplicable) + { + SetBuiltInEyeFollowDriverOverriden(true); + } BasisNetworkTransmitter.AfterAvatarChanges += ForceUpdate; } @@ -96,57 +141,151 @@ private void OnDisable() { SetBuiltInEyeFollowDriverOverriden(false); BasisNetworkTransmitter.AfterAvatarChanges -= ForceUpdate; + ClearRemoteOverrides(); } private void OnDestroy() { - if (IsLocal) + if (acquisition != null) { - acquisition.UnregisterAddresses(OurAddresses, OnAddressUpdated); - - if (IsLocal && Receiver != null) + acquisition.UnregisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated); + if (_registeredSourceAddresses) { - Receiver.RemotePlayer.RemoteFaceDriver.OverrideEye = false; - Receiver.RemotePlayer.RemoteFaceDriver.OverrideBlinking = false; + acquisition.UnregisterAddresses(_sourceEyeAddresses, OnAddressUpdated); } } + + ClearRemoteOverrides(); + SetBuiltInEyeFollowDriverOverriden(false); + } + + private void Update() + { + if (!IsLocal || !_trackingActive || !_eyeTrackingParametersActive) + { + return; + } + + if (Time.unscaledTime - _lastEyeParameterSampleTime > EyeParameterInactivityTimeoutSeconds) + { + SetLocalEyeParameterState(false); + SetBuiltInEyeFollowDriverOverriden(false); + SubmitNeutralEyesToNetwork(); + } } private void OnAddressUpdated(int address, float value) { - // FIXME: Temp fix, we'll need to hook to NetworkReady instead. - // This is a quick fix so that we don't need to reupload the avatar. - _anyAddressUpdated = _anyAddressUpdated || value != 0f; - if (_anyAddressUpdated) + if (!_trackingActive) + { + return; + } + + float sanitizedValue = SanitizeAndClampEyeValue(value); + switch (address) + { + case var _ when address == _eyeLeftXAddress: + _fEyeLeftX = sanitizedValue; + if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(LeftEyeFeatureIndex, sanitizedValue); + break; + case var _ when address == _eyeRightXAddress: + _fEyeRightX = sanitizedValue; + if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(RightEyeFeatureIndex, sanitizedValue); + break; + case var _ when address == _eyeYAddress: + _fEyeY = sanitizedValue; + if (featureInterpolator != null && IsLocal) featureInterpolator.SubmitAbsolute(EyeYFeatureIndex, sanitizedValue); + break; + default: + return; + } + + if (IsLocal) + { + _lastEyeParameterSampleTime = Time.unscaledTime; + if (!_eyeTrackingParametersActive) + { + SetLocalEyeParameterState(true); + SetBuiltInEyeFollowDriverOverriden(true); + } + } + } + + private void OnTrackingActivityUpdated(int address, float value) + { + if (address != FaceTrackingActivityRelay.ActivityAddressId) + { + return; + } + + bool isTrackingActive = value >= 0.5f; + if (_trackingActive == isTrackingActive) + { + return; + } + + _trackingActive = isTrackingActive; + if (IsLocal && !_trackingActive) { - BasisLocalEyeDriver.IsEnabled = false; + SetLocalEyeParameterState(false); } - if (address == EyeLeftXAddress) + bool shouldApplyEyeTracking = ShouldApplyEyeTracking(); + if (IsLocal) { - _fEyeLeftX = value; - if (featureInterpolator != null) featureInterpolator.SubmitAbsolute(0, value); + SetBuiltInEyeFollowDriverOverriden(shouldApplyEyeTracking); } - else if (address == EyeRightXAddress) + + if (_trackingActive) { - _fEyeRightX = value; - if (featureInterpolator != null) featureInterpolator.SubmitAbsolute(1, value); + if (IsLocal) + { + if (shouldApplyEyeTracking) + { + SubmitCurrentEyeStateToNetwork(); + } + else + { + SubmitNeutralEyesToNetwork(); + } + } + else if (!shouldApplyEyeTracking) + { + ClearRemoteOverrides(); + } + return; } - else if (address == EyeYAddress) + + ResetEyeValuesToZero(); + _eyeTrackingParametersActive = false; + + if (IsLocal) { - _fEyeY = value; - if (featureInterpolator != null) featureInterpolator.SubmitAbsolute(2, value); + SubmitNeutralEyesToNetwork(); + } + else + { + SetNeutralRemoteEyes(); + ClearRemoteOverrides(); } } private void ForceUpdate() { + if (!ShouldApplyEyeTracking()) + { + return; + } + SetEyeRotation(_fEyeLeftX, _fEyeY, EyeSide.Left); SetEyeRotation(_fEyeRightX, _fEyeY, EyeSide.Right); } private void SetEyeRotation(float x, float y, EyeSide side) { + x = SanitizeAndClampEyeValue(x); + y = SanitizeAndClampEyeValue(y); + if (_eyeFollowDriverApplicable) { var xDeg = Mathf.Asin(x) * Mathf.Rad2Deg * multiplyX; @@ -166,29 +305,22 @@ private void SetEyeRotation(float x, float y, EyeSide side) throw new ArgumentOutOfRangeException(nameof(side), side, null); } } - else + else if (!IsLocal && Receiver != null) { - if (IsLocal && Receiver != null) + Receiver.RemotePlayer.RemoteFaceDriver.OverrideEye = true; + Receiver.RemotePlayer.RemoteFaceDriver.OverrideBlinking = true; + switch (side) { - Receiver.RemotePlayer.RemoteFaceDriver.OverrideEye = true; - Receiver.RemotePlayer.RemoteFaceDriver.OverrideBlinking = true; - switch (side) - { - case EyeSide.Left: - float result0 = (y + 1) / 2; - float result1 = (x + 1) / 2; - Receiver.EyesAndMouth[0] = result0; - Receiver.EyesAndMouth[1] = result1; - break; - case EyeSide.Right: - result0 = (y + 1) / 2; - result1 = (x + 1) / 2; - Receiver.EyesAndMouth[2] = result0; - Receiver.EyesAndMouth[3] = result1; - break; - default: - throw new ArgumentOutOfRangeException(nameof(side), side, null); - } + case EyeSide.Left: + Receiver.EyesAndMouth[0] = (y + 1) / 2; + Receiver.EyesAndMouth[1] = (x + 1) / 2; + break; + case EyeSide.Right: + Receiver.EyesAndMouth[2] = (y + 1) / 2; + Receiver.EyesAndMouth[3] = (x + 1) / 2; + break; + default: + throw new ArgumentOutOfRangeException(nameof(side), side, null); } } } @@ -198,6 +330,93 @@ private void SetBuiltInEyeFollowDriverOverriden(bool value) BasisLocalEyeDriver.Override = value; } + private void SubmitCurrentEyeStateToNetwork() + { + if (!IsLocal || featureInterpolator == null) + { + return; + } + + featureInterpolator.SubmitAbsolute(LeftEyeFeatureIndex, SanitizeAndClampEyeValue(_fEyeLeftX)); + featureInterpolator.SubmitAbsolute(RightEyeFeatureIndex, SanitizeAndClampEyeValue(_fEyeRightX)); + featureInterpolator.SubmitAbsolute(EyeYFeatureIndex, SanitizeAndClampEyeValue(_fEyeY)); + } + + private void SubmitNeutralEyesToNetwork() + { + if (!IsLocal || featureInterpolator == null) + { + return; + } + + featureInterpolator.SubmitAbsolute(LeftEyeFeatureIndex, 0f); + featureInterpolator.SubmitAbsolute(RightEyeFeatureIndex, 0f); + featureInterpolator.SubmitAbsolute(EyeYFeatureIndex, 0f); + } + + private void SetLocalEyeParameterState(bool isActive) + { + _eyeTrackingParametersActive = isActive; + _lastEyeParameterSampleTime = isActive ? Time.unscaledTime : float.NegativeInfinity; + SubmitEyeTrackingParameterStateToNetwork(); + } + + private void SubmitEyeTrackingParameterStateToNetwork() + { + if (!IsLocal || featureInterpolator == null) + { + return; + } + + featureInterpolator.SubmitAbsolute(EyeTrackingActiveFeatureIndex, _eyeTrackingParametersActive ? 1f : 0f); + } + + private bool ShouldApplyEyeTracking() + { + return _trackingActive && _eyeTrackingParametersActive; + } + + private static float SanitizeAndClampEyeValue(float value) + { + if (float.IsNaN(value) || float.IsInfinity(value)) + { + return 0f; + } + + return Mathf.Clamp(value, -1f, 1f); + } + + private void ResetEyeValuesToZero() + { + _fEyeLeftX = 0f; + _fEyeRightX = 0f; + _fEyeY = 0f; + } + + private void SetNeutralRemoteEyes() + { + if (Receiver == null) + { + return; + } + + Receiver.EyesAndMouth[0] = 0.5f; + Receiver.EyesAndMouth[1] = 0.5f; + Receiver.EyesAndMouth[2] = 0.5f; + Receiver.EyesAndMouth[3] = 0.5f; + } + + private void ClearRemoteOverrides() + { + if (IsLocal || Receiver == null) + { + return; + } + + Receiver.RemotePlayer.RemoteFaceDriver.OverrideEye = false; + Receiver.RemotePlayer.RemoteFaceDriver.OverrideBlinking = false; + } + private enum EyeSide { Left, Right @@ -206,10 +425,39 @@ private enum EyeSide #region NetworkingMethods private void OnInterpolatedDataChanged(float[] current) { - _fEyeLeftX = current[0]; - _fEyeRightX = current[1]; - _fEyeY = current[2]; + if (current == null) + { + return; + } + + if (!IsLocal) + { + if (current.Length > EyeTrackingActiveFeatureIndex) + { + _eyeTrackingParametersActive = current[EyeTrackingActiveFeatureIndex] >= 0.5f; + } + else + { + // Legacy compatibility with senders that only stream 3 values. + _eyeTrackingParametersActive = true; + } + } + + bool shouldApply = ShouldApplyEyeTracking(); + if (!shouldApply || current.Length < 3) + { + if (!IsLocal && !shouldApply) + { + ClearRemoteOverrides(); + } + return; + } + + _fEyeLeftX = SanitizeAndClampEyeValue(current[LeftEyeFeatureIndex]); + _fEyeRightX = SanitizeAndClampEyeValue(current[RightEyeFeatureIndex]); + _fEyeY = SanitizeAndClampEyeValue(current[EyeYFeatureIndex]); } #endregion } } + diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/AutomaticFaceTracking.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/AutomaticFaceTracking.cs index 4639a8c33..2b2bf8edf 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/AutomaticFaceTracking.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/AutomaticFaceTracking.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; @@ -35,6 +35,7 @@ public class AutomaticFaceTracking : MonoBehaviour, IHVRInitializable [NonSerialized] internal OSCAcquisition oscAcquisition; [NonSerialized] internal BlendshapeActuation blendshapeActuation; [NonSerialized] internal EyeTrackingBoneActuation eyeTrackingBoneActuation; + [NonSerialized] internal FaceTrackingActivityRelay faceTrackingActivityRelay; private bool _isWearer; @@ -119,6 +120,9 @@ private void Failed() private void SetupFaceTracking(BlendshapeActuationDefinitionFile[] definitionFiles, List smrs) { renderers = smrs; + faceTrackingActivityRelay = FaceTrackingActivityRelay.GetOrCreate(_avatar); + faceTrackingActivityRelay.OnHVRAvatarReady(_isWearer); + if (_isWearer) { oscAcquisition = CreateOSCAcquisitionIfNotExists(); diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs new file mode 100644 index 000000000..2d8e2b74e --- /dev/null +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs @@ -0,0 +1 @@ +// The FaceTrackingActivityRelay implementation lives in Runtime/Networking/HVRCommsUtil.cs. diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs.meta b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs.meta new file mode 100644 index 000000000..eb7f2e13b --- /dev/null +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/FaceTracking/FaceTrackingActivityRelay.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 8317c8e0fc6d3eb418345948d6c69a1a \ No newline at end of file diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/HVRAvatarComms.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/HVRAvatarComms.cs index dac972ba7..4e95f3917 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/HVRAvatarComms.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/HVRAvatarComms.cs @@ -52,7 +52,7 @@ private void Awake() private void OnAvatarReady(bool isWearer) { - _isWearer = true; + _isWearer = isWearer; var allInitializables = avatar.GetComponentsInChildren(true); foreach (var initializable in allInitializables) @@ -103,14 +103,16 @@ private void DeclareMutualizedInterpolator(bool isWearer, HVRNetworkingCarrier c _streamedLateInit.transmitter = carrier; _streamedLateInit.isWearer = isWearer; _streamedLateInit.localIdentifier = 0; - _toStoreLater.Clear(); + var pendingStores = _toStoreLater.ToArray(); holder.SetActive(true); + _streamedLateInit.InitializeNormalizedValues(BuildNeutralNormalizedValues()); // StreamedAvatarFeature only gets the ability to store data AFTER Awake() runs, so order matters here. - foreach (var toStoreLater in _toStoreLater) + foreach (var toStoreLater in pendingStores) { var mutualizedIndex = toStoreLater.mutualizedIndex; _streamedLateInit.Store(mutualizedIndex, _ranges[mutualizedIndex].AbsoluteToRange(toStoreLater.absolute)); } + _toStoreLater.Clear(); _streamedLateInit.OnInterpolatedDataChanged += mutualizedData => { @@ -169,6 +171,7 @@ public MutualizedFeatureInterpolator NeedsMutualizedInterpolator(List= 0f + ? Mathf.Clamp01(range.AbsoluteToRange(0f)) + : 0f; + } + + return normalized; + } + private class HVRRedirectToStreamed : IFeatureReceiver { private readonly StreamedAvatarFeature streamed; diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/StreamedAvatarFeature.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/StreamedAvatarFeature.cs index cdc04cffc..4123288fc 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/StreamedAvatarFeature.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Networking/StreamedAvatarFeature.cs @@ -40,9 +40,7 @@ public class StreamedAvatarFeature : MonoBehaviour private void Awake() { - previous ??= new float[valueArraySize]; - target ??= new float[valueArraySize]; - current ??= new float[valueArraySize]; + EnsureBuffers(); } private void OnDisable() @@ -52,6 +50,7 @@ private void OnDisable() public void Store(int index, float value) { + EnsureBuffers(); current[index] = value; if (PrioritizeLargeChanges && isWearer) { @@ -66,6 +65,20 @@ public void Store(int index, float value) } } + public void InitializeNormalizedValues(IReadOnlyList normalizedValues) + { + EnsureBuffers(); + + int count = Mathf.Min(valueArraySize, normalizedValues?.Count ?? 0); + for (int i = 0; i < count; i++) + { + float clamped = Mathf.Clamp01(normalizedValues[i]); + current[i] = clamped; + previous[i] = clamped; + target[i] = clamped; + } + } + /// Exposed for testing purposes. public void QueueEvent(StreamedAvatarFeaturePayload message) { @@ -261,6 +274,13 @@ public void OnResyncRequested(ushort[] whoAsked) FloatValues = current }, whoAsked); } + + private void EnsureBuffers() + { + previous ??= new float[valueArraySize]; + target ??= new float[valueArraySize]; + current ??= new float[valueArraySize]; + } } public class StreamedAvatarFeaturePayload { diff --git a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Networking/HVRCommsUtil.cs b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Networking/HVRCommsUtil.cs index f4cb03bcd..dec9693c0 100644 --- a/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Networking/HVRCommsUtil.cs +++ b/Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Networking/HVRCommsUtil.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using Basis.Scripts.BasisSdk; using UnityEngine; @@ -43,4 +44,137 @@ public static T[] SlowSanitizeEndUserProvidedStructArray(T[] structuresNullab return structuresNullable; } } + + [AddComponentMenu("HVR.Basis/Comms/Internal/Face Tracking Activity Relay")] + public class FaceTrackingActivityRelay : MonoBehaviour, IHVRInitializable + { + public const string ActivityAddress = "HVR/Internal/FaceTrackingActive"; + public static readonly int ActivityAddressId = HVRAddress.AddressToId(ActivityAddress); + public const float InactivityTimeoutSeconds = 0.5f; + + [HideInInspector] [SerializeField] private BasisAvatar avatar; + [HideInInspector] [SerializeField] private AcquisitionService acquisition; + + [NonSerialized] internal MutualizedFeatureInterpolator featureInterpolator; + + private bool _isWearer; + private bool _isTrackingActive; + private float _lastActivityTime = float.NegativeInfinity; + + public bool IsTrackingActive => _isTrackingActive; + + public static FaceTrackingActivityRelay GetOrCreate(BasisAvatar avatar) + { + if (avatar == null) + { + return null; + } + + var relay = avatar.GetComponentInChildren(true); + if (relay != null) + { + return relay; + } + + var relayRoot = new GameObject("Generated__FaceTrackingActivityRelay") + { + transform = + { + parent = avatar.transform, + } + }; + return relayRoot.AddComponent(); + } + + private void Awake() + { + if (avatar == null) + { + avatar = HVRCommsUtil.GetAvatar(this); + } + + if (acquisition == null) + { + acquisition = AcquisitionService.SceneInstance; + } + } + + public void OnHVRAvatarReady(bool isWearer) + { + _isWearer = isWearer; + ApplyTrackingState(false, submitToNetwork: false); + } + + public void OnHVRReadyBothAvatarAndNetwork(bool isWearer) + { + _isWearer = isWearer; + featureInterpolator = CommsNetworking.UsingMutualizedInterpolator(avatar, new List + { + new MutualizedInterpolationRange + { + address = ActivityAddressId, + lower = 0f, + upper = 1f, + } + }, OnInterpolatedDataChanged); + + if (_isWearer && featureInterpolator != null) + { + featureInterpolator.SubmitAbsolute(0, _isTrackingActive ? 1f : 0f); + } + } + + private void Update() + { + if (!_isWearer || !_isTrackingActive) + { + return; + } + + if (Time.unscaledTime - _lastActivityTime > InactivityTimeoutSeconds) + { + ApplyTrackingState(false, submitToNetwork: true); + } + } + + public void NotifySourceSample() + { + if (!_isWearer) + { + return; + } + + _lastActivityTime = Time.unscaledTime; + if (!_isTrackingActive) + { + ApplyTrackingState(true, submitToNetwork: true); + } + } + + private void OnInterpolatedDataChanged(float[] current) + { + if (_isWearer || current == null || current.Length == 0) + { + return; + } + + ApplyTrackingState(current[0] >= 0.5f, submitToNetwork: false); + } + + private void ApplyTrackingState(bool isTrackingActive, bool submitToNetwork) + { + bool stateChanged = _isTrackingActive != isTrackingActive; + _isTrackingActive = isTrackingActive; + + if (acquisition != null && (stateChanged || submitToNetwork)) + { + acquisition.Submit(ActivityAddressId, isTrackingActive ? 1f : 0f); + } + + if (submitToNetwork && _isWearer && featureInterpolator != null) + { + featureInterpolator.SubmitAbsolute(0, isTrackingActive ? 1f : 0f); + } + } + } }