Skip to content

Commit bbb3df5

Browse files
Toys0125Toys0125
authored
fix(face-tracking): make host authoritative for activity state and neutralize startup sync (#582)
* fix(face-tracking): make host authoritative for activity state and neutralize startup sync * Added an internal `FaceTrackingActivityRelay` channel (`HVR/Internal/FaceTrackingActive`) with a host-only inactivity timeout (0.5s). Remotes now follow host activity and do not run their own timeout logic. * Wired OSC input to activity updates so any incoming face packet marks tracking active. * Fixed remote eye application in `EyeTrackingBoneActuation` (!IsLocal && Receiver != null) and ensured remote override flags are cleared when tracking is inactive/disabled. * Gated eye and blendshape actuation behind propagated tracking activity: Active: HVR drives local/remote face data. Inactive: reset HVR-driven values to neutral and return control to Basis default behavior. * Stopped applying startup/default blendshape overrides when tracking is inactive; defaults are only pushed while tracking is active. * Changed mutualized feature initialization to neutral normalized values based on interpolation ranges (AbsoluteToRange(0) when 0 is in range), preventing signed channels from booting at low-end values (e.g. unintended TongueLeft). * Updated networking init flow to seed neutral values before initial replay, and fixed wearer/range bookkeeping issues in `HVRAvatarComms`. * fix(face-tracking): make host authoritative for activity state and neutralize startup sync * Added an internal `FaceTrackingActivityRelay` channel (`HVR/Internal/FaceTrackingActive`) with a host-only inactivity timeout (0.5s). Remotes now follow host activity and do not run their own timeout logic. * Wired OSC input to activity updates so any incoming face packet marks tracking active. * Fixed remote eye application in `EyeTrackingBoneActuation` (!IsLocal && Receiver != null) and ensured remote override flags are cleared when tracking is inactive/disabled. * Gated eye and blendshape actuation behind propagated tracking activity: Active: HVR drives local/remote face data. Inactive: reset HVR-driven values to neutral and return control to Basis default behavior. * Stopped applying startup/default blendshape overrides when tracking is inactive; defaults are only pushed while tracking is active. * Changed mutualized feature initialization to neutral normalized values based on interpolation ranges (AbsoluteToRange(0) when 0 is in range), preventing signed channels from booting at low-end values (e.g. unintended TongueLeft). * Updated networking init flow to seed neutral values before initial replay, and fixed wearer/range bookkeeping issues in `HVRAvatarComms`. * Fix potential bug, in which the override might not be called again when enabled. * Reset changes not related to face tracking back to developer * Fix(face-tracking): fallback to default Basis eye driver when eye params are missing
1 parent 6d2802e commit bbb3df5

9 files changed

Lines changed: 665 additions & 104 deletions

File tree

‎Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Acquisition/OSCAcquisition.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
using Basis.Scripts.BasisSdk;
1+
using Basis.Scripts.BasisSdk;
22
using UnityEngine;
33

44
namespace HVR.Basis.Comms
@@ -13,11 +13,13 @@ public class OSCAcquisition : MonoBehaviour
1313

1414
private OSCAcquisitionServer _acquisitionServer;
1515
private bool _alreadyInitialized;
16+
private FaceTrackingActivityRelay _activityRelay;
1617

1718
private void Awake()
1819
{
1920
if (avatar == null) avatar = HVRCommsUtil.GetAvatar(this);
2021
if (acquisitionService == null) acquisitionService = AcquisitionService.SceneInstance;
22+
_activityRelay = FaceTrackingActivityRelay.GetOrCreate(avatar);
2123

2224
avatar.OnAvatarReady -= OnAvatarReady;
2325
avatar.OnAvatarReady += OnAvatarReady;
@@ -55,6 +57,7 @@ private void OnAddressUpdated(string address, float value)
5557
{
5658
if (!isActiveAndEnabled) return;
5759

60+
_activityRelay?.NotifySourceSample();
5861
acquisitionService.Submit(HVRAddress.AddressToId(address), value);
5962
}
6063
}

‎Basis/Packages/dev.hai-vr.basis.comms/Scripts/Systems/Runtime/Components/Actuation/BlendshapeActuation.cs‎

Lines changed: 159 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,14 @@ public class BlendshapeActuation : MonoBehaviour, IHVRInitializable
2222
[HideInInspector] [SerializeField] private AcquisitionService acquisition;
2323

2424
private Dictionary<int, int> _addessIdToBaseIndex = new();
25+
private readonly Dictionary<int, float> _latestAbsoluteByAddress = new();
2526
private ComputedActuator[] _computedActuators;
2627
private ComputedActuator[][] _addressBaseIndexToActuators;
2728
private Dictionary<int, (float, float)> _addressToStreamedLowerUpper;
29+
private AddressOverride[] _defaultOverrides = Array.Empty<AddressOverride>();
30+
private FaceTrackingActivityRelay _activityRelay;
31+
private bool _isWearer;
32+
private bool _trackingActive;
2833

2934
#region NetworkingFields
3035
// Can be null due to:
@@ -56,33 +61,24 @@ private void Awake()
5661
acquisition = AcquisitionService.SceneInstance;
5762
}
5863

64+
_activityRelay = FaceTrackingActivityRelay.GetOrCreate(avatar);
5965
renderers = HVRCommsUtil.SlowSanitizeEndUserProvidedObjectArray(renderers);
6066
definitionFiles = HVRCommsUtil.SlowSanitizeEndUserProvidedObjectArray(definitionFiles);
6167
definitions = HVRCommsUtil.SlowSanitizeEndUserProvidedStructArray(definitions);
6268
}
6369

6470
private void OnAddressUpdated(int address, float inRange)
6571
{
66-
if (!_addessIdToBaseIndex.TryGetValue(address, out var baseIndex)) return;
67-
68-
// TODO: Might need to queue and delay this change so that it executes on the Update loop.
69-
70-
var actuatorsForThisAddress = _addressBaseIndexToActuators[baseIndex];
71-
if (actuatorsForThisAddress == null) return; // There may be no actuator for an address when it does not exist in the renderers.
72-
73-
foreach (var actuator in actuatorsForThisAddress)
74-
{
75-
Actuate(actuator, inRange);
76-
}
77-
78-
if (featureInterpolator != null)
79-
{
80-
featureInterpolator.SubmitAbsolute(baseIndex, inRange);
81-
}
72+
ApplyAddressValue(address, inRange, forwardToNetwork: _isWearer);
8273
}
8374

8475
private void OnInterpolatedDataChanged(float[] current)
8576
{
77+
if (!_trackingActive || current == null)
78+
{
79+
return;
80+
}
81+
8682
foreach (var actuator in _computedActuators)
8783
{
8884
var absolute = current[actuator.AddressIndex];
@@ -112,6 +108,10 @@ private static void Actuate(ComputedActuator actuator, float inRange)
112108

113109
public void OnHVRAvatarReady(bool isWearer)
114110
{
111+
_isWearer = isWearer;
112+
acquisition.RegisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated);
113+
_trackingActive = _activityRelay != null && _activityRelay.IsTrackingActive;
114+
115115
var allDefinitions = definitions
116116
.Concat(definitionFiles.SelectMany(file => file.definitions))
117117
.ToArray();
@@ -128,7 +128,7 @@ public void OnHVRAvatarReady(bool isWearer)
128128
var inValuesForThisAddress = grouping
129129
// Reminder that InStart may be greater than InEnd.
130130
// We want the lower bound, not the minimum of InStart.
131-
.SelectMany(definition => new [] { definition.inStart, definition.inEnd })
131+
.SelectMany(definition => new[] { definition.inStart, definition.inEnd })
132132
.ToArray();
133133
return (inValuesForThisAddress.Min(), inValuesForThisAddress.Max());
134134
});
@@ -192,6 +192,12 @@ public void OnHVRAvatarReady(bool isWearer)
192192
_addressBaseIndexToActuators[computedActuator.Key] = computedActuator.ToArray();
193193
}
194194

195+
_defaultOverrides = definitionFiles
196+
.SelectMany(file => file.addressOverrides)
197+
.Concat(addressOverrides)
198+
.Where(it => it.overrideDefaultValue)
199+
.ToArray();
200+
195201
if (isWearer)
196202
{
197203
acquisition.RegisterAddresses(_addessIdToBaseIndex.Keys.ToArray(), OnAddressUpdated);
@@ -215,20 +221,21 @@ public static Dictionary<SkinnedMeshRenderer, List<string>> ResolveSmrToBlendsha
215221
public void OnHVRReadyBothAvatarAndNetwork(bool isLocallyOwned)
216222
{
217223
HVRLogging.ProtocolDebug("OnReadyBothAvatarAndNetwork called on BlendshapeActuation.");
224+
_isWearer = isLocallyOwned;
218225
// FIXME: We should be using the computed actuators instead of the address base, assuming that
219226
// the list of blendshapes is the same local and remote (no local-only or remote-only blendshapes).
220227
featureInterpolator = CommsNetworking.UsingMutualizedInterpolator(avatar, MakeMutualized(), OnInterpolatedDataChanged);
221228

222-
var overrides = definitionFiles
223-
.SelectMany(file => file.addressOverrides)
224-
.Concat(addressOverrides)
225-
.Where(it => it.overrideDefaultValue)
226-
.ToArray();
227-
foreach (var addressOverride in overrides)
229+
if (_isWearer)
228230
{
229-
if (_addessIdToBaseIndex.TryGetValue(HVRAddress.AddressToId(addressOverride.address), out var key))
231+
if (_trackingActive)
230232
{
231-
featureInterpolator.SubmitAbsolute(key, addressOverride.defaultValue);
233+
SubmitDefaultOverridesToNetwork();
234+
ReplayLatestTrackedValuesToNetwork();
235+
}
236+
else
237+
{
238+
SubmitNeutralValuesToNetwork();
232239
}
233240
}
234241
}
@@ -272,13 +279,138 @@ private void OnDisable()
272279

273280
private void OnDestroy()
274281
{
275-
avatar.OnAvatarReady -= OnHVRAvatarReady;
282+
if (avatar != null)
283+
{
284+
avatar.OnAvatarReady -= OnHVRAvatarReady;
285+
}
286+
287+
if (acquisition != null)
288+
{
289+
acquisition.UnregisterAddresses(new[] { FaceTrackingActivityRelay.ActivityAddressId }, OnTrackingActivityUpdated);
290+
if (_isWearer && _addessIdToBaseIndex.Count > 0)
291+
{
292+
acquisition.UnregisterAddresses(_addessIdToBaseIndex.Keys.ToArray(), OnAddressUpdated);
293+
}
294+
}
295+
}
296+
297+
private void OnTrackingActivityUpdated(int address, float value)
298+
{
299+
if (address != FaceTrackingActivityRelay.ActivityAddressId)
300+
{
301+
return;
302+
}
303+
304+
bool isTrackingActive = value >= 0.5f;
305+
if (_trackingActive == isTrackingActive)
306+
{
307+
return;
308+
}
309+
310+
_trackingActive = isTrackingActive;
311+
if (_trackingActive)
312+
{
313+
if (_isWearer)
314+
{
315+
ApplyDefaultOverrides();
316+
ReplayLatestTrackedValuesToNetwork();
317+
}
318+
return;
319+
}
276320

277-
acquisition.UnregisterAddresses(_addessIdToBaseIndex.Keys.ToArray(), OnAddressUpdated);
321+
ResetAllBlendshapesToZero();
322+
_latestAbsoluteByAddress.Clear();
323+
if (_isWearer)
324+
{
325+
SubmitNeutralValuesToNetwork();
326+
}
327+
}
328+
329+
private void ApplyAddressValue(int address, float inRange, bool forwardToNetwork)
330+
{
331+
if (!_trackingActive || !_addessIdToBaseIndex.TryGetValue(address, out var baseIndex))
332+
{
333+
return;
334+
}
335+
336+
var actuatorsForThisAddress = _addressBaseIndexToActuators[baseIndex];
337+
if (actuatorsForThisAddress == null)
338+
{
339+
return;
340+
}
341+
342+
_latestAbsoluteByAddress[address] = inRange;
343+
foreach (var actuator in actuatorsForThisAddress)
344+
{
345+
Actuate(actuator, inRange);
346+
}
347+
348+
if (forwardToNetwork && _isWearer && featureInterpolator != null)
349+
{
350+
featureInterpolator.SubmitAbsolute(baseIndex, inRange);
351+
}
352+
}
353+
354+
private void ApplyDefaultOverrides()
355+
{
356+
foreach (var addressOverride in _defaultOverrides)
357+
{
358+
ApplyAddressValue(HVRAddress.AddressToId(addressOverride.address), addressOverride.defaultValue, forwardToNetwork: true);
359+
}
360+
}
361+
362+
private void ReplayLatestTrackedValuesToNetwork()
363+
{
364+
if (!_isWearer || featureInterpolator == null)
365+
{
366+
return;
367+
}
368+
369+
foreach (var pair in _latestAbsoluteByAddress)
370+
{
371+
if (_addessIdToBaseIndex.TryGetValue(pair.Key, out var baseIndex))
372+
{
373+
featureInterpolator.SubmitAbsolute(baseIndex, pair.Value);
374+
}
375+
}
376+
}
377+
378+
private void SubmitDefaultOverridesToNetwork()
379+
{
380+
if (!_isWearer || featureInterpolator == null)
381+
{
382+
return;
383+
}
384+
385+
foreach (var addressOverride in _defaultOverrides)
386+
{
387+
if (_addessIdToBaseIndex.TryGetValue(HVRAddress.AddressToId(addressOverride.address), out var baseIndex))
388+
{
389+
featureInterpolator.SubmitAbsolute(baseIndex, addressOverride.defaultValue);
390+
}
391+
}
392+
}
393+
394+
private void SubmitNeutralValuesToNetwork()
395+
{
396+
if (!_isWearer || featureInterpolator == null)
397+
{
398+
return;
399+
}
400+
401+
foreach (var baseIndex in _addessIdToBaseIndex.Values)
402+
{
403+
featureInterpolator.SubmitAbsolute(baseIndex, 0f);
404+
}
278405
}
279406

280407
private void ResetAllBlendshapesToZero()
281408
{
409+
if (_computedActuators == null)
410+
{
411+
return;
412+
}
413+
282414
foreach (var computedActuator in _computedActuators)
283415
{
284416
foreach (var target in computedActuator.Targets)

0 commit comments

Comments
 (0)