@@ -14,6 +14,7 @@ import (
1414 historyspb "go.temporal.io/server/api/history/v1"
1515 persistencespb "go.temporal.io/server/api/persistence/v1"
1616 "go.temporal.io/server/common"
17+ "go.temporal.io/server/common/clock"
1718 "go.temporal.io/server/common/cluster"
1819 "go.temporal.io/server/common/log"
1920 "go.temporal.io/server/common/namespace"
@@ -1434,6 +1435,151 @@ func (s *taskRefresherSuite) TestRefreshSubStateMachineTasks() {
14341435 s .False (hsmRoot .Dirty ())
14351436}
14361437
1438+ // buildMutableStateWithTimeSkipping creates a MutableStateImpl with TimeSkippingInfo having
1439+ // the given total skipped duration in a single entry.
1440+ func (s * taskRefresherSuite ) buildMutableStateWithTimeSkipping (totalSkip time.Duration , enabled bool ) * MutableStateImpl {
1441+ dbRecord := & persistencespb.WorkflowMutableState {
1442+ ExecutionInfo : & persistencespb.WorkflowExecutionInfo {
1443+ NamespaceId : tests .NamespaceID .String (),
1444+ WorkflowId : tests .WorkflowID ,
1445+ TimeSkippingInfo : & persistencespb.TimeSkippingInfo {
1446+ Enabled : enabled ,
1447+ TimeSkippedDetails : []* persistencespb.TimeSkippedDetails {
1448+ {DurationToSkip : clock .TimeSkippedDurationToTimestamp (totalSkip )},
1449+ },
1450+ },
1451+ },
1452+ ExecutionState : & persistencespb.WorkflowExecutionState {
1453+ RunId : tests .RunID ,
1454+ State : enumsspb .WORKFLOW_EXECUTION_STATE_RUNNING ,
1455+ Status : enumspb .WORKFLOW_EXECUTION_STATUS_RUNNING ,
1456+ },
1457+ NextEventId : 1 ,
1458+ }
1459+ ms , err := NewMutableStateFromDB (
1460+ s .mockShard ,
1461+ s .mockShard .GetEventsCache (),
1462+ log .NewTestLogger (),
1463+ tests .LocalNamespaceEntry ,
1464+ dbRecord ,
1465+ 1 ,
1466+ )
1467+ s .NoError (err )
1468+ return ms
1469+ }
1470+
1471+ func (s * taskRefresherSuite ) TestApplyTimeSkippingOffset_AdjustsAllTimerTaskTypes () {
1472+ skipDuration := 2 * time .Hour
1473+ ms := s .buildMutableStateWithTimeSkipping (skipDuration , true )
1474+
1475+ now := s .mockShard .GetTimeSource ().Now ()
1476+ fireTime := now .Add (3 * time .Hour )
1477+ wfKey := ms .GetWorkflowKey ()
1478+
1479+ ms .InsertTasks [tasks .CategoryTimer ] = []tasks.Task {
1480+ & tasks.UserTimerTask {WorkflowKey : wfKey , VisibilityTimestamp : fireTime },
1481+ & tasks.ActivityTimeoutTask {WorkflowKey : wfKey , VisibilityTimestamp : fireTime },
1482+ & tasks.WorkflowRunTimeoutTask {WorkflowKey : wfKey , VisibilityTimestamp : fireTime },
1483+ & tasks.WorkflowExecutionTimeoutTask {NamespaceID : wfKey .NamespaceID , WorkflowID : wfKey .WorkflowID , VisibilityTimestamp : fireTime },
1484+ }
1485+
1486+ s .taskRefresher .applyTimeSkippingOffsetToTimerTasks (ms )
1487+
1488+ expected := fireTime .Add (- skipDuration ) // now + 1h
1489+ for _ , task := range ms .InsertTasks [tasks .CategoryTimer ] {
1490+ s .Equal (expected , task .GetVisibilityTime (), "task type %v should be adjusted" , task .GetType ())
1491+ }
1492+ }
1493+
1494+ func (s * taskRefresherSuite ) TestApplyTimeSkippingOffset_ExcludesDeleteAndTimeSkippingTasks () {
1495+ skipDuration := 2 * time .Hour
1496+ ms := s .buildMutableStateWithTimeSkipping (skipDuration , true )
1497+
1498+ now := s .mockShard .GetTimeSource ().Now ()
1499+ fireTime := now .Add (3 * time .Hour )
1500+ wfKey := ms .GetWorkflowKey ()
1501+
1502+ ms .InsertTasks [tasks .CategoryTimer ] = []tasks.Task {
1503+ & tasks.DeleteHistoryEventTask {WorkflowKey : wfKey , VisibilityTimestamp : fireTime },
1504+ & tasks.TimeSkippingTimerTask {WorkflowKey : wfKey , VisibilityTimestamp : fireTime },
1505+ }
1506+
1507+ s .taskRefresher .applyTimeSkippingOffsetToTimerTasks (ms )
1508+
1509+ for _ , task := range ms .InsertTasks [tasks .CategoryTimer ] {
1510+ s .Equal (fireTime , task .GetVisibilityTime (), "task type %v should not be adjusted" , task .GetType ())
1511+ }
1512+ }
1513+
1514+ func (s * taskRefresherSuite ) TestApplyTimeSkippingOffset_MultipleSkipEntries_UsesTotalOffset () {
1515+ // Two separate skip entries: 1h + 1h = 2h total. The old bug used
1516+ // latestTargetVirtualTime - realNow instead of summing DurationToSkip.
1517+ dbRecord := & persistencespb.WorkflowMutableState {
1518+ ExecutionInfo : & persistencespb.WorkflowExecutionInfo {
1519+ NamespaceId : tests .NamespaceID .String (),
1520+ WorkflowId : tests .WorkflowID ,
1521+ TimeSkippingInfo : & persistencespb.TimeSkippingInfo {
1522+ Enabled : true ,
1523+ TimeSkippedDetails : []* persistencespb.TimeSkippedDetails {
1524+ {DurationToSkip : clock .TimeSkippedDurationToTimestamp (time .Hour )},
1525+ {DurationToSkip : clock .TimeSkippedDurationToTimestamp (time .Hour )},
1526+ },
1527+ },
1528+ },
1529+ ExecutionState : & persistencespb.WorkflowExecutionState {
1530+ RunId : tests .RunID ,
1531+ State : enumsspb .WORKFLOW_EXECUTION_STATE_RUNNING ,
1532+ Status : enumspb .WORKFLOW_EXECUTION_STATUS_RUNNING ,
1533+ },
1534+ NextEventId : 1 ,
1535+ }
1536+ ms , err := NewMutableStateFromDB (s .mockShard , s .mockShard .GetEventsCache (), log .NewTestLogger (), tests .LocalNamespaceEntry , dbRecord , 1 )
1537+ s .NoError (err )
1538+
1539+ now := s .mockShard .GetTimeSource ().Now ()
1540+ fireTime := now .Add (3 * time .Hour )
1541+ ms .InsertTasks [tasks .CategoryTimer ] = []tasks.Task {
1542+ & tasks.UserTimerTask {WorkflowKey : ms .GetWorkflowKey (), VisibilityTimestamp : fireTime },
1543+ }
1544+
1545+ s .taskRefresher .applyTimeSkippingOffsetToTimerTasks (ms )
1546+
1547+ // Total offset = 2h, so adjusted = now + 3h - 2h = now + 1h
1548+ s .Equal (now .Add (time .Hour ), ms .InsertTasks [tasks .CategoryTimer ][0 ].GetVisibilityTime ())
1549+ }
1550+
1551+ func (s * taskRefresherSuite ) TestApplyTimeSkippingOffset_ClampsToNow () {
1552+ skipDuration := 5 * time .Hour
1553+ ms := s .buildMutableStateWithTimeSkipping (skipDuration , true )
1554+
1555+ now := s .mockShard .GetTimeSource ().Now ()
1556+ fireTime := now .Add (3 * time .Hour ) // adjusted = now + 3h - 5h = 2h in the past
1557+ ms .InsertTasks [tasks .CategoryTimer ] = []tasks.Task {
1558+ & tasks.UserTimerTask {WorkflowKey : ms .GetWorkflowKey (), VisibilityTimestamp : fireTime },
1559+ }
1560+
1561+ s .taskRefresher .applyTimeSkippingOffsetToTimerTasks (ms )
1562+
1563+ s .Equal (now , ms .InsertTasks [tasks .CategoryTimer ][0 ].GetVisibilityTime ())
1564+ }
1565+
1566+ func (s * taskRefresherSuite ) TestApplyTimeSkippingOffset_DisabledButHasSkips_StillAdjusts () {
1567+ // Even when Enabled=false, accumulated skips must be applied because virtual time
1568+ // has deviated from wall clock time and tasks must fire at the correct real time.
1569+ skipDuration := 2 * time .Hour
1570+ ms := s .buildMutableStateWithTimeSkipping (skipDuration , false )
1571+
1572+ now := s .mockShard .GetTimeSource ().Now ()
1573+ fireTime := now .Add (3 * time .Hour )
1574+ ms .InsertTasks [tasks .CategoryTimer ] = []tasks.Task {
1575+ & tasks.UserTimerTask {WorkflowKey : ms .GetWorkflowKey (), VisibilityTimestamp : fireTime },
1576+ }
1577+
1578+ s .taskRefresher .applyTimeSkippingOffsetToTimerTasks (ms )
1579+
1580+ s .Equal (fireTime .Add (- skipDuration ), ms .InsertTasks [tasks .CategoryTimer ][0 ].GetVisibilityTime ())
1581+ }
1582+
14371583type mockTaskGeneratorProvider struct {
14381584 mockTaskGenerator * MockTaskGenerator
14391585}
0 commit comments