From 00cfbf90e9c30d5c4f410597e09ab3cce2531487 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Thu, 4 Dec 2025 22:47:57 +0700 Subject: [PATCH 01/19] refactor: logstore interface & test suite & memlogstore implementation --- internal/logstore/driver/driver.go | 43 +- internal/logstore/drivertest/drivertest.go | 2531 ++++++++++++++--- internal/logstore/logstore.go | 12 +- internal/logstore/memlogstore/memlogstore.go | 319 +++ .../logstore/memlogstore/memlogstore_test.go | 31 + 5 files changed, 2461 insertions(+), 475 deletions(-) create mode 100644 internal/logstore/memlogstore/memlogstore.go create mode 100644 internal/logstore/memlogstore/memlogstore_test.go diff --git a/internal/logstore/driver/driver.go b/internal/logstore/driver/driver.go index f5dfa472..05b15247 100644 --- a/internal/logstore/driver/driver.go +++ b/internal/logstore/driver/driver.go @@ -8,41 +8,36 @@ import ( ) type LogStore interface { - ListEvent(context.Context, ListEventRequest) (ListEventResponse, error) - RetrieveEvent(ctx context.Context, tenantID, eventID string) (*models.Event, error) - RetrieveEventByDestination(ctx context.Context, tenantID, destinationID, eventID string) (*models.Event, error) - ListDelivery(ctx context.Context, request ListDeliveryRequest) ([]*models.Delivery, error) + ListDeliveryEvent(context.Context, ListDeliveryEventRequest) (ListDeliveryEventResponse, error) + RetrieveEvent(ctx context.Context, request RetrieveEventRequest) (*models.Event, error) InsertManyDeliveryEvent(context.Context, []*models.DeliveryEvent) error } -type ListEventRequest struct { +type ListDeliveryEventRequest struct { Next string Prev string Limit int - Start *time.Time // optional - lower bound, default End - 1h - End *time.Time // optional - upper bound, default now() + EventStart *time.Time // optional - filter events created after this time + EventEnd *time.Time // optional - filter events created before this time + DeliveryStart *time.Time // optional - filter deliveries after this time + DeliveryEnd *time.Time // optional - filter deliveries before this time TenantID string // required + EventID string // optional - filter for specific event DestinationIDs []string // optional - Status string // optional, "success", "failed" + Status string // optional: "success", "failed" Topics []string // optional + SortBy string // optional: "event_time", "delivery_time" (default: "delivery_time") + SortOrder string // optional: "asc", "desc" (default: "desc") } -type ListEventByDestinationRequest struct { - TenantID string // required - DestinationID string // required - Status string // optional, "success", "failed" - Cursor string - Limit int -} - -type ListDeliveryRequest struct { - EventID string - DestinationID string +type ListDeliveryEventResponse struct { + Data []*models.DeliveryEvent + Next string + Prev string } -type ListEventResponse struct { - Data []*models.Event - Next string - Prev string - Count int64 +type RetrieveEventRequest struct { + TenantID string // required + EventID string // required + DestinationID string // optional - if provided, scopes to that destination } diff --git a/internal/logstore/drivertest/drivertest.go b/internal/logstore/drivertest/drivertest.go index 8c5f8e71..4d06f745 100644 --- a/internal/logstore/drivertest/drivertest.go +++ b/internal/logstore/drivertest/drivertest.go @@ -3,6 +3,7 @@ package drivertest import ( "context" "fmt" + "sort" "strconv" "testing" "time" @@ -17,7 +18,6 @@ import ( type Harness interface { MakeDriver(ctx context.Context) (driver.LogStore, error) - Close() } @@ -26,15 +26,131 @@ type HarnessMaker func(ctx context.Context, t *testing.T) (Harness, error) func RunConformanceTests(t *testing.T, newHarness HarnessMaker) { t.Helper() - t.Run("TestIntegrationLogStore_EventCRUD", func(t *testing.T) { - testIntegrationLogStore_EventCRUD(t, newHarness) + t.Run("TestInsertManyDeliveryEvent", func(t *testing.T) { + testInsertManyDeliveryEvent(t, newHarness) + }) + t.Run("TestListDeliveryEvent", func(t *testing.T) { + testListDeliveryEvent(t, newHarness) + }) + t.Run("TestRetrieveEvent", func(t *testing.T) { + testRetrieveEvent(t, newHarness) + }) + t.Run("TestTenantIsolation", func(t *testing.T) { + testTenantIsolation(t, newHarness) + }) + t.Run("TestPaginationSimple", func(t *testing.T) { + testPaginationSimple(t, newHarness) + }) + t.Run("TestPaginationSuite", func(t *testing.T) { + testPaginationSuite(t, newHarness) + }) + t.Run("TestEdgeCases", func(t *testing.T) { + testEdgeCases(t, newHarness) + }) +} + +// testInsertManyDeliveryEvent tests the InsertManyDeliveryEvent method +func testInsertManyDeliveryEvent(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + tenantID := uuid.New().String() + destinationID := uuid.New().String() + startTime := time.Now().Add(-1 * time.Hour) + + t.Run("insert single delivery event", func(t *testing.T) { + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("success"), + ) + de := &models.DeliveryEvent{ + ID: uuid.New().String(), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + + err := logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{de}) + require.NoError(t, err) + + // Verify it was inserted + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventID: event.ID, + Limit: 10, + EventStart: &startTime, + }) + require.NoError(t, err) + require.Len(t, response.Data, 1) + assert.Equal(t, event.ID, response.Data[0].Event.ID) + assert.Equal(t, "success", response.Data[0].Delivery.Status) + }) + + t.Run("insert multiple delivery events", func(t *testing.T) { + eventID := uuid.New().String() + baseDeliveryTime := time.Now().Truncate(time.Second) + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + ) + + // Insert multiple deliveries for the same event (simulating retries) + deliveryEvents := []*models.DeliveryEvent{} + for i := 0; i < 3; i++ { + status := "failed" + if i == 2 { + status = "success" + } + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%d", i)), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus(status), + testutil.DeliveryFactory.WithTime(baseDeliveryTime.Add(time.Duration(i)*time.Second)), + ) + deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ + ID: fmt.Sprintf("de_%d", i), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + }) + } + + err := logStore.InsertManyDeliveryEvent(ctx, deliveryEvents) + require.NoError(t, err) + + // Verify all were inserted + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventID: eventID, + Limit: 10, + EventStart: &startTime, + }) + require.NoError(t, err) + require.Len(t, response.Data, 3) }) - t.Run("TestIntegrationLogStore_DeliveryCRUD", func(t *testing.T) { - testIntegrationLogStore_DeliveryCRUD(t, newHarness) + + t.Run("insert empty slice", func(t *testing.T) { + err := logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{}) + require.NoError(t, err) }) } -func testIntegrationLogStore_EventCRUD(t *testing.T, newHarness HarnessMaker) { +// testListDeliveryEvent tests the ListDeliveryEvent method with various filters and pagination +func testListDeliveryEvent(t *testing.T, newHarness HarnessMaker) { t.Helper() ctx := context.Background() @@ -51,56 +167,40 @@ func testIntegrationLogStore_EventCRUD(t *testing.T, newHarness HarnessMaker) { uuid.New().String(), uuid.New().String(), } - destinationEvents := map[string][]*models.Event{} - statusEvents := map[string][]*models.Event{} - destinationStatusEvents := map[string]map[string][]*models.Event{} - topicEvents := map[string][]*models.Event{} - timeEvents := map[string][]*models.Event{} // key is like "1h", "24h", etc. - deliveryEvents := []*models.DeliveryEvent{} - events := []*models.Event{} - - startTime := time.Now().Add(-48 * time.Hour) // before ALL events + + // Track events by various dimensions for assertions + destinationDeliveryEvents := map[string][]*models.DeliveryEvent{} + statusDeliveryEvents := map[string][]*models.DeliveryEvent{} + topicDeliveryEvents := map[string][]*models.DeliveryEvent{} + timeDeliveryEvents := map[string][]*models.DeliveryEvent{} // "1h", "3h", "6h", "24h" + allDeliveryEvents := []*models.DeliveryEvent{} + + // Use a fixed baseTime for deterministic tests + baseTime := time.Now().Truncate(time.Second) + startTime := baseTime.Add(-48 * time.Hour) // before ALL events start := &startTime - baseTime := time.Now() + for i := 0; i < 20; i++ { destinationID := destinationIDs[i%len(destinationIDs)] topic := testutil.TestTopics[i%len(testutil.TestTopics)] shouldSucceed := i%2 == 0 shouldRetry := i%3 == 0 + // Event times are distributed across time buckets: + // i=0-4: within last hour (1h bucket) + // i=5-9: 2-3 hours ago (3h bucket) + // i=10-14: 5-6 hours ago (6h bucket) + // i=15-19: 23-24 hours ago (24h bucket) var eventTime time.Time switch { case i < 5: eventTime = baseTime.Add(-time.Duration(i) * time.Minute) - // i=0: now-0m (newest) - // i=1: now-1m - // i=2: now-2m - // i=3: now-3m - // i=4: now-4m (all within first hour) - case i < 10: eventTime = baseTime.Add(-time.Duration(2*60+i) * time.Minute) - // i=5: now-125m (2h5m ago) - // i=6: now-126m (2h6m ago) - // i=7: now-127m (2h7m ago) - // i=8: now-128m (2h8m ago) - // i=9: now-129m (2h9m ago) - case i < 15: eventTime = baseTime.Add(-time.Duration(5*60+i) * time.Minute) - // i=10: now-310m (5h10m ago) - // i=11: now-311m (5h11m ago) - // i=12: now-312m (5h12m ago) - // i=13: now-313m (5h13m ago) - // i=14: now-314m (5h14m ago) - default: eventTime = baseTime.Add(-time.Duration(23*60+i) * time.Minute) - // i=15: now-1395m (23h15m ago) - // i=16: now-1396m (23h16m ago) - // i=17: now-1397m (23h17m ago) - // i=18: now-1398m (23h18m ago) - // i=19: now-1399m (23h19m ago) } event := testutil.EventFactory.AnyPointer( @@ -114,477 +214,439 @@ func testIntegrationLogStore_EventCRUD(t *testing.T, newHarness HarnessMaker) { "index": strconv.Itoa(i), }), ) - fmt.Printf("Creating event %d: id=%s, time=%s, index=%d\n", i, event.ID, event.Time.Format(time.RFC3339), i) - events = append(events, event) - destinationEvents[destinationID] = append(destinationEvents[destinationID], event) - if _, ok := destinationStatusEvents[destinationID]; !ok { - destinationStatusEvents[destinationID] = map[string][]*models.Event{} - } - topicEvents[topic] = append(topicEvents[topic], event) - switch { - case i < 5: - timeEvents["1h"] = append(timeEvents["1h"], event) - case i < 10: - timeEvents["3h"] = append(timeEvents["3h"], event) - case i < 15: - timeEvents["6h"] = append(timeEvents["6h"], event) - default: - timeEvents["24h"] = append(timeEvents["24h"], event) - } - var delivery *models.Delivery + // Delivery times are based on eventTime for consistency + // Each delivery is slightly after the event, with retries having earlier deliveryTime than final + deliveryTime := eventTime.Add(time.Duration(i) * time.Millisecond) + if shouldRetry { - delivery = testutil.DeliveryFactory.AnyPointer( + initDelivery := testutil.DeliveryFactory.AnyPointer( testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%02d_init", i)), testutil.DeliveryFactory.WithEventID(event.ID), testutil.DeliveryFactory.WithDestinationID(destinationID), testutil.DeliveryFactory.WithStatus("failed"), + testutil.DeliveryFactory.WithTime(deliveryTime), ) - deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ + de := &models.DeliveryEvent{ ID: fmt.Sprintf("de_%02d_init", i), DestinationID: destinationID, Event: *event, - Delivery: delivery, - }) + Delivery: initDelivery, + } + allDeliveryEvents = append(allDeliveryEvents, de) + destinationDeliveryEvents[destinationID] = append(destinationDeliveryEvents[destinationID], de) + statusDeliveryEvents["failed"] = append(statusDeliveryEvents["failed"], de) + topicDeliveryEvents[topic] = append(topicDeliveryEvents[topic], de) + categorizeByTime(i, de, timeDeliveryEvents) + + deliveryTime = deliveryTime.Add(time.Millisecond) // Final delivery is later } - // NOTE: Do NOT else if here; if shouldRetry is true, - // we need to append the failed delivery first. - // Then, we'll append a 2nd delivery after. + var finalStatus string if shouldSucceed { - statusEvents["success"] = append(statusEvents["success"], event) - destinationStatusEvents[destinationID]["success"] = append(destinationStatusEvents[destinationID]["success"], event) - delivery = testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%02d_final", i)), - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("success"), - ) + finalStatus = "success" } else { - statusEvents["failed"] = append(statusEvents["failed"], event) - destinationStatusEvents[destinationID]["failed"] = append(destinationStatusEvents[destinationID]["failed"], event) - delivery = testutil.DeliveryFactory.AnyPointer( - testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%02d_final", i)), - testutil.DeliveryFactory.WithEventID(event.ID), - testutil.DeliveryFactory.WithDestinationID(destinationID), - testutil.DeliveryFactory.WithStatus("failed"), - ) + finalStatus = "failed" } - deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ + finalDelivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%02d_final", i)), + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus(finalStatus), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + de := &models.DeliveryEvent{ ID: fmt.Sprintf("de_%02d_final", i), DestinationID: destinationID, Event: *event, - Delivery: delivery, - }) + Delivery: finalDelivery, + } + allDeliveryEvents = append(allDeliveryEvents, de) + destinationDeliveryEvents[destinationID] = append(destinationDeliveryEvents[destinationID], de) + statusDeliveryEvents[finalStatus] = append(statusDeliveryEvents[finalStatus], de) + topicDeliveryEvents[topic] = append(topicDeliveryEvents[topic], de) + categorizeByTime(i, de, timeDeliveryEvents) } - // Setup | Insert - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, deliveryEvents)) + // Insert all delivery events + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, allDeliveryEvents)) - // Queries - t.Run("list event empty", func(t *testing.T) { - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: "unknown", - Limit: 5, - Next: "", - Start: start, + // Sort allDeliveryEvents by delivery_time DESC for ordering assertions + // This is the expected order when querying + sortedDeliveryEvents := make([]*models.DeliveryEvent, len(allDeliveryEvents)) + copy(sortedDeliveryEvents, allDeliveryEvents) + sort.Slice(sortedDeliveryEvents, func(i, j int) bool { + return sortedDeliveryEvents[i].Delivery.Time.After(sortedDeliveryEvents[j].Delivery.Time) + }) + + t.Run("empty result for unknown tenant", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: "unknown", + Limit: 5, + EventStart: start, }) require.NoError(t, err) assert.Empty(t, response.Data) assert.Empty(t, response.Next) - assert.Empty(t, response.Prev, "prev cursor should be empty when no data") - assert.Equal(t, int64(0), response.Count, "count should be 0 for unknown tenant") + assert.Empty(t, response.Prev) }) - t.Run("comprehensive list & pagination test", func(t *testing.T) { - // First page (0-6) - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Limit: 7, - Start: start, + t.Run("default ordering (delivery_time DESC)", func(t *testing.T) { + // Verify default ordering is by delivery_time DESC + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Limit: 10, + EventStart: start, }) require.NoError(t, err) - require.Len(t, response.Data, 7, "first page should have 7 items") - firstPageNext := response.Next + require.Len(t, response.Data, 10) - // Second page (7-13) - response, err = logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Limit: 7, - Next: response.Next, - Start: start, + // Verify ordering: should be sorted by delivery_time DESC + for i, de := range response.Data { + assert.Equal(t, sortedDeliveryEvents[i].Delivery.ID, de.Delivery.ID, + "delivery ID mismatch at position %d", i) + } + + // Verify first page has next cursor but no prev cursor + assert.NotEmpty(t, response.Next, "should have next cursor with more data") + assert.Empty(t, response.Prev, "first page should have no prev cursor") + }) + + t.Run("filter by destination", func(t *testing.T) { + destID := destinationIDs[0] + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + DestinationIDs: []string{destID}, + Limit: 100, + EventStart: start, }) require.NoError(t, err) - require.Len(t, response.Data, 7, "second page should have 7 items") - secondPageNext := response.Next - secondPagePrev := response.Prev + require.Len(t, response.Data, len(destinationDeliveryEvents[destID])) + for _, de := range response.Data { + assert.Equal(t, destID, de.DestinationID) + } + }) - // Last page (14-19, partial) - response, err = logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Limit: 7, - Next: response.Next, - Start: start, + t.Run("filter by multiple destinations", func(t *testing.T) { + destIDs := []string{destinationIDs[0], destinationIDs[1]} + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + DestinationIDs: destIDs, + Limit: 100, + EventStart: start, }) require.NoError(t, err) - require.Len(t, response.Data, 6, "last page should have 6 items") + expectedCount := len(destinationDeliveryEvents[destIDs[0]]) + len(destinationDeliveryEvents[destIDs[1]]) + require.Len(t, response.Data, expectedCount) + for _, de := range response.Data { + assert.Contains(t, destIDs, de.DestinationID) + } + }) - // Go back to second page (7-13) - response, err = logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Limit: 7, - Prev: response.Prev, - Start: start, + t.Run("filter by status", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Status: "success", + Limit: 100, + EventStart: start, }) require.NoError(t, err) - require.Len(t, response.Data, 7, "going back to second page should have 7 items") - require.Equal(t, int64(20), response.Count) - require.NotEmpty(t, response.Prev, "prev cursor should be present") - require.NotEmpty(t, response.Next, "next cursor should be present") - require.Equal(t, secondPageNext, response.Next, "next cursor should match original second page next") - require.Equal(t, secondPagePrev, response.Prev, "prev cursor should match original second page prev") + require.Len(t, response.Data, len(statusDeliveryEvents["success"])) + for _, de := range response.Data { + assert.Equal(t, "success", de.Delivery.Status) + } + }) - // Back to first page (0-6) - response, err = logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Limit: 7, - Prev: response.Prev, - Start: start, + t.Run("filter by single topic", func(t *testing.T) { + topic := testutil.TestTopics[0] + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Topics: []string{topic}, + Limit: 100, + EventStart: start, }) require.NoError(t, err) - require.Len(t, response.Data, 7, "back to first page should have 7 items") - for i := 0; i < 7; i++ { - require.Equal(t, events[i].ID, response.Data[i].ID) + require.Len(t, response.Data, len(topicDeliveryEvents[topic])) + for _, de := range response.Data { + assert.Equal(t, topic, de.Event.Topic) } - require.Equal(t, int64(20), response.Count) - require.Empty(t, response.Prev, "prev cursor should be empty on first page") - require.NotEmpty(t, response.Next, "next cursor should be present") - require.Equal(t, firstPageNext, response.Next, "next cursor should match original first page next") }) - t.Run("query by destinations", func(t *testing.T) { - t.Run("list event with destination filter", func(t *testing.T) { - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - DestinationIDs: []string{destinationIDs[0]}, - Limit: 3, - Next: "", - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 3) - for i := 0; i < 3; i++ { - require.Equal(t, destinationEvents[destinationIDs[0]][i].ID, response.Data[i].ID) - } - assert.Equal(t, int64(len(destinationEvents[destinationIDs[0]])), response.Count, "count should match events for destination") + t.Run("filter by multiple topics", func(t *testing.T) { + topics := []string{testutil.TestTopics[0], testutil.TestTopics[1]} + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Topics: topics, + Limit: 100, + EventStart: start, + }) + require.NoError(t, err) + expectedCount := len(topicDeliveryEvents[topics[0]]) + len(topicDeliveryEvents[topics[1]]) + require.Len(t, response.Data, expectedCount) + for _, de := range response.Data { + assert.Contains(t, topics, de.Event.Topic) + } + }) - // Step 2: list with cursor - response, err = logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - DestinationIDs: []string{destinationIDs[0]}, - Limit: 3, - Next: response.Next, - Start: start, + t.Run("filter by event ID (replaces ListDelivery)", func(t *testing.T) { + t.Run("returns empty for unknown event", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventID: "unknown-event", + Limit: 100, + EventStart: start, }) require.NoError(t, err) - require.Len(t, response.Data, 3) - for i := 0; i < 3; i++ { - require.Equal(t, destinationEvents[destinationIDs[0]][3+i].ID, response.Data[i].ID) - } + assert.Empty(t, response.Data) }) - t.Run("list event with destination array filter", func(t *testing.T) { - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - DestinationIDs: []string{destinationIDs[0], destinationIDs[1]}, - Limit: 3, - Next: "", - Start: start, + t.Run("returns all deliveries for event", func(t *testing.T) { + eventID := "evt_00" // This event has retry (i%3==0), so 2 deliveries + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventID: eventID, + Limit: 100, + EventStart: start, }) require.NoError(t, err) - require.Len(t, response.Data, 3) - - // should equal events index 0, 1, 3 - require.Equal(t, events[0].ID, response.Data[0].ID) - require.Equal(t, events[1].ID, response.Data[1].ID) - require.Equal(t, events[3].ID, response.Data[2].ID) + require.Len(t, response.Data, 2, "evt_00 should have 2 deliveries (init failed + final)") + for _, de := range response.Data { + assert.Equal(t, eventID, de.Event.ID) + } + }) - // Step 2: list with cursor - response, err = logStore.ListEvent(ctx, driver.ListEventRequest{ + t.Run("filter by event ID and destination", func(t *testing.T) { + eventID := "evt_00" + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ TenantID: tenantID, - DestinationIDs: []string{destinationIDs[0], destinationIDs[1]}, - Limit: 3, - Next: response.Next, - Start: start, + EventID: eventID, + DestinationIDs: []string{destinationIDs[0]}, // evt_00 goes to destinationIDs[0] + Limit: 100, + EventStart: start, }) require.NoError(t, err) - require.Len(t, response.Data, 3) - - // should equal events index 4, 6, 7 - require.Equal(t, events[4].ID, response.Data[0].ID) - require.Equal(t, events[6].ID, response.Data[1].ID) - require.Equal(t, events[7].ID, response.Data[2].ID) + require.Len(t, response.Data, 2) + for _, de := range response.Data { + assert.Equal(t, eventID, de.Event.ID) + assert.Equal(t, destinationIDs[0], de.DestinationID) + } }) }) - t.Run("query by status", func(t *testing.T) { - t.Run("list event with status filter (success)", func(t *testing.T) { - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Status: "success", - Limit: 5, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 5) - for i := 0; i < 5; i++ { - require.Equal(t, statusEvents["success"][i].ID, response.Data[i].ID) - require.Equal(t, "success", response.Data[i].Status) - } - assert.Equal(t, int64(len(statusEvents["success"])), response.Count, "count should match successful events") - - // Step 2: list with cursor - response, err = logStore.ListEvent(ctx, driver.ListEventRequest{ + t.Run("time range filtering", func(t *testing.T) { + t.Run("default time range (last hour)", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ TenantID: tenantID, - Status: "success", - Limit: 5, - Next: response.Next, - Start: start, + Limit: 100, + // No Start/End - defaults to last hour }) require.NoError(t, err) - require.Len(t, response.Data, 5) - for i := 0; i < 5; i++ { - require.Equal(t, statusEvents["success"][5+i].ID, response.Data[i].ID) - require.Equal(t, "success", response.Data[i].Status) - } + require.Len(t, response.Data, len(timeDeliveryEvents["1h"])) }) - t.Run("list event with status filter (failed)", func(t *testing.T) { - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Status: "failed", - Start: start, + t.Run("explicit time window", func(t *testing.T) { + sevenHoursAgo := baseTime.Add(-7 * time.Hour) + fiveHoursAgo := baseTime.Add(-5 * time.Hour) + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &sevenHoursAgo, + EventEnd: &fiveHoursAgo, + Limit: 100, }) require.NoError(t, err) - require.Len(t, response.Data, len(statusEvents["failed"])) - for i := 0; i < len(statusEvents["failed"]); i++ { - require.Equal(t, statusEvents["failed"][i].ID, response.Data[i].ID) - require.Equal(t, "failed", response.Data[i].Status) - } + require.Len(t, response.Data, len(timeDeliveryEvents["6h"])) }) + }) - t.Run("retrieve event status", func(t *testing.T) { - // Test success case - event, err := logStore.RetrieveEvent(ctx, tenantID, statusEvents["success"][0].ID) - require.NoError(t, err) - require.Equal(t, "success", event.Status) + t.Run("combined filters", func(t *testing.T) { + threeHoursAgo := baseTime.Add(-3 * time.Hour) + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &threeHoursAgo, + DestinationIDs: []string{destinationIDs[0]}, + Status: "success", + Topics: []string{testutil.TestTopics[0]}, + Limit: 100, + }) + require.NoError(t, err) + for _, de := range response.Data { + assert.Equal(t, destinationIDs[0], de.DestinationID) + assert.Equal(t, "success", de.Delivery.Status) + assert.Equal(t, testutil.TestTopics[0], de.Event.Topic) + assert.True(t, de.Event.Time.After(threeHoursAgo)) + } + }) - // Test failed case - event, err = logStore.RetrieveEvent(ctx, tenantID, statusEvents["failed"][0].ID) - require.NoError(t, err) - require.Equal(t, "failed", event.Status) + t.Run("verify all fields returned correctly", func(t *testing.T) { + // Get first delivery event and verify all fields + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventID: "evt_00", // Known event with specific data + Limit: 1, + EventStart: start, }) + require.NoError(t, err) + require.Len(t, response.Data, 1) + + de := response.Data[0] + + // Event fields + assert.Equal(t, "evt_00", de.Event.ID) + assert.Equal(t, tenantID, de.Event.TenantID) + assert.Equal(t, destinationIDs[0], de.Event.DestinationID) // i=0 -> destinationIDs[0%3] + assert.Equal(t, testutil.TestTopics[0], de.Event.Topic) // i=0 -> TestTopics[0%len] + assert.Equal(t, true, de.Event.EligibleForRetry) // i=0 -> 0%3==0 -> true + assert.NotNil(t, de.Event.Metadata) + assert.Equal(t, "0", de.Event.Metadata["index"]) + + // Delivery fields + assert.NotEmpty(t, de.Delivery.ID) + assert.Equal(t, "evt_00", de.Delivery.EventID) + assert.Equal(t, destinationIDs[0], de.Delivery.DestinationID) + assert.Contains(t, []string{"success", "failed"}, de.Delivery.Status) + assert.False(t, de.Delivery.Time.IsZero()) }) - t.Run("query by status and destination", func(t *testing.T) { - t.Run("list event with status and destination filter (success)", func(t *testing.T) { - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - DestinationIDs: []string{destinationIDs[0]}, - Status: "success", - Limit: 2, - Start: start, + t.Run("limit edge cases", func(t *testing.T) { + t.Run("limit 1 returns single item", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Limit: 1, + EventStart: start, }) require.NoError(t, err) - require.Len(t, response.Data, 2) - for i := 0; i < 2; i++ { - require.Equal(t, destinationStatusEvents[destinationIDs[0]]["success"][i].ID, response.Data[i].ID) - } + require.Len(t, response.Data, 1) + assert.NotEmpty(t, response.Next, "should have next cursor with more data") + }) - // Step 2: list with cursor - response, err = logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - DestinationIDs: []string{destinationIDs[0]}, - Status: "success", - Limit: 2, - Next: response.Next, - Start: start, + t.Run("limit greater than total returns all", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Limit: 1000, + EventStart: start, }) require.NoError(t, err) - require.Len(t, response.Data, 2) - for i := 0; i < 2; i++ { - require.Equal(t, destinationStatusEvents[destinationIDs[0]]["success"][2+i].ID, response.Data[i].ID) - } + require.Len(t, response.Data, len(allDeliveryEvents)) + assert.Empty(t, response.Next, "should have no next cursor when all data returned") }) }) - t.Run("query by topic", func(t *testing.T) { +} - t.Run("list events with single topic", func(t *testing.T) { - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Topics: []string{testutil.TestTopics[0]}, - Limit: 2, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 2) - for index, e := range response.Data { - require.Equal(t, testutil.TestTopics[0], e.Topic) - require.Equal(t, topicEvents[e.Topic][index].ID, e.ID) - } +// testRetrieveEvent tests the RetrieveEvent method +func testRetrieveEvent(t *testing.T, newHarness HarnessMaker) { + t.Helper() - // Step 2: list with cursor - response, err = logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Topics: []string{testutil.TestTopics[0]}, - Limit: 2, - Next: response.Next, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 2) - for index, e := range response.Data { - require.Equal(t, testutil.TestTopics[0], e.Topic) - require.Equal(t, topicEvents[e.Topic][2+index].ID, e.ID) - } - }) + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) - t.Run("list events with multiple topics", func(t *testing.T) { - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Topics: testutil.TestTopics[:2], // first two topics - Limit: 2, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 2) - for _, e := range response.Data { - require.Contains(t, testutil.TestTopics[:2], e.Topic) - require.Equal(t, topicEvents[e.Topic][0].ID, e.ID) // first event of each topic - } + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) - // Step 2: list with cursor - response, err = logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Topics: testutil.TestTopics[:2], - Limit: 2, - Next: response.Next, - Start: start, - }) - require.NoError(t, err) - require.Len(t, response.Data, 2) - for _, e := range response.Data { - require.Contains(t, testutil.TestTopics[:2], e.Topic) - require.Equal(t, topicEvents[e.Topic][1].ID, e.ID) // second event of each topic - } - }) - }) + tenantID := uuid.New().String() + destinationID := uuid.New().String() + eventID := uuid.New().String() + eventTime := time.Now().Truncate(time.Millisecond) - t.Run("query by time", func(t *testing.T) { - t.Run("list events with no time params (defaults to last hour)", func(t *testing.T) { + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("user.created"), + testutil.EventFactory.WithTime(eventTime), + testutil.EventFactory.WithEligibleForRetry(true), + testutil.EventFactory.WithMetadata(map[string]string{ + "source": "test", + "env": "development", + }), + testutil.EventFactory.WithData(map[string]interface{}{ + "user_id": "usr_123", + "email": "test@example.com", + }), + ) - // First page - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Limit: 2, // Smaller limit to test pagination - }) - require.NoError(t, err) - require.Len(t, response.Data, 2) - for i, e := range response.Data { - require.Equal(t, timeEvents["1h"][i].ID, e.ID) - } - assert.Equal(t, int64(len(timeEvents["1h"])), response.Count, "count should match events in last hour") + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("success"), + ) - // second window - response, err = logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Limit: 2, - Next: response.Next, - }) - require.NoError(t, err) - require.Len(t, response.Data, 2) - for i, e := range response.Data { - require.Equal(t, timeEvents["1h"][i+2].ID, e.ID) - } - // Should still have one more event in the 1h bucket + de := &models.DeliveryEvent{ + ID: uuid.New().String(), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } - // Final page - response, err = logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Limit: 2, - Next: response.Next, - }) - require.NoError(t, err) - require.Len(t, response.Data, 1) // Last event in the 1h bucket - require.Equal(t, timeEvents["1h"][4].ID, response.Data[0].ID) - // require.Empty(t, response.Next) // No more events + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{de})) + + t.Run("retrieve existing event with all fields", func(t *testing.T) { + retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenantID, + EventID: eventID, }) + require.NoError(t, err) + require.NotNil(t, retrieved) - t.Run("list events from 3 hours ago", func(t *testing.T) { - threeHoursAgo := baseTime.Add(-3 * time.Hour) - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Start: &threeHoursAgo, - }) - require.NoError(t, err) - // Should include events from 1h and 3h buckets - expectedCount := len(timeEvents["1h"]) + len(timeEvents["3h"]) - require.Len(t, response.Data, expectedCount) + // Verify all event fields + assert.Equal(t, eventID, retrieved.ID) + assert.Equal(t, tenantID, retrieved.TenantID) + assert.Equal(t, destinationID, retrieved.DestinationID) + assert.Equal(t, "user.created", retrieved.Topic) + assert.Equal(t, true, retrieved.EligibleForRetry) + assert.WithinDuration(t, eventTime, retrieved.Time, time.Second) + assert.Equal(t, "test", retrieved.Metadata["source"]) + assert.Equal(t, "development", retrieved.Metadata["env"]) + assert.Equal(t, "usr_123", retrieved.Data["user_id"]) + assert.Equal(t, "test@example.com", retrieved.Data["email"]) + }) + + t.Run("retrieve with destination filter", func(t *testing.T) { + retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenantID, + EventID: eventID, + DestinationID: destinationID, }) + require.NoError(t, err) + require.NotNil(t, retrieved) + assert.Equal(t, eventID, retrieved.ID) + assert.Equal(t, destinationID, retrieved.DestinationID) + }) - t.Run("list events with explicit window", func(t *testing.T) { - sevenHoursAgo := baseTime.Add(-7 * time.Hour) - fiveHoursAgo := baseTime.Add(-5 * time.Hour) - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Start: &sevenHoursAgo, - End: &fiveHoursAgo, - Limit: 5, - }) - require.NoError(t, err) - // Should only include events from 6h bucket (5-6h ago) - require.Len(t, response.Data, len(timeEvents["6h"])) - for i, e := range response.Data { - require.Equal(t, timeEvents["6h"][i].ID, e.ID) - } + t.Run("retrieve non-existent event", func(t *testing.T) { + retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenantID, + EventID: "non-existent", }) + require.NoError(t, err) + assert.Nil(t, retrieved) + }) - t.Run("list events with end time only (defaults start to end-1h)", func(t *testing.T) { - twoHoursAgo := baseTime.Add(-2 * time.Hour) - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - End: &twoHoursAgo, - Limit: 5, - }) - require.NoError(t, err) - // Should include events from 3h bucket only (2-3h ago) - require.Len(t, response.Data, len(timeEvents["3h"])) + t.Run("retrieve with wrong tenant", func(t *testing.T) { + retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: "wrong-tenant", + EventID: eventID, }) + require.NoError(t, err) + assert.Nil(t, retrieved, "should not return event for wrong tenant") + }) - t.Run("list events with combined time and other filters", func(t *testing.T) { - threeHoursAgo := baseTime.Add(-3 * time.Hour) - response, err := logStore.ListEvent(ctx, driver.ListEventRequest{ - TenantID: tenantID, - Start: &threeHoursAgo, - Topics: []string{testutil.TestTopics[0]}, - DestinationIDs: []string{destinationIDs[0]}, - Status: "success", - Limit: 5, - }) - require.NoError(t, err) - for _, e := range response.Data { - require.Equal(t, testutil.TestTopics[0], e.Topic) - require.Equal(t, destinationIDs[0], e.DestinationID) - require.Equal(t, "success", e.Status) - require.True(t, e.Time.After(threeHoursAgo)) - } + t.Run("retrieve with wrong destination", func(t *testing.T) { + retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenantID, + EventID: eventID, + DestinationID: "wrong-destination", }) + require.NoError(t, err) + assert.Nil(t, retrieved, "should not return event for wrong destination") }) } -func testIntegrationLogStore_DeliveryCRUD(t *testing.T, newHarness HarnessMaker) { +// testTenantIsolation ensures data from one tenant cannot be accessed by another +func testTenantIsolation(t *testing.T, newHarness HarnessMaker) { t.Helper() ctx := context.Background() @@ -595,49 +657,1630 @@ func testIntegrationLogStore_DeliveryCRUD(t *testing.T, newHarness HarnessMaker) logStore, err := h.MakeDriver(ctx) require.NoError(t, err) + tenant1ID := uuid.New().String() + tenant2ID := uuid.New().String() destinationID := uuid.New().String() - event := testutil.EventFactory.Any() - deliveryEvents := []*models.DeliveryEvent{} - baseTime := time.Now() - for i := 0; i < 20; i++ { - delivery := &models.Delivery{ - ID: uuid.New().String(), - EventID: event.ID, - DeliveryEventID: uuid.New().String(), - DestinationID: destinationID, - Status: "success", - Time: baseTime.Add(-time.Duration(i) * time.Second), - } - deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ - ID: delivery.ID, - DestinationID: delivery.DestinationID, - Event: event, - Delivery: delivery, + + // Create events for tenant1 + event1 := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID("tenant1-event"), + testutil.EventFactory.WithTenantID(tenant1ID), + testutil.EventFactory.WithDestinationID(destinationID), + ) + delivery1 := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithEventID(event1.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + ) + + // Create events for tenant2 + event2 := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID("tenant2-event"), + testutil.EventFactory.WithTenantID(tenant2ID), + testutil.EventFactory.WithDestinationID(destinationID), + ) + delivery2 := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithEventID(event2.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + ) + + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ + {ID: uuid.New().String(), DestinationID: destinationID, Event: *event1, Delivery: delivery1}, + {ID: uuid.New().String(), DestinationID: destinationID, Event: *event2, Delivery: delivery2}, + })) + + startTime := time.Now().Add(-1 * time.Hour) + + t.Run("ListDeliveryEvent isolates by tenant", func(t *testing.T) { + // Tenant1 should only see their events + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenant1ID, + Limit: 100, + EventStart: &startTime, }) - } + require.NoError(t, err) + require.Len(t, response.Data, 1) + assert.Equal(t, "tenant1-event", response.Data[0].Event.ID) - t.Run("insert many delivery", func(t *testing.T) { - require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, deliveryEvents)) + // Tenant2 should only see their events + response, err = logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenant2ID, + Limit: 100, + EventStart: &startTime, + }) + require.NoError(t, err) + require.Len(t, response.Data, 1) + assert.Equal(t, "tenant2-event", response.Data[0].Event.ID) }) - t.Run("list delivery empty", func(t *testing.T) { - queriedDeliveries, err := logStore.ListDelivery(ctx, driver.ListDeliveryRequest{ - EventID: "unknown", - DestinationID: "", + t.Run("RetrieveEvent isolates by tenant", func(t *testing.T) { + // Tenant1 cannot access tenant2's event + retrieved, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenant1ID, + EventID: "tenant2-event", }) require.NoError(t, err) - assert.Empty(t, queriedDeliveries) - }) + assert.Nil(t, retrieved) - t.Run("list delivery", func(t *testing.T) { - queriedDeliveries, err := logStore.ListDelivery(ctx, driver.ListDeliveryRequest{ - EventID: event.ID, - DestinationID: destinationID, + // Tenant2 cannot access tenant1's event + retrieved, err = logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenant2ID, + EventID: "tenant1-event", }) require.NoError(t, err) - assert.Len(t, queriedDeliveries, len(deliveryEvents)) - for i := 0; i < len(deliveryEvents); i++ { - assert.Equal(t, deliveryEvents[i].Delivery.ID, queriedDeliveries[i].ID) - } + assert.Nil(t, retrieved) }) } + +// Helper function to categorize delivery events by time bucket +func categorizeByTime(i int, de *models.DeliveryEvent, timeDeliveryEvents map[string][]*models.DeliveryEvent) { + switch { + case i < 5: + timeDeliveryEvents["1h"] = append(timeDeliveryEvents["1h"], de) + case i < 10: + timeDeliveryEvents["3h"] = append(timeDeliveryEvents["3h"], de) + case i < 15: + timeDeliveryEvents["6h"] = append(timeDeliveryEvents["6h"], de) + default: + timeDeliveryEvents["24h"] = append(timeDeliveryEvents["24h"], de) + } +} + +// ============================================================================= +// SIMPLE PAGINATION TEST +// ============================================================================= +// +// A quick sanity check for pagination during development. Tests core mechanics +// with minimal data. Run the full TestPaginationSuite for comprehensive testing. +// +// Usage: make test TEST=./internal/logstore/memlogstore TESTARGS="-run TestPaginationSimple" +// ============================================================================= + +func testPaginationSimple(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + tenantID := uuid.New().String() + destinationID := uuid.New().String() + baseTime := time.Now().Truncate(time.Second) + + // Create 5 delivery events with distinct times + var allEvents []*models.DeliveryEvent + for i := 0; i < 5; i++ { + event := &models.Event{ + ID: fmt.Sprintf("evt_%d", i), + TenantID: tenantID, + DestinationID: destinationID, + Topic: "test.topic", + Time: baseTime.Add(-time.Duration(i) * time.Hour), + } + delivery := &models.Delivery{ + ID: fmt.Sprintf("del_%d", i), + EventID: event.ID, + DestinationID: destinationID, + Status: "success", + Time: baseTime.Add(-time.Duration(i) * time.Hour), + } + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("de_%d", i), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + allEvents = append(allEvents, de) + } + + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, allEvents)) + + startTime := baseTime.Add(-48 * time.Hour) + + t.Run("forward pagination collects all items", func(t *testing.T) { + var collected []string + req := driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &startTime, + Limit: 2, + } + + response, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + for _, de := range response.Data { + collected = append(collected, de.Delivery.ID) + } + + for response.Next != "" { + req.Next = response.Next + response, err = logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + for _, de := range response.Data { + collected = append(collected, de.Delivery.ID) + } + } + + assert.Len(t, collected, 5, "should collect all 5 items") + // Default sort is delivery_time DESC, so del_0 (most recent) comes first + assert.Equal(t, "del_0", collected[0], "first item should be most recent") + assert.Equal(t, "del_4", collected[4], "last item should be oldest") + }) + + t.Run("backward pagination returns to start", func(t *testing.T) { + req := driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &startTime, + Limit: 2, + } + + // Get first page + firstPage, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + require.NotEmpty(t, firstPage.Next, "should have next cursor") + assert.Empty(t, firstPage.Prev, "first page should have no prev") + + // Go to second page + req.Next = firstPage.Next + secondPage, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + require.NotEmpty(t, secondPage.Prev, "second page should have prev") + + // Go back to first page + req.Next = "" + req.Prev = secondPage.Prev + backToFirst, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + assert.Equal(t, firstPage.Data[0].Delivery.ID, backToFirst.Data[0].Delivery.ID, + "returning to first page should show same data") + assert.Empty(t, backToFirst.Prev, "first page should have no prev") + }) + + t.Run("sorting changes order", func(t *testing.T) { + req := driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &startTime, + SortOrder: "asc", + Limit: 5, + } + + response, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + require.Len(t, response.Data, 5) + + // ASC order: oldest first + assert.Equal(t, "del_4", response.Data[0].Delivery.ID, "ASC: oldest should be first") + assert.Equal(t, "del_0", response.Data[4].Delivery.ID, "ASC: newest should be last") + }) + + t.Run("cursor stability", func(t *testing.T) { + req := driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &startTime, + Limit: 2, + } + + // Same request twice should return identical results + resp1, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + resp2, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + assert.Equal(t, resp1.Next, resp2.Next, "cursors should be stable") + assert.Equal(t, len(resp1.Data), len(resp2.Data)) + for i := range resp1.Data { + assert.Equal(t, resp1.Data[i].Delivery.ID, resp2.Data[i].Delivery.ID) + } + }) +} + +// ============================================================================= +// PAGINATION TEST SUITE +// ============================================================================= +// +// This suite tests pagination behavior with various filters and sort options. +// It uses dedicated test data with realistic timestamps to verify: +// - Forward and backward traversal +// - Cursor stability +// - Different sort orders (event_time vs delivery_time) +// - Time-based filtering (EventStart/End, DeliveryStart/End) +// +// SORTING LOGIC +// ============= +// +// Sorting uses multi-column ordering to ensure deterministic pagination. +// This is critical because: +// 1. Multiple deliveries can have the same event_time (same event, multiple attempts) +// 2. In rare cases, deliveries could have identical timestamps +// 3. Cursor-based pagination requires stable, repeatable ordering +// +// The sorting columns are: +// +// | SortBy | SortOrder | SQL Equivalent | +// |---------------|-----------|-----------------------------------------------------| +// | delivery_time | desc | ORDER BY delivery_time DESC, delivery_id DESC | +// | delivery_time | asc | ORDER BY delivery_time ASC, delivery_id ASC | +// | event_time | desc | ORDER BY event_time DESC, event_id DESC, delivery_time DESC | +// | event_time | asc | ORDER BY event_time ASC, event_id ASC, delivery_time ASC | +// +// Why these columns? +// +// For delivery_time sorting: +// - Primary: delivery_time - the user's requested sort +// - Secondary: delivery_id - tiebreaker for identical timestamps (rare but possible) +// +// For event_time sorting: +// - Primary: event_time - the user's requested sort +// - Secondary: event_id - groups all deliveries for the same event together +// - Tertiary: delivery_time - orders retries within an event chronologically +// +// The secondary/tertiary columns always use the same direction (ASC/DESC) as the +// primary column to maintain consistent ordering semantics. +// +// TEST DATA STRUCTURE +// =================== +// +// We create 10 events spread over 24 hours, each with 1-5 delivery attempts. +// This models realistic webhook delivery with retries. +// +// Timeline (times relative to baseTime, which is "now"): +// +// Event 0: event_time = -1h +// └── del_0_0: delivery_time = -55m (success) +// └── 1 delivery, immediate success +// +// Event 1: event_time = -2h +// ├── del_1_0: delivery_time = -1h55m (failed) +// └── del_1_1: delivery_time = -1h50m (success) +// └── 2 deliveries, 1 retry +// +// Event 2: event_time = -3h +// ├── del_2_0: delivery_time = -2h55m (failed) +// ├── del_2_1: delivery_time = -2h50m (failed) +// └── del_2_2: delivery_time = -2h45m (success) +// └── 3 deliveries, 2 retries +// +// Event 3: event_time = -5h +// ├── del_3_0: delivery_time = -4h55m (failed) +// ├── del_3_1: delivery_time = -4h50m (failed) +// ├── del_3_2: delivery_time = -4h45m (failed) +// └── del_3_3: delivery_time = -4h40m (success) +// └── 4 deliveries, 3 retries +// +// Event 4: event_time = -6h +// ├── del_4_0: delivery_time = -5h55m (failed) +// ├── del_4_1: delivery_time = -5h50m (failed) +// ├── del_4_2: delivery_time = -5h45m (failed) +// ├── del_4_3: delivery_time = -5h40m (failed) +// └── del_4_4: delivery_time = -5h35m (success) +// └── 5 deliveries, 4 retries +// +// Event 5: event_time = -8h +// ├── del_5_0: delivery_time = -7h55m (failed) +// └── del_5_1: delivery_time = -7h50m (success) +// └── 2 deliveries, 1 retry +// +// Event 6: event_time = -12h +// ├── del_6_0: delivery_time = -11h55m (failed) +// ├── del_6_1: delivery_time = -11h50m (failed) +// └── del_6_2: delivery_time = -11h45m (success) +// └── 3 deliveries, 2 retries +// +// Event 7: event_time = -18h +// └── del_7_0: delivery_time = -17h55m (success) +// └── 1 delivery, immediate success +// +// Event 8: event_time = -20h +// ├── del_8_0: delivery_time = -19h55m (failed) +// └── del_8_1: delivery_time = -19h50m (success) +// └── 2 deliveries, 1 retry +// +// Event 9: event_time = -23h +// ├── del_9_0: delivery_time = -22h55m (failed) +// ├── del_9_1: delivery_time = -22h50m (failed) +// ├── del_9_2: delivery_time = -22h45m (failed) +// └── del_9_3: delivery_time = -22h40m (success) +// └── 4 deliveries, 3 retries +// +// NAMING CONVENTION +// ================= +// +// - Event ID: evt_{event_index} +// Example: evt_3 is the 4th event (0-indexed) +// +// - Delivery ID: del_{event_index}_{delivery_index} +// Example: del_3_2 is event 3's 3rd delivery attempt (0-indexed) +// +// This makes it easy to understand relationships when debugging: +// - del_3_2 → belongs to evt_3, is the 3rd attempt +// +// TOTALS +// ====== +// +// - 10 events +// - 27 delivery events total (1+2+3+4+5+2+3+1+2+4) +// - Event times span: -1h to -23h +// - Delivery times span: -55m to -22h40m +// +// ============================================================================= + +// ============================================================================= +// PAGINATION SUITE TYPES AND HELPERS +// ============================================================================= + +// PaginationTestCase defines a single filter/sort combination to test +type PaginationTestCase struct { + Name string + Request driver.ListDeliveryEventRequest // Base request (TenantID will be set by suite) + Expected []*models.DeliveryEvent // Expected results in exact order +} + +// PaginationSuiteData contains everything needed to run the pagination suite +type PaginationSuiteData struct { + // Name describes this test data set (e.g., "realistic_timestamps", "identical_timestamps") + Name string + + // TenantID for all test data + TenantID string + + // TestCases are the filter/sort combinations to test + // Each test case runs through all pagination behaviors (forward, backward, zigzag, etc.) + TestCases []PaginationTestCase +} + +// PaginationDataGenerator creates test data and returns the suite configuration +// It receives the logStore to insert data and returns the suite data for verification +type PaginationDataGenerator func(t *testing.T, logStore driver.LogStore) *PaginationSuiteData + +// Sorting helper functions for multi-column sorting (see SORTING LOGIC above) +// These are used by generators to compute expected results. + +func compareByDeliveryTime(a, b *models.DeliveryEvent, desc bool) bool { + // Primary: delivery_time + if !a.Delivery.Time.Equal(b.Delivery.Time) { + if desc { + return a.Delivery.Time.After(b.Delivery.Time) + } + return a.Delivery.Time.Before(b.Delivery.Time) + } + // Secondary: delivery_id + if desc { + return a.Delivery.ID > b.Delivery.ID + } + return a.Delivery.ID < b.Delivery.ID +} + +func compareByEventTime(a, b *models.DeliveryEvent, desc bool) bool { + // Primary: event_time + if !a.Event.Time.Equal(b.Event.Time) { + if desc { + return a.Event.Time.After(b.Event.Time) + } + return a.Event.Time.Before(b.Event.Time) + } + // Secondary: event_id + if a.Event.ID != b.Event.ID { + if desc { + return a.Event.ID > b.Event.ID + } + return a.Event.ID < b.Event.ID + } + // Tertiary: delivery_time + if desc { + return a.Delivery.Time.After(b.Delivery.Time) + } + return a.Delivery.Time.Before(b.Delivery.Time) +} + +func sortDeliveryEvents(events []*models.DeliveryEvent, sortBy string, desc bool) []*models.DeliveryEvent { + result := make([]*models.DeliveryEvent, len(events)) + copy(result, events) + sort.Slice(result, func(i, j int) bool { + if sortBy == "event_time" { + return compareByEventTime(result[i], result[j], desc) + } + return compareByDeliveryTime(result[i], result[j], desc) + }) + return result +} + +// ============================================================================= +// DATA GENERATORS +// ============================================================================= + +// generateRealisticTimestampData creates test data with varied, realistic timestamps. +// This is the primary test data set that exercises all filters and sort options. +// +// See TEST DATA STRUCTURE documentation above for details on the data layout. +func generateRealisticTimestampData(t *testing.T, logStore driver.LogStore) *PaginationSuiteData { + t.Helper() + + ctx := context.Background() + tenantID := uuid.New().String() + destinationIDs := []string{ + uuid.New().String(), + uuid.New().String(), + } + baseTime := time.Now().Truncate(time.Second) + + // Event configuration: [event_index] = {hours_ago, num_deliveries} + eventConfigs := []struct { + hoursAgo int + numDeliveries int + }{ + {1, 1}, // evt_0: 1 delivery + {2, 2}, // evt_1: 2 deliveries + {3, 3}, // evt_2: 3 deliveries + {5, 4}, // evt_3: 4 deliveries + {6, 5}, // evt_4: 5 deliveries + {8, 2}, // evt_5: 2 deliveries + {12, 3}, // evt_6: 3 deliveries + {18, 1}, // evt_7: 1 delivery + {20, 2}, // evt_8: 2 deliveries + {23, 4}, // evt_9: 4 deliveries + } + + var allDeliveryEvents []*models.DeliveryEvent + byDestination := make(map[string][]*models.DeliveryEvent) + byStatus := make(map[string][]*models.DeliveryEvent) + + for eventIdx, cfg := range eventConfigs { + eventTime := baseTime.Add(-time.Duration(cfg.hoursAgo) * time.Hour) + destinationID := destinationIDs[eventIdx%len(destinationIDs)] + topic := testutil.TestTopics[eventIdx%len(testutil.TestTopics)] + + event := &models.Event{ + ID: fmt.Sprintf("evt_%d", eventIdx), + TenantID: tenantID, + DestinationID: destinationID, + Topic: topic, + EligibleForRetry: cfg.numDeliveries > 1, + Time: eventTime, + Metadata: map[string]string{"event_index": strconv.Itoa(eventIdx)}, + Data: map[string]interface{}{"test": true}, + } + + // Create deliveries: first delivery is 5 minutes after event, + // subsequent deliveries are 5 minutes apart + for delIdx := 0; delIdx < cfg.numDeliveries; delIdx++ { + deliveryTime := eventTime.Add(5*time.Minute + time.Duration(delIdx)*5*time.Minute) + + // Last delivery succeeds, others fail + status := "failed" + if delIdx == cfg.numDeliveries-1 { + status = "success" + } + + delivery := &models.Delivery{ + ID: fmt.Sprintf("del_%d_%d", eventIdx, delIdx), + EventID: event.ID, + DestinationID: destinationID, + Status: status, + Time: deliveryTime, + Code: "200", + } + + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("de_%d_%d", eventIdx, delIdx), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + + allDeliveryEvents = append(allDeliveryEvents, de) + byDestination[destinationID] = append(byDestination[destinationID], de) + byStatus[status] = append(byStatus[status], de) + } + } + + // Insert all delivery events + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, allDeliveryEvents)) + + // Pre-compute sorted lists + sortedByDeliveryTimeDesc := sortDeliveryEvents(allDeliveryEvents, "delivery_time", true) + sortedByDeliveryTimeAsc := sortDeliveryEvents(allDeliveryEvents, "delivery_time", false) + sortedByEventTimeDesc := sortDeliveryEvents(allDeliveryEvents, "event_time", true) + sortedByEventTimeAsc := sortDeliveryEvents(allDeliveryEvents, "event_time", false) + + // Pre-compute filtered subsets + sixHoursAgo := baseTime.Add(-6 * time.Hour) + twelveHoursAgo := baseTime.Add(-12 * time.Hour) + farPast := baseTime.Add(-48 * time.Hour) + + var eventsInLast6Hours []*models.DeliveryEvent + var eventsFrom6hTo12h []*models.DeliveryEvent + var deliveriesInLast6Hours []*models.DeliveryEvent + var deliveriesFrom6hTo12h []*models.DeliveryEvent + + for _, de := range sortedByDeliveryTimeDesc { + // Filter by event time (inclusive semantics) + if !de.Event.Time.Before(sixHoursAgo) { + eventsInLast6Hours = append(eventsInLast6Hours, de) + } + if !de.Event.Time.Before(twelveHoursAgo) && !de.Event.Time.After(sixHoursAgo) { + eventsFrom6hTo12h = append(eventsFrom6hTo12h, de) + } + + // Filter by delivery time (inclusive semantics) + if !de.Delivery.Time.Before(sixHoursAgo) { + deliveriesInLast6Hours = append(deliveriesInLast6Hours, de) + } + if !de.Delivery.Time.Before(twelveHoursAgo) && !de.Delivery.Time.After(sixHoursAgo) { + deliveriesFrom6hTo12h = append(deliveriesFrom6hTo12h, de) + } + } + + // Sort byDestination and byStatus + for destID := range byDestination { + byDestination[destID] = sortDeliveryEvents(byDestination[destID], "delivery_time", true) + } + for status := range byStatus { + byStatus[status] = sortDeliveryEvents(byStatus[status], "delivery_time", true) + } + successSortedByEventTime := sortDeliveryEvents(byStatus["success"], "event_time", true) + + // Build test cases + return &PaginationSuiteData{ + Name: "realistic_timestamps", + TenantID: tenantID, + TestCases: []PaginationTestCase{ + { + Name: "default sort (delivery_time DESC)", + Request: driver.ListDeliveryEventRequest{EventStart: &farPast}, + Expected: sortedByDeliveryTimeDesc, + }, + { + Name: "sort by event_time DESC", + Request: driver.ListDeliveryEventRequest{EventStart: &farPast, SortBy: "event_time"}, + Expected: sortedByEventTimeDesc, + }, + { + Name: "sort by delivery_time ASC", + Request: driver.ListDeliveryEventRequest{EventStart: &farPast, SortOrder: "asc"}, + Expected: sortedByDeliveryTimeAsc, + }, + { + Name: "sort by event_time ASC", + Request: driver.ListDeliveryEventRequest{EventStart: &farPast, SortBy: "event_time", SortOrder: "asc"}, + Expected: sortedByEventTimeAsc, + }, + { + Name: "filter by event time (last 6 hours)", + Request: driver.ListDeliveryEventRequest{EventStart: &sixHoursAgo}, + Expected: eventsInLast6Hours, + }, + { + Name: "filter by event time range (6h to 12h ago)", + Request: driver.ListDeliveryEventRequest{EventStart: &twelveHoursAgo, EventEnd: &sixHoursAgo}, + Expected: eventsFrom6hTo12h, + }, + { + Name: "filter by delivery time (last 6 hours)", + Request: driver.ListDeliveryEventRequest{DeliveryStart: &sixHoursAgo, EventStart: &farPast}, + Expected: deliveriesInLast6Hours, + }, + { + Name: "filter by delivery time range (6h to 12h ago)", + Request: driver.ListDeliveryEventRequest{DeliveryStart: &twelveHoursAgo, DeliveryEnd: &sixHoursAgo, EventStart: &farPast}, + Expected: deliveriesFrom6hTo12h, + }, + { + Name: "filter by destination", + Request: driver.ListDeliveryEventRequest{DestinationIDs: []string{destinationIDs[0]}, EventStart: &farPast}, + Expected: byDestination[destinationIDs[0]], + }, + { + Name: "filter by status (success)", + Request: driver.ListDeliveryEventRequest{Status: "success", EventStart: &farPast}, + Expected: byStatus["success"], + }, + { + Name: "filter by status with event_time sort", + Request: driver.ListDeliveryEventRequest{Status: "success", SortBy: "event_time", EventStart: &farPast}, + Expected: successSortedByEventTime, + }, + }, + } +} + +// generateIdenticalTimestampData creates test data where all events and deliveries +// have the SAME timestamp. This forces sorting to rely entirely on secondary/tertiary +// columns (event_id, delivery_id) and verifies deterministic ordering. +// +// This is a critical edge case for cursor-based pagination stability. +func generateIdenticalTimestampData(t *testing.T, logStore driver.LogStore) *PaginationSuiteData { + t.Helper() + + ctx := context.Background() + tenantID := uuid.New().String() + destinationID := uuid.New().String() + + // All events and deliveries share the SAME timestamp + sameTime := time.Now().Truncate(time.Second) + farPast := sameTime.Add(-1 * time.Hour) + + // Create 10 events, each with 2 deliveries, all at the same timestamp + var allDeliveryEvents []*models.DeliveryEvent + + for eventIdx := 0; eventIdx < 10; eventIdx++ { + event := &models.Event{ + ID: fmt.Sprintf("evt_%02d", eventIdx), + TenantID: tenantID, + DestinationID: destinationID, + Topic: "test.topic", + EligibleForRetry: true, + Time: sameTime, + Metadata: map[string]string{}, + Data: map[string]interface{}{}, + } + + for delIdx := 0; delIdx < 2; delIdx++ { + status := "failed" + if delIdx == 1 { + status = "success" + } + + delivery := &models.Delivery{ + ID: fmt.Sprintf("del_%02d_%d", eventIdx, delIdx), + EventID: event.ID, + DestinationID: destinationID, + Status: status, + Time: sameTime, + Code: "200", + } + + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("de_%02d_%d", eventIdx, delIdx), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + + allDeliveryEvents = append(allDeliveryEvents, de) + } + } + + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, allDeliveryEvents)) + + // With identical timestamps, sorting falls back to ID columns + sortedByDeliveryTimeDesc := sortDeliveryEvents(allDeliveryEvents, "delivery_time", true) + sortedByDeliveryTimeAsc := sortDeliveryEvents(allDeliveryEvents, "delivery_time", false) + sortedByEventTimeDesc := sortDeliveryEvents(allDeliveryEvents, "event_time", true) + sortedByEventTimeAsc := sortDeliveryEvents(allDeliveryEvents, "event_time", false) + + return &PaginationSuiteData{ + Name: "identical_timestamps", + TenantID: tenantID, + TestCases: []PaginationTestCase{ + { + Name: "delivery_time DESC (falls back to delivery_id)", + Request: driver.ListDeliveryEventRequest{EventStart: &farPast}, + Expected: sortedByDeliveryTimeDesc, + }, + { + Name: "delivery_time ASC (falls back to delivery_id)", + Request: driver.ListDeliveryEventRequest{EventStart: &farPast, SortOrder: "asc"}, + Expected: sortedByDeliveryTimeAsc, + }, + { + Name: "event_time DESC (falls back to event_id)", + Request: driver.ListDeliveryEventRequest{EventStart: &farPast, SortBy: "event_time"}, + Expected: sortedByEventTimeDesc, + }, + { + Name: "event_time ASC (falls back to event_id)", + Request: driver.ListDeliveryEventRequest{EventStart: &farPast, SortBy: "event_time", SortOrder: "asc"}, + Expected: sortedByEventTimeAsc, + }, + }, + } +} + +// testPaginationSuite runs the pagination test suite with multiple data generators. +// Each generator creates a different test data set (e.g., realistic timestamps, +// identical timestamps) and the suite runs all pagination behaviors against each. +func testPaginationSuite(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + // List of data generators to run + generators := []PaginationDataGenerator{ + generateRealisticTimestampData, + generateIdenticalTimestampData, + } + + for _, generator := range generators { + generator := generator // capture range variable + + // Each generator gets its own harness/logstore for isolation + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + // Generate test data + suiteData := generator(t, logStore) + + t.Run(suiteData.Name, func(t *testing.T) { + for _, tc := range suiteData.TestCases { + tc := tc // capture range variable + t.Run(tc.Name, func(t *testing.T) { + // Set tenant ID + tc.Request.TenantID = suiteData.TenantID + + // Skip if no expected data + if len(tc.Expected) == 0 { + t.Skip("No expected data for this test case") + } + + runPaginationTests(t, logStore, tc) + }) + } + }) + + h.Close() + } +} + +// runPaginationTests runs all pagination behavior tests for a single test case +func runPaginationTests(t *testing.T, logStore driver.LogStore, tc PaginationTestCase) { + t.Helper() + ctx := context.Background() + + // Use a page size that creates multiple pages but not too many + pageSize := 5 + if len(tc.Expected) <= pageSize { + pageSize = 2 // Ensure at least 2 pages for small datasets + } + if len(tc.Expected) <= 2 { + pageSize = 1 + } + + t.Run("forward traversal", func(t *testing.T) { + var collected []*models.DeliveryEvent + req := tc.Request + req.Limit = pageSize + + response, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + collected = append(collected, response.Data...) + + for response.Next != "" { + req.Next = response.Next + req.Prev = "" + response, err = logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + collected = append(collected, response.Data...) + } + + require.Len(t, collected, len(tc.Expected), "forward traversal should collect all items") + for i, de := range collected { + assert.Equal(t, tc.Expected[i].Delivery.ID, de.Delivery.ID, + "forward traversal: mismatch at position %d", i) + } + }) + + t.Run("backward traversal", func(t *testing.T) { + // First go to the last page + req := tc.Request + req.Limit = pageSize + + response, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + for response.Next != "" { + req.Next = response.Next + req.Prev = "" + response, err = logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + } + + // Now traverse backward + var collected []*models.DeliveryEvent + collected = append(collected, response.Data...) + + for response.Prev != "" { + req.Prev = response.Prev + req.Next = "" + response, err = logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + collected = append(response.Data, collected...) // Prepend + } + + require.Len(t, collected, len(tc.Expected), "backward traversal should collect all items") + for i, de := range collected { + assert.Equal(t, tc.Expected[i].Delivery.ID, de.Delivery.ID, + "backward traversal: mismatch at position %d", i) + } + }) + + t.Run("forward then backward", func(t *testing.T) { + req := tc.Request + req.Limit = pageSize + + // First page + firstPage, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + if firstPage.Next == "" { + t.Skip("Only one page, cannot test forward then backward") + } + + // Second page + req.Next = firstPage.Next + secondPage, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + // Back to first page + req.Next = "" + req.Prev = secondPage.Prev + backToFirst, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + // Verify we got the same first page + require.Len(t, backToFirst.Data, len(firstPage.Data)) + for i, de := range backToFirst.Data { + assert.Equal(t, firstPage.Data[i].Delivery.ID, de.Delivery.ID, + "back to first page: mismatch at position %d", i) + } + assert.Empty(t, backToFirst.Prev, "first page should have no prev cursor") + }) + + t.Run("zigzag navigation", func(t *testing.T) { + req := tc.Request + req.Limit = pageSize + + // First page + page1, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + if page1.Next == "" { + t.Skip("Only one page, cannot test zigzag") + } + + // Second page + req.Next = page1.Next + page2, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + // Back to first + req.Next = "" + req.Prev = page2.Prev + page1Again, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + // Forward to second again + req.Prev = "" + req.Next = page1Again.Next + page2Again, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + // Verify page 2 is consistent + require.Len(t, page2Again.Data, len(page2.Data)) + for i, de := range page2Again.Data { + assert.Equal(t, page2.Data[i].Delivery.ID, de.Delivery.ID, + "zigzag page 2: mismatch at position %d", i) + } + + // If there's a third page, go there + if page2Again.Next != "" { + req.Prev = "" + req.Next = page2Again.Next + page3, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + assert.NotEmpty(t, page3.Prev, "page 3 should have prev cursor") + } + }) + + t.Run("cursor stability", func(t *testing.T) { + req := tc.Request + req.Limit = pageSize + + // Get first page twice + first1, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + first2, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + // Same data + require.Len(t, first1.Data, len(first2.Data)) + for i := range first1.Data { + assert.Equal(t, first1.Data[i].Delivery.ID, first2.Data[i].Delivery.ID) + } + + // Same cursors + assert.Equal(t, first1.Next, first2.Next) + assert.Equal(t, first1.Prev, first2.Prev) + }) + + t.Run("boundary conditions", func(t *testing.T) { + req := tc.Request + req.Limit = pageSize + + // First page + firstPage, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + assert.Empty(t, firstPage.Prev, "first page should have no prev cursor") + + if len(tc.Expected) > pageSize { + assert.NotEmpty(t, firstPage.Next, "first page should have next cursor when more data exists") + } + + // Go to last page + response := firstPage + for response.Next != "" { + req.Next = response.Next + req.Prev = "" + response, err = logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + } + + assert.Empty(t, response.Next, "last page should have no next cursor") + if len(tc.Expected) > pageSize { + assert.NotEmpty(t, response.Prev, "last page should have prev cursor when more data exists") + } + }) + + t.Run("single item pages", func(t *testing.T) { + req := tc.Request + req.Limit = 1 + + var collected []*models.DeliveryEvent + response, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + require.Len(t, response.Data, 1, "limit 1 should return exactly 1 item") + collected = append(collected, response.Data...) + + for response.Next != "" { + req.Next = response.Next + req.Prev = "" + response, err = logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + require.LessOrEqual(t, len(response.Data), 1, "limit 1 should return at most 1 item") + collected = append(collected, response.Data...) + } + + require.Len(t, collected, len(tc.Expected), "single item pagination should collect all items") + }) +} + +// ============================================================================= +// EDGE CASES TEST SUITE +// ============================================================================= +// +// This suite tests edge cases and boundary conditions that implementations +// must handle correctly: +// - Invalid/unknown sort values (should use defaults) +// - Empty vs nil filter semantics (should be equivalent) +// - Time boundary precision (inclusive semantics) +// - EventID filtering with pagination +// - Data immutability (returned data shouldn't affect stored data) +// ============================================================================= + +func testEdgeCases(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + t.Run("invalid sort values use defaults", func(t *testing.T) { + testInvalidSortValues(t, newHarness) + }) + t.Run("empty vs nil filter semantics", func(t *testing.T) { + testEmptyVsNilFilters(t, newHarness) + }) + t.Run("time boundary precision", func(t *testing.T) { + testTimeBoundaryPrecision(t, newHarness) + }) + t.Run("eventID filtering with pagination", func(t *testing.T) { + testEventIDPagination(t, newHarness) + }) + t.Run("data immutability", func(t *testing.T) { + testDataImmutability(t, newHarness) + }) +} + +// testInvalidSortValues verifies that invalid sort values fall back to defaults +// rather than causing errors. This is important for API robustness. +func testInvalidSortValues(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + tenantID := uuid.New().String() + destinationID := uuid.New().String() + baseTime := time.Now().Truncate(time.Second) + + // Insert test data with distinct delivery times + var deliveryEvents []*models.DeliveryEvent + for i := 0; i < 3; i++ { + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(fmt.Sprintf("evt_%d", i)), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%d", i)), + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), + ) + deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ + ID: fmt.Sprintf("de_%d", i), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + }) + } + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, deliveryEvents)) + + startTime := baseTime.Add(-48 * time.Hour) + + t.Run("invalid SortBy uses default (delivery_time)", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortBy: "invalid_column", + EventStart: &startTime, + Limit: 10, + }) + require.NoError(t, err) + require.Len(t, response.Data, 3) + + // Should be sorted by delivery_time DESC (default) + // del_0 is most recent, del_2 is oldest + assert.Equal(t, "del_0", response.Data[0].Delivery.ID) + assert.Equal(t, "del_1", response.Data[1].Delivery.ID) + assert.Equal(t, "del_2", response.Data[2].Delivery.ID) + }) + + t.Run("invalid SortOrder uses default (desc)", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortOrder: "sideways", + EventStart: &startTime, + Limit: 10, + }) + require.NoError(t, err) + require.Len(t, response.Data, 3) + + // Should be sorted DESC (default) + assert.Equal(t, "del_0", response.Data[0].Delivery.ID) + assert.Equal(t, "del_2", response.Data[2].Delivery.ID) + }) + + t.Run("empty SortBy uses default", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortBy: "", + EventStart: &startTime, + Limit: 10, + }) + require.NoError(t, err) + require.Len(t, response.Data, 3) + assert.Equal(t, "del_0", response.Data[0].Delivery.ID) + }) +} + +// testEmptyVsNilFilters verifies that empty slices and nil slices are treated +// equivalently (both mean "no filter"). +func testEmptyVsNilFilters(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + tenantID := uuid.New().String() + destinationID := uuid.New().String() + startTime := time.Now().Add(-1 * time.Hour) + + // Insert test data + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("test.topic"), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + ) + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ + {ID: uuid.New().String(), DestinationID: destinationID, Event: *event, Delivery: delivery}, + })) + + t.Run("nil DestinationIDs equals empty DestinationIDs", func(t *testing.T) { + responseNil, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + DestinationIDs: nil, + EventStart: &startTime, + Limit: 10, + }) + require.NoError(t, err) + + responseEmpty, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + DestinationIDs: []string{}, + EventStart: &startTime, + Limit: 10, + }) + require.NoError(t, err) + + assert.Equal(t, len(responseNil.Data), len(responseEmpty.Data), + "nil and empty DestinationIDs should return same count") + assert.Equal(t, 1, len(responseNil.Data), "should return all data when no filter") + }) + + t.Run("nil Topics equals empty Topics", func(t *testing.T) { + responseNil, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Topics: nil, + EventStart: &startTime, + Limit: 10, + }) + require.NoError(t, err) + + responseEmpty, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Topics: []string{}, + EventStart: &startTime, + Limit: 10, + }) + require.NoError(t, err) + + assert.Equal(t, len(responseNil.Data), len(responseEmpty.Data), + "nil and empty Topics should return same count") + }) + + t.Run("empty Status equals no status filter", func(t *testing.T) { + responseEmpty, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + Status: "", + EventStart: &startTime, + Limit: 10, + }) + require.NoError(t, err) + + responseNoFilter, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &startTime, + Limit: 10, + }) + require.NoError(t, err) + + assert.Equal(t, len(responseEmpty.Data), len(responseNoFilter.Data), + "empty Status should return same as no Status filter") + }) +} + +// testTimeBoundaryPrecision verifies that time filters use inclusive semantics: +// - EventStart: >= (events at exactly start time are included) +// - EventEnd: <= (events at exactly end time are included) +// - DeliveryStart: >= (deliveries at exactly start time are included) +// - DeliveryEnd: <= (deliveries at exactly end time are included) +func testTimeBoundaryPrecision(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + tenantID := uuid.New().String() + destinationID := uuid.New().String() + + // Create events at precise times + boundaryTime := time.Now().Truncate(time.Second) + beforeBoundary := boundaryTime.Add(-1 * time.Second) + afterBoundary := boundaryTime.Add(1 * time.Second) + + // Event exactly at boundary + eventAtBoundary := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID("evt_at_boundary"), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(boundaryTime), + ) + deliveryAtBoundary := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID("del_at_boundary"), + testutil.DeliveryFactory.WithEventID(eventAtBoundary.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithTime(boundaryTime), + ) + + // Event before boundary + eventBefore := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID("evt_before"), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(beforeBoundary), + ) + deliveryBefore := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID("del_before"), + testutil.DeliveryFactory.WithEventID(eventBefore.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithTime(beforeBoundary), + ) + + // Event after boundary + eventAfter := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID("evt_after"), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(afterBoundary), + ) + deliveryAfter := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID("del_after"), + testutil.DeliveryFactory.WithEventID(eventAfter.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithTime(afterBoundary), + ) + + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ + {ID: "de_at", DestinationID: destinationID, Event: *eventAtBoundary, Delivery: deliveryAtBoundary}, + {ID: "de_before", DestinationID: destinationID, Event: *eventBefore, Delivery: deliveryBefore}, + {ID: "de_after", DestinationID: destinationID, Event: *eventAfter, Delivery: deliveryAfter}, + })) + + t.Run("EventStart is inclusive (>=)", func(t *testing.T) { + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &boundaryTime, + Limit: 10, + }) + require.NoError(t, err) + + // Should include event at boundary and after, but not before + ids := extractEventIDs(response.Data) + assert.Contains(t, ids, "evt_at_boundary", "EventStart should include events at exact boundary") + assert.Contains(t, ids, "evt_after", "EventStart should include events after boundary") + assert.NotContains(t, ids, "evt_before", "EventStart should exclude events before boundary") + }) + + t.Run("EventEnd is inclusive (<=)", func(t *testing.T) { + farPast := boundaryTime.Add(-1 * time.Hour) + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &farPast, + EventEnd: &boundaryTime, + Limit: 10, + }) + require.NoError(t, err) + + // Should include event at boundary and before, but not after + ids := extractEventIDs(response.Data) + assert.Contains(t, ids, "evt_at_boundary", "EventEnd should include events at exact boundary") + assert.Contains(t, ids, "evt_before", "EventEnd should include events before boundary") + assert.NotContains(t, ids, "evt_after", "EventEnd should exclude events after boundary") + }) + + t.Run("DeliveryStart is inclusive (>=)", func(t *testing.T) { + farPast := boundaryTime.Add(-1 * time.Hour) + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &farPast, + DeliveryStart: &boundaryTime, + Limit: 10, + }) + require.NoError(t, err) + + // Should include delivery at boundary and after, but not before + ids := extractDeliveryIDs(response.Data) + assert.Contains(t, ids, "del_at_boundary", "DeliveryStart should include deliveries at exact boundary") + assert.Contains(t, ids, "del_after", "DeliveryStart should include deliveries after boundary") + assert.NotContains(t, ids, "del_before", "DeliveryStart should exclude deliveries before boundary") + }) + + t.Run("DeliveryEnd is inclusive (<=)", func(t *testing.T) { + farPast := boundaryTime.Add(-1 * time.Hour) + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &farPast, + DeliveryEnd: &boundaryTime, + Limit: 10, + }) + require.NoError(t, err) + + // Should include delivery at boundary and before, but not after + ids := extractDeliveryIDs(response.Data) + assert.Contains(t, ids, "del_at_boundary", "DeliveryEnd should include deliveries at exact boundary") + assert.Contains(t, ids, "del_before", "DeliveryEnd should include deliveries before boundary") + assert.NotContains(t, ids, "del_after", "DeliveryEnd should exclude deliveries after boundary") + }) + + t.Run("exact range includes boundary items", func(t *testing.T) { + // Range from exactly beforeBoundary to exactly afterBoundary + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &beforeBoundary, + EventEnd: &afterBoundary, + Limit: 10, + }) + require.NoError(t, err) + + // Should include all three + assert.Len(t, response.Data, 3, "range should include all items including boundaries") + }) +} + +// testEventIDPagination verifies pagination works correctly when filtering by EventID. +// This is a common use case: viewing all delivery attempts for a specific event. +func testEventIDPagination(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + tenantID := uuid.New().String() + destinationID := uuid.New().String() + eventID := "evt_with_many_deliveries" + baseTime := time.Now().Truncate(time.Second) + + // Create one event with many delivery attempts (simulating retries) + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(baseTime.Add(-1*time.Hour)), + ) + + // Create 10 delivery attempts + var deliveryEvents []*models.DeliveryEvent + for i := 0; i < 10; i++ { + status := "failed" + if i == 9 { + status = "success" + } + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(fmt.Sprintf("del_%02d", i)), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus(status), + testutil.DeliveryFactory.WithTime(baseTime.Add(-time.Duration(60-i)*time.Minute)), + ) + deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ + ID: fmt.Sprintf("de_%02d", i), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + }) + } + + // Also insert a different event to ensure EventID filter works + otherEvent := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID("other_event"), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + ) + otherDelivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithEventID(otherEvent.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + ) + deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ + ID: "de_other", + DestinationID: destinationID, + Event: *otherEvent, + Delivery: otherDelivery, + }) + + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, deliveryEvents)) + + startTime := baseTime.Add(-2 * time.Hour) + + t.Run("forward pagination with EventID filter", func(t *testing.T) { + var collected []*models.DeliveryEvent + req := driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventID: eventID, + EventStart: &startTime, + Limit: 3, // Small page size to test pagination + } + + response, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + collected = append(collected, response.Data...) + + for response.Next != "" { + req.Next = response.Next + response, err = logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + collected = append(collected, response.Data...) + } + + assert.Len(t, collected, 10, "should collect exactly 10 deliveries for the event") + + // All should belong to the same event + for _, de := range collected { + assert.Equal(t, eventID, de.Event.ID, "all deliveries should be for the filtered event") + } + }) + + t.Run("backward pagination with EventID filter", func(t *testing.T) { + req := driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventID: eventID, + EventStart: &startTime, + Limit: 3, + } + + // Go to last page + response, err := logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + + for response.Next != "" { + req.Next = response.Next + req.Prev = "" + response, err = logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + } + + // Now go backward + var collected []*models.DeliveryEvent + collected = append(collected, response.Data...) + + for response.Prev != "" { + req.Prev = response.Prev + req.Next = "" + response, err = logStore.ListDeliveryEvent(ctx, req) + require.NoError(t, err) + collected = append(response.Data, collected...) + } + + assert.Len(t, collected, 10, "backward traversal should collect all deliveries") + }) +} + +// testDataImmutability verifies that modifying returned data doesn't affect +// subsequent queries. This is important for data integrity. +func testDataImmutability(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + tenantID := uuid.New().String() + destinationID := uuid.New().String() + eventID := "immutable_event" + startTime := time.Now().Add(-1 * time.Hour) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("original.topic"), + testutil.EventFactory.WithMetadata(map[string]string{"key": "original"}), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID("original_delivery"), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("success"), + ) + + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ + {ID: "de_immutable", DestinationID: destinationID, Event: *event, Delivery: delivery}, + })) + + t.Run("modifying ListDeliveryEvent result doesn't affect subsequent queries", func(t *testing.T) { + // First query + response1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &startTime, + Limit: 10, + }) + require.NoError(t, err) + require.Len(t, response1.Data, 1) + + // Mutate the returned data + response1.Data[0].Event.Topic = "MUTATED" + response1.Data[0].Event.Metadata["key"] = "MUTATED" + response1.Data[0].Delivery.Status = "MUTATED" + + // Second query should return original values + response2, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &startTime, + Limit: 10, + }) + require.NoError(t, err) + require.Len(t, response2.Data, 1) + + assert.Equal(t, "original.topic", response2.Data[0].Event.Topic, + "Event.Topic should not be affected by mutation") + assert.Equal(t, "original", response2.Data[0].Event.Metadata["key"], + "Event.Metadata should not be affected by mutation") + assert.Equal(t, "success", response2.Data[0].Delivery.Status, + "Delivery.Status should not be affected by mutation") + }) + + t.Run("modifying RetrieveEvent result doesn't affect subsequent queries", func(t *testing.T) { + // First retrieval + event1, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenantID, + EventID: eventID, + }) + require.NoError(t, err) + require.NotNil(t, event1) + + // Mutate the returned event + event1.Topic = "MUTATED" + event1.Metadata["key"] = "MUTATED" + + // Second retrieval should return original values + event2, err := logStore.RetrieveEvent(ctx, driver.RetrieveEventRequest{ + TenantID: tenantID, + EventID: eventID, + }) + require.NoError(t, err) + require.NotNil(t, event2) + + assert.Equal(t, "original.topic", event2.Topic, + "RetrieveEvent should return fresh copy") + assert.Equal(t, "original", event2.Metadata["key"], + "RetrieveEvent metadata should not be affected by mutation") + }) +} + +// Helper function to extract event IDs from delivery events +func extractEventIDs(des []*models.DeliveryEvent) []string { + ids := make([]string, len(des)) + for i, de := range des { + ids[i] = de.Event.ID + } + return ids +} + +// Helper function to extract delivery IDs from delivery events +func extractDeliveryIDs(des []*models.DeliveryEvent) []string { + ids := make([]string, len(des)) + for i, de := range des { + ids[i] = de.Delivery.ID + } + return ids +} diff --git a/internal/logstore/logstore.go b/internal/logstore/logstore.go index a94239ff..63d0211d 100644 --- a/internal/logstore/logstore.go +++ b/internal/logstore/logstore.go @@ -12,15 +12,13 @@ import ( "github.com/jackc/pgx/v5/pgxpool" ) -type ListEventRequest = driver.ListEventRequest -type ListEventResponse = driver.ListEventResponse -type ListDeliveryRequest = driver.ListDeliveryRequest +type ListDeliveryEventRequest = driver.ListDeliveryEventRequest +type ListDeliveryEventResponse = driver.ListDeliveryEventResponse +type RetrieveEventRequest = driver.RetrieveEventRequest type LogStore interface { - ListEvent(context.Context, ListEventRequest) (ListEventResponse, error) - RetrieveEvent(ctx context.Context, tenantID, eventID string) (*models.Event, error) - RetrieveEventByDestination(ctx context.Context, tenantID, destinationID, eventID string) (*models.Event, error) - ListDelivery(ctx context.Context, request ListDeliveryRequest) ([]*models.Delivery, error) + ListDeliveryEvent(context.Context, ListDeliveryEventRequest) (ListDeliveryEventResponse, error) + RetrieveEvent(ctx context.Context, request RetrieveEventRequest) (*models.Event, error) InsertManyDeliveryEvent(context.Context, []*models.DeliveryEvent) error } diff --git a/internal/logstore/memlogstore/memlogstore.go b/internal/logstore/memlogstore/memlogstore.go new file mode 100644 index 00000000..57a3352c --- /dev/null +++ b/internal/logstore/memlogstore/memlogstore.go @@ -0,0 +1,319 @@ +package memlogstore + +import ( + "context" + "sort" + "sync" + "time" + + "github.com/hookdeck/outpost/internal/logstore/driver" + "github.com/hookdeck/outpost/internal/models" +) + +// memLogStore is an in-memory implementation of driver.LogStore. +// It serves as a reference implementation and is useful for testing. +type memLogStore struct { + mu sync.RWMutex + deliveryEvents []*models.DeliveryEvent +} + +var _ driver.LogStore = (*memLogStore)(nil) + +func NewLogStore() driver.LogStore { + return &memLogStore{ + deliveryEvents: make([]*models.DeliveryEvent, 0), + } +} + +func (s *memLogStore) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents []*models.DeliveryEvent) error { + s.mu.Lock() + defer s.mu.Unlock() + + // Deep copy to avoid external mutation + for _, de := range deliveryEvents { + copied := &models.DeliveryEvent{ + ID: de.ID, + DestinationID: de.DestinationID, + Event: de.Event, + Delivery: de.Delivery, + } + s.deliveryEvents = append(s.deliveryEvents, copied) + } + return nil +} + +func (s *memLogStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeliveryEventRequest) (driver.ListDeliveryEventResponse, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + // Apply default time range if not specified + now := time.Now() + if req.EventStart == nil && req.EventEnd == nil && req.DeliveryStart == nil && req.DeliveryEnd == nil { + // Default to last hour based on event time + defaultStart := now.Add(-1 * time.Hour) + req.EventStart = &defaultStart + } + + // Filter + var filtered []*models.DeliveryEvent + for _, de := range s.deliveryEvents { + if !s.matchesFilter(de, req) { + continue + } + filtered = append(filtered, de) + } + + // Sort using multi-column ordering for deterministic pagination. + // See drivertest.go for detailed documentation on sorting logic. + // + // Summary: + // - delivery_time: ORDER BY delivery_time, delivery_id + // - event_time: ORDER BY event_time, event_id, delivery_time + // + // The secondary/tertiary columns ensure deterministic ordering when + // primary sort values are identical (e.g., multiple deliveries for same event). + sortBy := req.SortBy + if sortBy != "event_time" && sortBy != "delivery_time" { + sortBy = "delivery_time" + } + sortOrder := req.SortOrder + if sortOrder != "asc" && sortOrder != "desc" { + sortOrder = "desc" + } + isDesc := sortOrder == "desc" + + sort.Slice(filtered, func(i, j int) bool { + if sortBy == "event_time" { + // Primary: event_time + if !filtered[i].Event.Time.Equal(filtered[j].Event.Time) { + if isDesc { + return filtered[i].Event.Time.After(filtered[j].Event.Time) + } + return filtered[i].Event.Time.Before(filtered[j].Event.Time) + } + // Secondary: event_id (groups deliveries for same event) + if filtered[i].Event.ID != filtered[j].Event.ID { + if isDesc { + return filtered[i].Event.ID > filtered[j].Event.ID + } + return filtered[i].Event.ID < filtered[j].Event.ID + } + // Tertiary: delivery_time + if isDesc { + return filtered[i].Delivery.Time.After(filtered[j].Delivery.Time) + } + return filtered[i].Delivery.Time.Before(filtered[j].Delivery.Time) + } + + // Default: delivery_time + // Primary: delivery_time + if !filtered[i].Delivery.Time.Equal(filtered[j].Delivery.Time) { + if isDesc { + return filtered[i].Delivery.Time.After(filtered[j].Delivery.Time) + } + return filtered[i].Delivery.Time.Before(filtered[j].Delivery.Time) + } + // Secondary: delivery_id (for deterministic ordering) + if isDesc { + return filtered[i].Delivery.ID > filtered[j].Delivery.ID + } + return filtered[i].Delivery.ID < filtered[j].Delivery.ID + }) + + // Handle pagination + limit := req.Limit + if limit <= 0 { + limit = 100 + } + + // Find start index based on cursor + startIdx := 0 + if req.Next != "" { + // Next cursor contains the index to start from + for i, de := range filtered { + if de.ID == req.Next { + startIdx = i + break + } + } + } else if req.Prev != "" { + // Prev cursor: find the item and go back by limit + for i, de := range filtered { + if de.ID == req.Prev { + startIdx = i - limit + if startIdx < 0 { + startIdx = 0 + } + break + } + } + } + + // Slice the results + endIdx := startIdx + limit + if endIdx > len(filtered) { + endIdx = len(filtered) + } + + // Deep copy to ensure immutability + data := make([]*models.DeliveryEvent, endIdx-startIdx) + for i, de := range filtered[startIdx:endIdx] { + data[i] = copyDeliveryEvent(de) + } + + // Build cursors + var next, prev string + if endIdx < len(filtered) { + next = filtered[endIdx].ID + } + if startIdx > 0 { + prev = filtered[startIdx].ID + } + + return driver.ListDeliveryEventResponse{ + Data: data, + Next: next, + Prev: prev, + }, nil +} + +func (s *memLogStore) RetrieveEvent(ctx context.Context, req driver.RetrieveEventRequest) (*models.Event, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + for _, de := range s.deliveryEvents { + if de.Event.ID == req.EventID && de.Event.TenantID == req.TenantID { + if req.DestinationID != "" && de.Event.DestinationID != req.DestinationID { + continue + } + // Return a deep copy to ensure immutability + return copyEvent(&de.Event), nil + } + } + return nil, nil +} + +func (s *memLogStore) matchesFilter(de *models.DeliveryEvent, req driver.ListDeliveryEventRequest) bool { + // Tenant filter (required) + if de.Event.TenantID != req.TenantID { + return false + } + + // Event ID filter + if req.EventID != "" && de.Event.ID != req.EventID { + return false + } + + // Destination filter + if len(req.DestinationIDs) > 0 { + found := false + for _, destID := range req.DestinationIDs { + if de.DestinationID == destID { + found = true + break + } + } + if !found { + return false + } + } + + // Status filter + if req.Status != "" && de.Delivery.Status != req.Status { + return false + } + + // Topics filter + if len(req.Topics) > 0 { + found := false + for _, topic := range req.Topics { + if de.Event.Topic == topic { + found = true + break + } + } + if !found { + return false + } + } + + // Event time filter + if req.EventStart != nil && de.Event.Time.Before(*req.EventStart) { + return false + } + if req.EventEnd != nil && de.Event.Time.After(*req.EventEnd) { + return false + } + + // Delivery time filter + if req.DeliveryStart != nil && de.Delivery.Time.Before(*req.DeliveryStart) { + return false + } + if req.DeliveryEnd != nil && de.Delivery.Time.After(*req.DeliveryEnd) { + return false + } + + return true +} + +// Deep copy helpers to ensure data immutability + +func copyDeliveryEvent(de *models.DeliveryEvent) *models.DeliveryEvent { + return &models.DeliveryEvent{ + ID: de.ID, + DestinationID: de.DestinationID, + Event: *copyEvent(&de.Event), + Delivery: copyDelivery(de.Delivery), + } +} + +func copyEvent(e *models.Event) *models.Event { + copied := &models.Event{ + ID: e.ID, + TenantID: e.TenantID, + DestinationID: e.DestinationID, + Topic: e.Topic, + EligibleForRetry: e.EligibleForRetry, + Time: e.Time, + } + + // Deep copy maps + if e.Metadata != nil { + copied.Metadata = make(map[string]string, len(e.Metadata)) + for k, v := range e.Metadata { + copied.Metadata[k] = v + } + } + if e.Data != nil { + copied.Data = make(map[string]interface{}, len(e.Data)) + for k, v := range e.Data { + copied.Data[k] = v + } + } + + return copied +} + +func copyDelivery(d *models.Delivery) *models.Delivery { + if d == nil { + return nil + } + copied := &models.Delivery{ + ID: d.ID, + EventID: d.EventID, + DestinationID: d.DestinationID, + Status: d.Status, + Time: d.Time, + Code: d.Code, + } + + // Deep copy ResponseData map if present + if d.ResponseData != nil { + copied.ResponseData = make(map[string]interface{}, len(d.ResponseData)) + for k, v := range d.ResponseData { + copied.ResponseData[k] = v + } + } + + return copied +} diff --git a/internal/logstore/memlogstore/memlogstore_test.go b/internal/logstore/memlogstore/memlogstore_test.go new file mode 100644 index 00000000..c79394fb --- /dev/null +++ b/internal/logstore/memlogstore/memlogstore_test.go @@ -0,0 +1,31 @@ +package memlogstore + +import ( + "context" + "testing" + + "github.com/hookdeck/outpost/internal/logstore/driver" + "github.com/hookdeck/outpost/internal/logstore/drivertest" +) + +type memLogStoreHarness struct { + logStore driver.LogStore +} + +func (h *memLogStoreHarness) MakeDriver(ctx context.Context) (driver.LogStore, error) { + return h.logStore, nil +} + +func (h *memLogStoreHarness) Close() { + // No-op for in-memory store +} + +func newHarness(ctx context.Context, t *testing.T) (drivertest.Harness, error) { + return &memLogStoreHarness{ + logStore: NewLogStore(), + }, nil +} + +func TestMemLogStoreConformance(t *testing.T) { + drivertest.RunConformanceTests(t, newHarness) +} From a14d2fada06a4ae4dd17c7a30f78ad38915a29a8 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Fri, 5 Dec 2025 00:28:12 +0700 Subject: [PATCH 02/19] chore: fix noop logstore --- internal/logstore/noop.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/internal/logstore/noop.go b/internal/logstore/noop.go index 70f0dd5d..af084c62 100644 --- a/internal/logstore/noop.go +++ b/internal/logstore/noop.go @@ -3,6 +3,7 @@ package logstore import ( "context" + "github.com/hookdeck/outpost/internal/logstore/driver" "github.com/hookdeck/outpost/internal/models" ) @@ -12,19 +13,13 @@ func NewNoopLogStore() LogStore { type noopLogStore struct{} -func (l *noopLogStore) ListEvent(ctx context.Context, request ListEventRequest) (ListEventResponse, error) { - return ListEventResponse{}, nil -} +var _ LogStore = (*noopLogStore)(nil) -func (l *noopLogStore) RetrieveEvent(ctx context.Context, tenantID, eventID string) (*models.Event, error) { - return nil, nil -} - -func (l *noopLogStore) RetrieveEventByDestination(ctx context.Context, tenantID, destinationID, eventID string) (*models.Event, error) { - return nil, nil +func (l *noopLogStore) ListDeliveryEvent(ctx context.Context, request driver.ListDeliveryEventRequest) (driver.ListDeliveryEventResponse, error) { + return driver.ListDeliveryEventResponse{}, nil } -func (l *noopLogStore) ListDelivery(ctx context.Context, request ListDeliveryRequest) ([]*models.Delivery, error) { +func (l *noopLogStore) RetrieveEvent(ctx context.Context, request driver.RetrieveEventRequest) (*models.Event, error) { return nil, nil } From 4bc003be48b6d14357b9d3eb7c615d3dcc5c579e Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Fri, 5 Dec 2025 00:36:32 +0700 Subject: [PATCH 03/19] feat: cursor --- internal/logstore/cursor/cursor.go | 127 ++++++++ internal/logstore/driver/driver.go | 5 + internal/logstore/drivertest/drivertest.go | 306 ++++++++++++++++++- internal/logstore/memlogstore/memlogstore.go | 67 ++-- 4 files changed, 482 insertions(+), 23 deletions(-) create mode 100644 internal/logstore/cursor/cursor.go diff --git a/internal/logstore/cursor/cursor.go b/internal/logstore/cursor/cursor.go new file mode 100644 index 00000000..6c10bf01 --- /dev/null +++ b/internal/logstore/cursor/cursor.go @@ -0,0 +1,127 @@ +package cursor + +import ( + "fmt" + "math/big" + "strings" + + "github.com/hookdeck/outpost/internal/logstore/driver" +) + +// cursorVersion is the current cursor format version. +// Increment this when making breaking changes to cursor format. +const cursorVersion = "v1" + +// Cursor represents a pagination cursor with embedded sort parameters. +// This ensures cursors are only valid for queries with matching sort configuration. +type Cursor struct { + SortBy string // "event_time" or "delivery_time" + SortOrder string // "asc" or "desc" + Position string // implementation-specific position value +} + +// IsEmpty returns true if this cursor has no position (i.e., no cursor was provided). +func (c Cursor) IsEmpty() bool { + return c.Position == "" +} + +// Encode converts a Cursor to a URL-safe base62 string. +// Format: v1:{sortBy}:{sortOrder}:{position} -> base62 encoded +func Encode(c Cursor) string { + raw := fmt.Sprintf("%s:%s:%s:%s", cursorVersion, c.SortBy, c.SortOrder, c.Position) + num := new(big.Int) + num.SetBytes([]byte(raw)) + return num.Text(62) +} + +// Decode converts a base62 encoded cursor string back to a Cursor. +// Returns driver.ErrInvalidCursor if the cursor is malformed or has an unsupported version. +func Decode(encoded string) (Cursor, error) { + if encoded == "" { + return Cursor{}, nil + } + + num := new(big.Int) + num, ok := num.SetString(encoded, 62) + if !ok { + return Cursor{}, driver.ErrInvalidCursor + } + + raw := string(num.Bytes()) + parts := strings.SplitN(raw, ":", 4) + if len(parts) != 4 { + return Cursor{}, driver.ErrInvalidCursor + } + + version := parts[0] + sortBy := parts[1] + sortOrder := parts[2] + position := parts[3] + + // Validate version + if version != cursorVersion { + return Cursor{}, fmt.Errorf("%w: unsupported cursor version %q", driver.ErrInvalidCursor, version) + } + + // Validate sortBy + if sortBy != "event_time" && sortBy != "delivery_time" { + return Cursor{}, driver.ErrInvalidCursor + } + + // Validate sortOrder + if sortOrder != "asc" && sortOrder != "desc" { + return Cursor{}, driver.ErrInvalidCursor + } + + return Cursor{ + SortBy: sortBy, + SortOrder: sortOrder, + Position: position, + }, nil +} + +// Validate checks if the cursor matches the expected sort parameters. +// Returns driver.ErrInvalidCursor if there's a mismatch. +func Validate(c Cursor, expectedSortBy, expectedSortOrder string) error { + if c.IsEmpty() { + // Empty cursor is always valid + return nil + } + + if c.SortBy != expectedSortBy { + return fmt.Errorf("%w: cursor sortBy %q does not match request sortBy %q", + driver.ErrInvalidCursor, c.SortBy, expectedSortBy) + } + + if c.SortOrder != expectedSortOrder { + return fmt.Errorf("%w: cursor sortOrder %q does not match request sortOrder %q", + driver.ErrInvalidCursor, c.SortOrder, expectedSortOrder) + } + + return nil +} + +// DecodeAndValidate is a helper that decodes and validates both Next and Prev cursors. +// This is the common pattern used by all LogStore implementations. +// Returns the decoded cursors or an error if either cursor is invalid or mismatched. +func DecodeAndValidate(next, prev, sortBy, sortOrder string) (nextCursor, prevCursor Cursor, err error) { + if next != "" { + nextCursor, err = Decode(next) + if err != nil { + return Cursor{}, Cursor{}, err + } + if err := Validate(nextCursor, sortBy, sortOrder); err != nil { + return Cursor{}, Cursor{}, err + } + } + if prev != "" { + prevCursor, err = Decode(prev) + if err != nil { + return Cursor{}, Cursor{}, err + } + if err := Validate(prevCursor, sortBy, sortOrder); err != nil { + return Cursor{}, Cursor{}, err + } + } + return nextCursor, prevCursor, nil +} diff --git a/internal/logstore/driver/driver.go b/internal/logstore/driver/driver.go index 05b15247..1eebd9d4 100644 --- a/internal/logstore/driver/driver.go +++ b/internal/logstore/driver/driver.go @@ -2,11 +2,16 @@ package driver import ( "context" + "errors" "time" "github.com/hookdeck/outpost/internal/models" ) +// ErrInvalidCursor is returned when the cursor is malformed or doesn't match +// the current query parameters (e.g., different sort order). +var ErrInvalidCursor = errors.New("invalid cursor") + type LogStore interface { ListDeliveryEvent(context.Context, ListDeliveryEventRequest) (ListDeliveryEventResponse, error) RetrieveEvent(ctx context.Context, request RetrieveEventRequest) (*models.Event, error) diff --git a/internal/logstore/drivertest/drivertest.go b/internal/logstore/drivertest/drivertest.go index 4d06f745..f184de28 100644 --- a/internal/logstore/drivertest/drivertest.go +++ b/internal/logstore/drivertest/drivertest.go @@ -2,6 +2,7 @@ package drivertest import ( "context" + "errors" "fmt" "sort" "strconv" @@ -9,6 +10,7 @@ import ( "time" "github.com/google/uuid" + "github.com/hookdeck/outpost/internal/logstore/cursor" "github.com/hookdeck/outpost/internal/logstore/driver" "github.com/hookdeck/outpost/internal/models" "github.com/hookdeck/outpost/internal/util/testutil" @@ -47,6 +49,9 @@ func RunConformanceTests(t *testing.T, newHarness HarnessMaker) { t.Run("TestEdgeCases", func(t *testing.T) { testEdgeCases(t, newHarness) }) + t.Run("TestCursorValidation", func(t *testing.T) { + testCursorValidation(t, newHarness) + }) } // testInsertManyDeliveryEvent tests the InsertManyDeliveryEvent method @@ -1090,10 +1095,17 @@ func compareByEventTime(a, b *models.DeliveryEvent, desc bool) bool { return a.Event.ID < b.Event.ID } // Tertiary: delivery_time + if !a.Delivery.Time.Equal(b.Delivery.Time) { + if desc { + return a.Delivery.Time.After(b.Delivery.Time) + } + return a.Delivery.Time.Before(b.Delivery.Time) + } + // Quaternary: delivery_id (for deterministic ordering when all above are equal) if desc { - return a.Delivery.Time.After(b.Delivery.Time) + return a.Delivery.ID > b.Delivery.ID } - return a.Delivery.Time.Before(b.Delivery.Time) + return a.Delivery.ID < b.Delivery.ID } func sortDeliveryEvents(events []*models.DeliveryEvent, sortBy string, desc bool) []*models.DeliveryEvent { @@ -2284,3 +2296,293 @@ func extractDeliveryIDs(des []*models.DeliveryEvent) []string { } return ids } + +// ============================================================================= +// Cursor Validation Tests +// ============================================================================= +// These tests verify that cursors encode sort parameters and that using a cursor +// with different sort parameters returns an error. This prevents confusing +// behavior when changing query params mid-pagination. +// ============================================================================= + +func testCursorValidation(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + t.Run("cursor with mismatched sortBy returns error", func(t *testing.T) { + testCursorMismatchedSortBy(t, newHarness) + }) + t.Run("cursor with mismatched sortOrder returns error", func(t *testing.T) { + testCursorMismatchedSortOrder(t, newHarness) + }) + t.Run("malformed cursor returns error", func(t *testing.T) { + testMalformedCursor(t, newHarness) + }) + t.Run("cursor works with matching sort params", func(t *testing.T) { + testCursorMatchingSortParams(t, newHarness) + }) +} + +// testCursorMismatchedSortBy verifies that using a cursor generated with one +// sortBy value with a different sortBy value returns ErrInvalidCursor. +func testCursorMismatchedSortBy(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + tenantID := uuid.New().String() + destinationID := uuid.New().String() + baseTime := time.Now().Truncate(time.Second) + startTime := baseTime.Add(-48 * time.Hour) + + // Insert enough data to get a next cursor + var deliveryEvents []*models.DeliveryEvent + for i := 0; i < 5; i++ { + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(fmt.Sprintf("evt_cursor_%d", i)), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(fmt.Sprintf("del_cursor_%d", i)), + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), + ) + deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ + ID: fmt.Sprintf("de_cursor_%d", i), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + }) + } + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, deliveryEvents)) + + // Get a cursor with sortBy=delivery_time + response1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortBy: "delivery_time", + SortOrder: "desc", + EventStart: &startTime, + Limit: 2, + }) + require.NoError(t, err) + require.NotEmpty(t, response1.Next, "expected next cursor") + + // Try to use the cursor with sortBy=event_time - should fail + _, err = logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortBy: "event_time", // Different from cursor + SortOrder: "desc", + EventStart: &startTime, + Next: response1.Next, + Limit: 2, + }) + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor), + "expected ErrInvalidCursor, got: %v", err) +} + +// testCursorMismatchedSortOrder verifies that using a cursor generated with one +// sortOrder value with a different sortOrder value returns ErrInvalidCursor. +func testCursorMismatchedSortOrder(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + tenantID := uuid.New().String() + destinationID := uuid.New().String() + baseTime := time.Now().Truncate(time.Second) + startTime := baseTime.Add(-48 * time.Hour) + + // Insert enough data to get a next cursor + var deliveryEvents []*models.DeliveryEvent + for i := 0; i < 5; i++ { + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(fmt.Sprintf("evt_order_%d", i)), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(fmt.Sprintf("del_order_%d", i)), + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), + ) + deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ + ID: fmt.Sprintf("de_order_%d", i), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + }) + } + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, deliveryEvents)) + + // Get a cursor with sortOrder=desc + response1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortBy: "delivery_time", + SortOrder: "desc", + EventStart: &startTime, + Limit: 2, + }) + require.NoError(t, err) + require.NotEmpty(t, response1.Next, "expected next cursor") + + // Try to use the cursor with sortOrder=asc - should fail + _, err = logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortBy: "delivery_time", + SortOrder: "asc", // Different from cursor + EventStart: &startTime, + Next: response1.Next, + Limit: 2, + }) + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor), + "expected ErrInvalidCursor, got: %v", err) +} + +// testMalformedCursor verifies that a malformed cursor string returns ErrInvalidCursor. +func testMalformedCursor(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + tenantID := uuid.New().String() + startTime := time.Now().Add(-1 * time.Hour) + + testCases := []struct { + name string + cursor string + }{ + {"completely invalid base62", "!!!invalid!!!"}, + {"random string", "abcdef123456"}, + {"empty after decode", cursor.Encode(cursor.Cursor{})}, // Empty cursor should be fine, but let's test edge cases + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortBy: "delivery_time", + SortOrder: "desc", + EventStart: &startTime, + Next: tc.cursor, + Limit: 10, + }) + // Some of these might not error (e.g., if cursor decodes to valid format) + // but completely invalid base62 should error + if tc.name == "completely invalid base62" { + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor), + "expected ErrInvalidCursor for %s, got: %v", tc.name, err) + } + }) + } +} + +// testCursorMatchingSortParams verifies that cursors work correctly when +// sort parameters match between cursor generation and usage. +func testCursorMatchingSortParams(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + tenantID := uuid.New().String() + destinationID := uuid.New().String() + baseTime := time.Now().Truncate(time.Second) + startTime := baseTime.Add(-48 * time.Hour) + + // Insert data + var deliveryEvents []*models.DeliveryEvent + for i := 0; i < 5; i++ { + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(fmt.Sprintf("evt_match_%d", i)), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(fmt.Sprintf("del_match_%d", i)), + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithTime(baseTime.Add(-time.Duration(i)*time.Hour)), + ) + deliveryEvents = append(deliveryEvents, &models.DeliveryEvent{ + ID: fmt.Sprintf("de_match_%d", i), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + }) + } + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, deliveryEvents)) + + sortConfigs := []struct { + sortBy string + sortOrder string + }{ + {"delivery_time", "desc"}, + {"delivery_time", "asc"}, + {"event_time", "desc"}, + {"event_time", "asc"}, + } + + for _, sc := range sortConfigs { + t.Run(fmt.Sprintf("%s_%s", sc.sortBy, sc.sortOrder), func(t *testing.T) { + // Get first page with cursor + response1, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortBy: sc.sortBy, + SortOrder: sc.sortOrder, + EventStart: &startTime, + Limit: 2, + }) + require.NoError(t, err) + require.NotEmpty(t, response1.Next, "expected next cursor for %s %s", sc.sortBy, sc.sortOrder) + + // Use cursor with same sort params - should succeed + response2, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + SortBy: sc.sortBy, + SortOrder: sc.sortOrder, + EventStart: &startTime, + Next: response1.Next, + Limit: 2, + }) + require.NoError(t, err, "cursor should work with matching sort params for %s %s", sc.sortBy, sc.sortOrder) + require.NotEmpty(t, response2.Data, "should return data for second page") + + // Verify we got different data (not the same page) + if len(response1.Data) > 0 && len(response2.Data) > 0 { + assert.NotEqual(t, response1.Data[0].ID, response2.Data[0].ID, + "second page should have different data") + } + }) + } +} diff --git a/internal/logstore/memlogstore/memlogstore.go b/internal/logstore/memlogstore/memlogstore.go index 57a3352c..e84b8edd 100644 --- a/internal/logstore/memlogstore/memlogstore.go +++ b/internal/logstore/memlogstore/memlogstore.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/hookdeck/outpost/internal/logstore/cursor" "github.com/hookdeck/outpost/internal/logstore/driver" "github.com/hookdeck/outpost/internal/models" ) @@ -46,6 +47,22 @@ func (s *memLogStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeli s.mu.RLock() defer s.mu.RUnlock() + // Validate and set defaults for sort parameters + sortBy := req.SortBy + if sortBy != "event_time" && sortBy != "delivery_time" { + sortBy = "delivery_time" + } + sortOrder := req.SortOrder + if sortOrder != "asc" && sortOrder != "desc" { + sortOrder = "desc" + } + + // Decode and validate cursors + nextCursor, prevCursor, err := cursor.DecodeAndValidate(req.Next, req.Prev, sortBy, sortOrder) + if err != nil { + return driver.ListDeliveryEventResponse{}, err + } + // Apply default time range if not specified now := time.Now() if req.EventStart == nil && req.EventEnd == nil && req.DeliveryStart == nil && req.DeliveryEnd == nil { @@ -72,14 +89,6 @@ func (s *memLogStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeli // // The secondary/tertiary columns ensure deterministic ordering when // primary sort values are identical (e.g., multiple deliveries for same event). - sortBy := req.SortBy - if sortBy != "event_time" && sortBy != "delivery_time" { - sortBy = "delivery_time" - } - sortOrder := req.SortOrder - if sortOrder != "asc" && sortOrder != "desc" { - sortOrder = "desc" - } isDesc := sortOrder == "desc" sort.Slice(filtered, func(i, j int) bool { @@ -99,10 +108,17 @@ func (s *memLogStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeli return filtered[i].Event.ID < filtered[j].Event.ID } // Tertiary: delivery_time + if !filtered[i].Delivery.Time.Equal(filtered[j].Delivery.Time) { + if isDesc { + return filtered[i].Delivery.Time.After(filtered[j].Delivery.Time) + } + return filtered[i].Delivery.Time.Before(filtered[j].Delivery.Time) + } + // Quaternary: delivery_id (for deterministic ordering when all above are equal) if isDesc { - return filtered[i].Delivery.Time.After(filtered[j].Delivery.Time) + return filtered[i].Delivery.ID > filtered[j].Delivery.ID } - return filtered[i].Delivery.Time.Before(filtered[j].Delivery.Time) + return filtered[i].Delivery.ID < filtered[j].Delivery.ID } // Default: delivery_time @@ -127,19 +143,20 @@ func (s *memLogStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeli } // Find start index based on cursor + // The cursor position is the DeliveryEvent.ID startIdx := 0 - if req.Next != "" { - // Next cursor contains the index to start from + if !nextCursor.IsEmpty() { + // Next cursor: find the item and start from there for i, de := range filtered { - if de.ID == req.Next { + if de.ID == nextCursor.Position { startIdx = i break } } - } else if req.Prev != "" { + } else if !prevCursor.IsEmpty() { // Prev cursor: find the item and go back by limit for i, de := range filtered { - if de.ID == req.Prev { + if de.ID == prevCursor.Position { startIdx = i - limit if startIdx < 0 { startIdx = 0 @@ -161,19 +178,27 @@ func (s *memLogStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeli data[i] = copyDeliveryEvent(de) } - // Build cursors - var next, prev string + // Build cursors with sort parameters encoded + var nextEncoded, prevEncoded string if endIdx < len(filtered) { - next = filtered[endIdx].ID + nextEncoded = cursor.Encode(cursor.Cursor{ + SortBy: sortBy, + SortOrder: sortOrder, + Position: filtered[endIdx].ID, + }) } if startIdx > 0 { - prev = filtered[startIdx].ID + prevEncoded = cursor.Encode(cursor.Cursor{ + SortBy: sortBy, + SortOrder: sortOrder, + Position: filtered[startIdx].ID, + }) } return driver.ListDeliveryEventResponse{ Data: data, - Next: next, - Prev: prev, + Next: nextEncoded, + Prev: prevEncoded, }, nil } From ebcca8abd03ae42991162eb8ce704ef4205b2749 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Fri, 5 Dec 2025 00:38:52 +0700 Subject: [PATCH 04/19] test: cursor --- internal/logstore/cursor/cursor_test.go | 248 ++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 internal/logstore/cursor/cursor_test.go diff --git a/internal/logstore/cursor/cursor_test.go b/internal/logstore/cursor/cursor_test.go new file mode 100644 index 00000000..df7300b9 --- /dev/null +++ b/internal/logstore/cursor/cursor_test.go @@ -0,0 +1,248 @@ +package cursor + +import ( + "errors" + "math/big" + "testing" + + "github.com/hookdeck/outpost/internal/logstore/driver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCursor_IsEmpty(t *testing.T) { + t.Run("empty cursor", func(t *testing.T) { + c := Cursor{} + assert.True(t, c.IsEmpty()) + }) + + t.Run("cursor with position", func(t *testing.T) { + c := Cursor{Position: "abc123"} + assert.False(t, c.IsEmpty()) + }) + + t.Run("cursor with only sort params", func(t *testing.T) { + c := Cursor{SortBy: "event_time", SortOrder: "desc"} + assert.True(t, c.IsEmpty(), "cursor without position is empty") + }) +} + +func TestEncode(t *testing.T) { + t.Run("encodes cursor to base62", func(t *testing.T) { + c := Cursor{ + SortBy: "delivery_time", + SortOrder: "desc", + Position: "1234567890_del_abc", + } + encoded := Encode(c) + assert.NotEmpty(t, encoded) + assert.NotContains(t, encoded, ":", "encoded cursor should not contain raw separators") + }) + + t.Run("different cursors produce different encodings", func(t *testing.T) { + c1 := Cursor{SortBy: "delivery_time", SortOrder: "desc", Position: "pos1"} + c2 := Cursor{SortBy: "delivery_time", SortOrder: "desc", Position: "pos2"} + assert.NotEqual(t, Encode(c1), Encode(c2)) + }) + + t.Run("same cursor produces same encoding", func(t *testing.T) { + c := Cursor{SortBy: "event_time", SortOrder: "asc", Position: "pos"} + assert.Equal(t, Encode(c), Encode(c)) + }) +} + +func TestDecode(t *testing.T) { + t.Run("empty string returns empty cursor", func(t *testing.T) { + c, err := Decode("") + require.NoError(t, err) + assert.True(t, c.IsEmpty()) + }) + + t.Run("decodes valid cursor", func(t *testing.T) { + original := Cursor{ + SortBy: "delivery_time", + SortOrder: "desc", + Position: "1234567890_del_abc", + } + encoded := Encode(original) + + decoded, err := Decode(encoded) + require.NoError(t, err) + assert.Equal(t, original.SortBy, decoded.SortBy) + assert.Equal(t, original.SortOrder, decoded.SortOrder) + assert.Equal(t, original.Position, decoded.Position) + }) + + t.Run("decodes cursor with colons in position", func(t *testing.T) { + original := Cursor{ + SortBy: "event_time", + SortOrder: "asc", + Position: "time:with:colons:in:it", + } + encoded := Encode(original) + + decoded, err := Decode(encoded) + require.NoError(t, err) + assert.Equal(t, original.Position, decoded.Position) + }) + + t.Run("invalid base62 returns error", func(t *testing.T) { + _, err := Decode("!!!invalid!!!") + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + }) + + t.Run("invalid format returns error", func(t *testing.T) { + // Encode something that's not in the right format + _, err := Decode("abc123") + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + }) + + t.Run("invalid sortBy returns error", func(t *testing.T) { + // Manually create a cursor with invalid sortBy by encoding raw bytes + raw := "v1:invalid_sort:desc:position" + encoded := encodeRaw(raw) + + _, err := Decode(encoded) + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + }) + + t.Run("invalid sortOrder returns error", func(t *testing.T) { + raw := "v1:event_time:invalid_order:position" + encoded := encodeRaw(raw) + + _, err := Decode(encoded) + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + }) + + t.Run("unsupported version returns error", func(t *testing.T) { + raw := "v99:event_time:desc:position" + encoded := encodeRaw(raw) + + _, err := Decode(encoded) + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + assert.Contains(t, err.Error(), "unsupported cursor version") + }) +} + +func TestValidate(t *testing.T) { + t.Run("empty cursor is always valid", func(t *testing.T) { + c := Cursor{} + err := Validate(c, "event_time", "desc") + assert.NoError(t, err) + }) + + t.Run("matching params is valid", func(t *testing.T) { + c := Cursor{SortBy: "event_time", SortOrder: "desc", Position: "pos"} + err := Validate(c, "event_time", "desc") + assert.NoError(t, err) + }) + + t.Run("mismatched sortBy returns error", func(t *testing.T) { + c := Cursor{SortBy: "event_time", SortOrder: "desc", Position: "pos"} + err := Validate(c, "delivery_time", "desc") + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + assert.Contains(t, err.Error(), "sortBy") + }) + + t.Run("mismatched sortOrder returns error", func(t *testing.T) { + c := Cursor{SortBy: "event_time", SortOrder: "desc", Position: "pos"} + err := Validate(c, "event_time", "asc") + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + assert.Contains(t, err.Error(), "sortOrder") + }) +} + +func TestDecodeAndValidate(t *testing.T) { + t.Run("empty cursors return empty results", func(t *testing.T) { + next, prev, err := DecodeAndValidate("", "", "delivery_time", "desc") + require.NoError(t, err) + assert.True(t, next.IsEmpty()) + assert.True(t, prev.IsEmpty()) + }) + + t.Run("valid next cursor", func(t *testing.T) { + original := Cursor{SortBy: "delivery_time", SortOrder: "desc", Position: "pos"} + encoded := Encode(original) + + next, prev, err := DecodeAndValidate(encoded, "", "delivery_time", "desc") + require.NoError(t, err) + assert.Equal(t, "pos", next.Position) + assert.True(t, prev.IsEmpty()) + }) + + t.Run("valid prev cursor", func(t *testing.T) { + original := Cursor{SortBy: "event_time", SortOrder: "asc", Position: "pos"} + encoded := Encode(original) + + next, prev, err := DecodeAndValidate("", encoded, "event_time", "asc") + require.NoError(t, err) + assert.True(t, next.IsEmpty()) + assert.Equal(t, "pos", prev.Position) + }) + + t.Run("invalid next cursor returns error", func(t *testing.T) { + _, _, err := DecodeAndValidate("!!!invalid!!!", "", "delivery_time", "desc") + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + }) + + t.Run("invalid prev cursor returns error", func(t *testing.T) { + _, _, err := DecodeAndValidate("", "!!!invalid!!!", "delivery_time", "desc") + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + }) + + t.Run("mismatched next cursor returns error", func(t *testing.T) { + original := Cursor{SortBy: "delivery_time", SortOrder: "desc", Position: "pos"} + encoded := Encode(original) + + _, _, err := DecodeAndValidate(encoded, "", "event_time", "desc") + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + }) + + t.Run("mismatched prev cursor returns error", func(t *testing.T) { + original := Cursor{SortBy: "delivery_time", SortOrder: "desc", Position: "pos"} + encoded := Encode(original) + + _, _, err := DecodeAndValidate("", encoded, "delivery_time", "asc") + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + }) +} + +func TestRoundTrip(t *testing.T) { + testCases := []Cursor{ + {SortBy: "delivery_time", SortOrder: "desc", Position: "simple"}, + {SortBy: "delivery_time", SortOrder: "asc", Position: "1234567890_del_abc123"}, + {SortBy: "event_time", SortOrder: "desc", Position: "1234567890_evt_abc_1234567891_del_xyz"}, + {SortBy: "event_time", SortOrder: "asc", Position: "with:colons:and_underscores"}, + {SortBy: "delivery_time", SortOrder: "desc", Position: "unicode-émoji-🎉"}, + } + + for _, tc := range testCases { + t.Run(tc.Position, func(t *testing.T) { + encoded := Encode(tc) + decoded, err := Decode(encoded) + require.NoError(t, err) + + assert.Equal(t, tc.SortBy, decoded.SortBy) + assert.Equal(t, tc.SortOrder, decoded.SortOrder) + assert.Equal(t, tc.Position, decoded.Position) + }) + } +} + +// encodeRaw is a helper to encode raw strings for testing invalid formats +func encodeRaw(raw string) string { + num := new(big.Int) + num.SetBytes([]byte(raw)) + return num.Text(62) +} From 9c96ac2d23e7e07eaea786527e8c433ce1e8628f Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Fri, 5 Dec 2025 01:11:53 +0700 Subject: [PATCH 05/19] chore: cursor support backward compat --- internal/logstore/cursor/cursor.go | 139 ++++++++++++++++-------- internal/logstore/cursor/cursor_test.go | 96 +++++++++++++--- 2 files changed, 177 insertions(+), 58 deletions(-) diff --git a/internal/logstore/cursor/cursor.go b/internal/logstore/cursor/cursor.go index 6c10bf01..5ed0e2f7 100644 --- a/internal/logstore/cursor/cursor.go +++ b/internal/logstore/cursor/cursor.go @@ -8,12 +8,9 @@ import ( "github.com/hookdeck/outpost/internal/logstore/driver" ) -// cursorVersion is the current cursor format version. -// Increment this when making breaking changes to cursor format. -const cursorVersion = "v1" - // Cursor represents a pagination cursor with embedded sort parameters. // This ensures cursors are only valid for queries with matching sort configuration. +// Implementations use this type directly - versioning is handled internally by Encode/Decode. type Cursor struct { SortBy string // "event_time" or "delivery_time" SortOrder string // "asc" or "desc" @@ -26,65 +23,37 @@ func (c Cursor) IsEmpty() bool { } // Encode converts a Cursor to a URL-safe base62 string. -// Format: v1:{sortBy}:{sortOrder}:{position} -> base62 encoded +// Always encodes using the current version format. func Encode(c Cursor) string { - raw := fmt.Sprintf("%s:%s:%s:%s", cursorVersion, c.SortBy, c.SortOrder, c.Position) - num := new(big.Int) - num.SetBytes([]byte(raw)) - return num.Text(62) + return encodeV1(c) } // Decode converts a base62 encoded cursor string back to a Cursor. -// Returns driver.ErrInvalidCursor if the cursor is malformed or has an unsupported version. +// Automatically detects and handles different cursor versions. +// Returns driver.ErrInvalidCursor if the cursor is malformed. func Decode(encoded string) (Cursor, error) { if encoded == "" { return Cursor{}, nil } - num := new(big.Int) - num, ok := num.SetString(encoded, 62) - if !ok { - return Cursor{}, driver.ErrInvalidCursor - } - - raw := string(num.Bytes()) - parts := strings.SplitN(raw, ":", 4) - if len(parts) != 4 { - return Cursor{}, driver.ErrInvalidCursor - } - - version := parts[0] - sortBy := parts[1] - sortOrder := parts[2] - position := parts[3] - - // Validate version - if version != cursorVersion { - return Cursor{}, fmt.Errorf("%w: unsupported cursor version %q", driver.ErrInvalidCursor, version) - } - - // Validate sortBy - if sortBy != "event_time" && sortBy != "delivery_time" { - return Cursor{}, driver.ErrInvalidCursor + raw, err := decodeBase62(encoded) + if err != nil { + return Cursor{}, err } - // Validate sortOrder - if sortOrder != "asc" && sortOrder != "desc" { - return Cursor{}, driver.ErrInvalidCursor + // Detect version and decode accordingly + if strings.HasPrefix(raw, v1Prefix) { + return decodeV1(raw) } - return Cursor{ - SortBy: sortBy, - SortOrder: sortOrder, - Position: position, - }, nil + // Fall back to v0 format (legacy) + return decodeV0(raw) } // Validate checks if the cursor matches the expected sort parameters. // Returns driver.ErrInvalidCursor if there's a mismatch. func Validate(c Cursor, expectedSortBy, expectedSortOrder string) error { if c.IsEmpty() { - // Empty cursor is always valid return nil } @@ -103,7 +72,6 @@ func Validate(c Cursor, expectedSortBy, expectedSortOrder string) error { // DecodeAndValidate is a helper that decodes and validates both Next and Prev cursors. // This is the common pattern used by all LogStore implementations. -// Returns the decoded cursors or an error if either cursor is invalid or mismatched. func DecodeAndValidate(next, prev, sortBy, sortOrder string) (nextCursor, prevCursor Cursor, err error) { if next != "" { nextCursor, err = Decode(next) @@ -125,3 +93,84 @@ func DecodeAndValidate(next, prev, sortBy, sortOrder string) (nextCursor, prevCu } return nextCursor, prevCursor, nil } + +// ============================================================================= +// Internal: Base62 encoding/decoding +// ============================================================================= + +func encodeBase62(raw string) string { + num := new(big.Int) + num.SetBytes([]byte(raw)) + return num.Text(62) +} + +func decodeBase62(encoded string) (string, error) { + num := new(big.Int) + num, ok := num.SetString(encoded, 62) + if !ok { + return "", driver.ErrInvalidCursor + } + return string(num.Bytes()), nil +} + +// ============================================================================= +// Internal: v1 cursor format +// Format: v1:{sortBy}:{sortOrder}:{position} +// ============================================================================= + +const v1Prefix = "v1:" + +func encodeV1(c Cursor) string { + raw := fmt.Sprintf("v1:%s:%s:%s", c.SortBy, c.SortOrder, c.Position) + return encodeBase62(raw) +} + +func decodeV1(raw string) (Cursor, error) { + parts := strings.SplitN(raw, ":", 4) + if len(parts) != 4 { + return Cursor{}, driver.ErrInvalidCursor + } + + sortBy := parts[1] + sortOrder := parts[2] + position := parts[3] + + if sortBy != "event_time" && sortBy != "delivery_time" { + return Cursor{}, driver.ErrInvalidCursor + } + + if sortOrder != "asc" && sortOrder != "desc" { + return Cursor{}, driver.ErrInvalidCursor + } + + if position == "" { + return Cursor{}, driver.ErrInvalidCursor + } + + return Cursor{ + SortBy: sortBy, + SortOrder: sortOrder, + Position: position, + }, nil +} + +// ============================================================================= +// Internal: v0 cursor format (legacy, backward compatibility) +// Format: {position} (no version prefix, no sort params) +// Defaults: sortBy=event_time, sortOrder=desc +// ============================================================================= + +const ( + v0DefaultSortBy = "event_time" + v0DefaultSortOrder = "desc" +) + +func decodeV0(raw string) (Cursor, error) { + // v0 cursors are just the position, no validation needed + // If position is invalid, the DB query will simply not find it + return Cursor{ + SortBy: v0DefaultSortBy, + SortOrder: v0DefaultSortOrder, + Position: raw, + }, nil +} diff --git a/internal/logstore/cursor/cursor_test.go b/internal/logstore/cursor/cursor_test.go index df7300b9..dc2eab4b 100644 --- a/internal/logstore/cursor/cursor_test.go +++ b/internal/logstore/cursor/cursor_test.go @@ -58,7 +58,7 @@ func TestDecode(t *testing.T) { assert.True(t, c.IsEmpty()) }) - t.Run("decodes valid cursor", func(t *testing.T) { + t.Run("decodes v1 cursor", func(t *testing.T) { original := Cursor{ SortBy: "delivery_time", SortOrder: "desc", @@ -92,16 +92,17 @@ func TestDecode(t *testing.T) { assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) }) - t.Run("invalid format returns error", func(t *testing.T) { - // Encode something that's not in the right format - _, err := Decode("abc123") + t.Run("v1 invalid sortBy returns error", func(t *testing.T) { + raw := "v1:invalid_sort:desc:position" + encoded := encodeRaw(raw) + + _, err := Decode(encoded) require.Error(t, err) assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) }) - t.Run("invalid sortBy returns error", func(t *testing.T) { - // Manually create a cursor with invalid sortBy by encoding raw bytes - raw := "v1:invalid_sort:desc:position" + t.Run("v1 invalid sortOrder returns error", func(t *testing.T) { + raw := "v1:event_time:invalid_order:position" encoded := encodeRaw(raw) _, err := Decode(encoded) @@ -109,8 +110,8 @@ func TestDecode(t *testing.T) { assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) }) - t.Run("invalid sortOrder returns error", func(t *testing.T) { - raw := "v1:event_time:invalid_order:position" + t.Run("v1 empty position returns error", func(t *testing.T) { + raw := "v1:event_time:desc:" encoded := encodeRaw(raw) _, err := Decode(encoded) @@ -118,14 +119,83 @@ func TestDecode(t *testing.T) { assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) }) - t.Run("unsupported version returns error", func(t *testing.T) { - raw := "v99:event_time:desc:position" + t.Run("v1 missing parts returns error", func(t *testing.T) { + raw := "v1:event_time:desc" // missing position encoded := encodeRaw(raw) _, err := Decode(encoded) require.Error(t, err) assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) - assert.Contains(t, err.Error(), "unsupported cursor version") + }) +} + +func TestDecodeV0BackwardCompatibility(t *testing.T) { + t.Run("decodes v0 cursor with defaults", func(t *testing.T) { + // v0 format: just position, no version prefix + position := "1704067200_evt_abc" + encoded := encodeRaw(position) + + decoded, err := Decode(encoded) + require.NoError(t, err) + assert.Equal(t, position, decoded.Position) + assert.Equal(t, "event_time", decoded.SortBy, "v0 defaults to event_time") + assert.Equal(t, "desc", decoded.SortOrder, "v0 defaults to desc") + }) + + t.Run("decodes v0 composite cursor", func(t *testing.T) { + // v0 composite cursor for event_time sort + position := "1704067200_evt_abc_1704067500_del_xyz" + encoded := encodeRaw(position) + + decoded, err := Decode(encoded) + require.NoError(t, err) + assert.Equal(t, position, decoded.Position) + assert.Equal(t, "event_time", decoded.SortBy) + assert.Equal(t, "desc", decoded.SortOrder) + }) + + t.Run("v0 cursor validates with matching defaults", func(t *testing.T) { + position := "1704067200_evt_abc" + encoded := encodeRaw(position) + + // Should work with default sort params + next, _, err := DecodeAndValidate(encoded, "", "event_time", "desc") + require.NoError(t, err) + assert.Equal(t, position, next.Position) + }) + + t.Run("v0 cursor fails validation with non-default sort params", func(t *testing.T) { + position := "1704067200_del_xyz" + encoded := encodeRaw(position) + + // Should fail because v0 defaults to event_time, not delivery_time + _, _, err := DecodeAndValidate(encoded, "", "delivery_time", "desc") + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + assert.Contains(t, err.Error(), "sortBy") + }) + + t.Run("v0 cursor fails validation with different sort order", func(t *testing.T) { + position := "1704067200_evt_abc" + encoded := encodeRaw(position) + + // Should fail because v0 defaults to desc, not asc + _, _, err := DecodeAndValidate(encoded, "", "event_time", "asc") + require.Error(t, err) + assert.True(t, errors.Is(err, driver.ErrInvalidCursor)) + assert.Contains(t, err.Error(), "sortOrder") + }) + + t.Run("random string treated as v0 position", func(t *testing.T) { + // Any valid base62 that doesn't start with "v1:" is treated as v0 + position := "some_random_position_string" + encoded := encodeRaw(position) + + decoded, err := Decode(encoded) + require.NoError(t, err) + assert.Equal(t, position, decoded.Position) + assert.Equal(t, "event_time", decoded.SortBy) + assert.Equal(t, "desc", decoded.SortOrder) }) } @@ -240,7 +310,7 @@ func TestRoundTrip(t *testing.T) { } } -// encodeRaw is a helper to encode raw strings for testing invalid formats +// encodeRaw is a helper to encode raw strings for testing func encodeRaw(raw string) string { num := new(big.Int) num.SetBytes([]byte(raw)) From 537d4c17b133c3c8ea3a84c7e51c653eb237c44c Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Fri, 5 Dec 2025 01:27:28 +0700 Subject: [PATCH 06/19] chore: remove default time window query --- internal/logstore/drivertest/drivertest.go | 30 ++++++-------------- internal/logstore/memlogstore/memlogstore.go | 9 ------ 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/internal/logstore/drivertest/drivertest.go b/internal/logstore/drivertest/drivertest.go index f184de28..b2af218f 100644 --- a/internal/logstore/drivertest/drivertest.go +++ b/internal/logstore/drivertest/drivertest.go @@ -440,28 +440,16 @@ func testListDeliveryEvent(t *testing.T, newHarness HarnessMaker) { }) t.Run("time range filtering", func(t *testing.T) { - t.Run("default time range (last hour)", func(t *testing.T) { - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - Limit: 100, - // No Start/End - defaults to last hour - }) - require.NoError(t, err) - require.Len(t, response.Data, len(timeDeliveryEvents["1h"])) - }) - - t.Run("explicit time window", func(t *testing.T) { - sevenHoursAgo := baseTime.Add(-7 * time.Hour) - fiveHoursAgo := baseTime.Add(-5 * time.Hour) - response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ - TenantID: tenantID, - EventStart: &sevenHoursAgo, - EventEnd: &fiveHoursAgo, - Limit: 100, - }) - require.NoError(t, err) - require.Len(t, response.Data, len(timeDeliveryEvents["6h"])) + sevenHoursAgo := baseTime.Add(-7 * time.Hour) + fiveHoursAgo := baseTime.Add(-5 * time.Hour) + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: tenantID, + EventStart: &sevenHoursAgo, + EventEnd: &fiveHoursAgo, + Limit: 100, }) + require.NoError(t, err) + require.Len(t, response.Data, len(timeDeliveryEvents["6h"])) }) t.Run("combined filters", func(t *testing.T) { diff --git a/internal/logstore/memlogstore/memlogstore.go b/internal/logstore/memlogstore/memlogstore.go index e84b8edd..c402f6f6 100644 --- a/internal/logstore/memlogstore/memlogstore.go +++ b/internal/logstore/memlogstore/memlogstore.go @@ -4,7 +4,6 @@ import ( "context" "sort" "sync" - "time" "github.com/hookdeck/outpost/internal/logstore/cursor" "github.com/hookdeck/outpost/internal/logstore/driver" @@ -63,14 +62,6 @@ func (s *memLogStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeli return driver.ListDeliveryEventResponse{}, err } - // Apply default time range if not specified - now := time.Now() - if req.EventStart == nil && req.EventEnd == nil && req.DeliveryStart == nil && req.DeliveryEnd == nil { - // Default to last hour based on event time - defaultStart := now.Add(-1 * time.Hour) - req.EventStart = &defaultStart - } - // Filter var filtered []*models.DeliveryEvent for _, de := range s.deliveryEvents { From 4aa2639f348eaeef3d29da0734d8f7b02fe0c1b7 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Fri, 5 Dec 2025 01:32:42 +0700 Subject: [PATCH 07/19] feat: pglogstore implementation --- internal/logstore/pglogstore/README.md | 72 +++ internal/logstore/pglogstore/cursor.go | 46 -- internal/logstore/pglogstore/pglogstore.go | 616 +++++++++++---------- internal/logstore/pglogstore/schema.sql | 25 - 4 files changed, 402 insertions(+), 357 deletions(-) create mode 100644 internal/logstore/pglogstore/README.md delete mode 100644 internal/logstore/pglogstore/cursor.go delete mode 100644 internal/logstore/pglogstore/schema.sql diff --git a/internal/logstore/pglogstore/README.md b/internal/logstore/pglogstore/README.md new file mode 100644 index 00000000..7d76df6d --- /dev/null +++ b/internal/logstore/pglogstore/README.md @@ -0,0 +1,72 @@ +# pglogstore + +PostgreSQL implementation of the LogStore interface. + +## Schema + +| Table | Purpose | Primary Key | +|-------|---------|-------------| +| `events` | Event data (data, metadata) | `(time, id)` | +| `deliveries` | Delivery attempts (status, response) | `(time, id)` | +| `event_delivery_index` | Query index with cursor columns | `(delivery_time, event_id, delivery_id)` | + +All tables are partitioned by time. + +## Operations + +### ListDeliveryEvent + +Query pattern: **Index → Hydrate** + +1. CTE filters and paginates on `event_delivery_index` +2. JOIN `events` and `deliveries` by primary key to populate full data + +```sql +WITH filtered AS ( + SELECT ... FROM event_delivery_index WHERE [filters] ORDER BY ... LIMIT N +) +SELECT ... FROM filtered f +JOIN events e ON (e.time, e.id) = (f.event_time, f.event_id) +JOIN deliveries d ON (d.time, d.id) = (f.delivery_time, f.delivery_id) +``` + +**Key considerations:** +- Cursor encodes sort params - rejects mismatched sort order + +### RetrieveEvent + +Direct lookup by `(tenant_id, event_id)`. + +```sql +SELECT id, tenant_id, destination_id, time, topic, eligible_for_retry, data, metadata +FROM events +WHERE tenant_id = $1 AND id = $2 + +-- With destination filter: +SELECT id, tenant_id, $3 as destination_id, time, topic, eligible_for_retry, data, metadata +FROM events +WHERE tenant_id = $1 AND id = $2 +AND EXISTS (SELECT 1 FROM event_delivery_index WHERE event_id = $2 AND destination_id = $3) +``` + +### InsertManyDeliveryEvent + +Batch insert using `unnest()` arrays in a single transaction across all 3 tables. + +```sql +BEGIN; + +INSERT INTO events (id, tenant_id, destination_id, time, topic, eligible_for_retry, data, metadata) +SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::timestamptz[], $5::text[], $6::boolean[], $7::jsonb[], $8::jsonb[]) +ON CONFLICT (time, id) DO NOTHING; + +INSERT INTO deliveries (id, event_id, destination_id, status, time, code, response_data) +SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::timestamptz[], $6::text[], $7::jsonb[]) +ON CONFLICT (time, id) DO UPDATE SET status = EXCLUDED.status, code = EXCLUDED.code, response_data = EXCLUDED.response_data; + +INSERT INTO event_delivery_index (event_id, delivery_id, tenant_id, destination_id, event_time, delivery_time, topic, status) +SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::timestamptz[], $6::timestamptz[], $7::text[], $8::text[]) +ON CONFLICT (delivery_time, event_id, delivery_id) DO UPDATE SET status = EXCLUDED.status; + +COMMIT; +``` diff --git a/internal/logstore/pglogstore/cursor.go b/internal/logstore/pglogstore/cursor.go deleted file mode 100644 index 0ea6002c..00000000 --- a/internal/logstore/pglogstore/cursor.go +++ /dev/null @@ -1,46 +0,0 @@ -package pglogstore - -import ( - "fmt" - "math/big" -) - -type cursorEncoder interface { - Encode(raw string) string - Decode(encoded string) (string, error) -} - -type base62CursorEncoder struct{} - -func (e *base62CursorEncoder) Encode(raw string) string { - num := new(big.Int) - num.SetBytes([]byte(raw)) - return num.Text(62) -} - -func (e *base62CursorEncoder) Decode(encoded string) (string, error) { - num := new(big.Int) - num, ok := num.SetString(encoded, 62) - if !ok { - return "", fmt.Errorf("invalid cursor encoding") - } - return string(num.Bytes()), nil -} - -type eventCursorParser struct { - encoder cursorEncoder -} - -func newEventCursorParser() eventCursorParser { - return eventCursorParser{ - encoder: &base62CursorEncoder{}, - } -} - -func (p *eventCursorParser) Parse(cursor string) (string, error) { - return p.encoder.Decode(cursor) -} - -func (p *eventCursorParser) Format(timeID string) string { - return p.encoder.Encode(timeID) -} diff --git a/internal/logstore/pglogstore/pglogstore.go b/internal/logstore/pglogstore/pglogstore.go index c6e8628c..fe11f2db 100644 --- a/internal/logstore/pglogstore/pglogstore.go +++ b/internal/logstore/pglogstore/pglogstore.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/hookdeck/outpost/internal/logstore/cursor" "github.com/hookdeck/outpost/internal/logstore/driver" "github.com/hookdeck/outpost/internal/models" "github.com/jackc/pgx/v5" @@ -12,311 +13,383 @@ import ( ) type logStore struct { - db *pgxpool.Pool - cursorParser eventCursorParser + db *pgxpool.Pool } func NewLogStore(db *pgxpool.Pool) driver.LogStore { return &logStore{ - db: db, - cursorParser: newEventCursorParser(), + db: db, } } -func (s *logStore) ListEvent(ctx context.Context, req driver.ListEventRequest) (driver.ListEventResponse, error) { - // TODO: validate only one of next or prev is set - - var decodedNext, decodedPrev string - if req.Next != "" { - var err error - decodedNext, err = s.cursorParser.Parse(req.Next) - if err != nil { - return driver.ListEventResponse{}, fmt.Errorf("invalid cursor: %v", err) - } +// ListDeliveryEvent returns delivery events matching the filter criteria. +// It joins event_delivery_index with events and deliveries tables to return +// complete DeliveryEvent records. +// +// Sorting uses multi-column ordering for deterministic pagination: +// - delivery_time: ORDER BY delivery_time, delivery_id +// - event_time: ORDER BY event_time, event_id, delivery_time +func (s *logStore) ListDeliveryEvent(ctx context.Context, req driver.ListDeliveryEventRequest) (driver.ListDeliveryEventResponse, error) { + // Validate and set defaults + sortBy := req.SortBy + if sortBy != "event_time" && sortBy != "delivery_time" { + sortBy = "delivery_time" } - if req.Prev != "" { - var err error - decodedPrev, err = s.cursorParser.Parse(req.Prev) - if err != nil { - return driver.ListEventResponse{}, fmt.Errorf("invalid cursor: %v", err) - } + sortOrder := req.SortOrder + if sortOrder != "asc" && sortOrder != "desc" { + sortOrder = "desc" } - // Step 1: Query the index to get relevant event IDs and their status - indexQuery := ` - -- Step 1: Apply some filters & dedup index table to get event with status - WITH latest_status AS ( - SELECT DISTINCT ON (event_id, destination_id) - event_id, - destination_id, - delivery_time, - event_time, - time_event_id, - status - FROM event_delivery_index - WHERE tenant_id = $6 -- tenant_id - AND event_time >= COALESCE($4, COALESCE($5, NOW()) - INTERVAL '1 hour') - AND event_time <= COALESCE($5, NOW()) - AND (array_length($7::text[], 1) IS NULL OR destination_id = ANY($7)) -- destination_ids - AND (array_length($8::text[], 1) IS NULL OR topic = ANY($8)) -- topics - ORDER BY event_id, destination_id, delivery_time DESC - ), - -- Step 2: Apply status filter - filtered_before_cursor AS ( - SELECT * - FROM latest_status - WHERE ($9 = '' OR status = $9) -- status filter - ), - -- Step 3: Apply pagination (cursor & limit) - filtered AS ( - SELECT - event_id, - destination_id, - delivery_time, - event_time, - time_event_id, - status, - (SELECT COUNT(*) FROM filtered_before_cursor) as total_count - FROM filtered_before_cursor - WHERE ($1 = '' OR time_event_id < $1) AND ($2 = '' OR time_event_id > $2) -- cursor pagination - ORDER BY - CASE WHEN $2 != '' THEN time_event_id END ASC, -- prev cursor: sort ascending to get right window - time_event_id DESC -- default sort and next cursor sort - LIMIT CASE WHEN $3 = 0 THEN NULL ELSE $3 + 1 END - ), - -- Step 4: Re-sort for consistent response - final AS ( - SELECT * - FROM filtered - ORDER BY time_event_id DESC - ) - SELECT * FROM final` - - indexRows, err := s.db.Query(ctx, indexQuery, - decodedNext, - decodedPrev, - req.Limit, - req.Start, - req.End, - req.TenantID, - req.DestinationIDs, - req.Topics, - req.Status, - ) - if err != nil { - return driver.ListEventResponse{}, err - } - defer indexRows.Close() - - // Collect event IDs and their status - type eventInfo struct { - eventID string - destinationID string - deliveryTime time.Time - eventTime time.Time - timeEventID string - status string + limit := req.Limit + if limit <= 0 { + limit = 100 } - eventInfos := []eventInfo{} - var totalCount int64 - for indexRows.Next() { - var info eventInfo - err := indexRows.Scan(&info.eventID, &info.destinationID, &info.deliveryTime, &info.eventTime, &info.timeEventID, &info.status, &totalCount) - if err != nil { - return driver.ListEventResponse{}, err - } - eventInfos = append(eventInfos, info) + + // Decode and validate cursors + nextCursor, prevCursor, err := cursor.DecodeAndValidate(req.Next, req.Prev, sortBy, sortOrder) + if err != nil { + return driver.ListDeliveryEventResponse{}, err } - if len(eventInfos) == 0 { - return driver.ListEventResponse{ - Data: []*models.Event{}, - Next: "", - Prev: "", - Count: 0, - }, nil + // Build the cursor column expression + // For event_time sort, we need a cursor that captures the full sort key + // since multiple deliveries for the same event share time_event_id. + // We concatenate time_event_id with time_delivery_id to ensure uniqueness. + cursorCol := "time_delivery_id" + if sortBy == "event_time" { + // Composite cursor: time_event_id || '_' || time_delivery_id + // This ensures uniqueness while maintaining lexicographic sort order + cursorCol = "(time_event_id || '_' || time_delivery_id)" } - // Handle pagination - var hasNext, hasPrev bool - if req.Prev != "" { - hasNext = true // We came backwards, so definitely more ahead - hasPrev = len(eventInfos) > req.Limit || req.Limit == 0 // Check if more behind - if len(eventInfos) > req.Limit && req.Limit > 0 { - eventInfos = eventInfos[1:] // Trim first item (newest) when going backward - } - } else if req.Next != "" { - hasPrev = true // We came forwards, so definitely more behind - hasNext = len(eventInfos) > req.Limit || req.Limit == 0 // Check if more ahead - if len(eventInfos) > req.Limit && req.Limit > 0 { - eventInfos = eventInfos[:len(eventInfos)-1] // Trim last item when going forward + // Determine if we're going backward (using prev cursor) + goingBackward := !prevCursor.IsEmpty() + + // Build ORDER BY clause with tiebreakers for deterministic pagination + // When going backward, we flip the order in the query and reverse results after + // For event_time sort: ORDER BY event_time, event_id, delivery_time, delivery_id + // For delivery_time sort: ORDER BY delivery_time, delivery_id + var orderByClause, finalOrderByClause string + if sortBy == "event_time" { + if sortOrder == "desc" { + if goingBackward { + orderByClause = "event_time ASC, event_id ASC, delivery_time ASC, delivery_id ASC" + } else { + orderByClause = "event_time DESC, event_id DESC, delivery_time DESC, delivery_id DESC" + } + finalOrderByClause = "event_time DESC, event_id DESC, delivery_time DESC, delivery_id DESC" + } else { + if goingBackward { + orderByClause = "event_time DESC, event_id DESC, delivery_time DESC, delivery_id DESC" + } else { + orderByClause = "event_time ASC, event_id ASC, delivery_time ASC, delivery_id ASC" + } + finalOrderByClause = "event_time ASC, event_id ASC, delivery_time ASC, delivery_id ASC" } } else { - // First page - hasPrev = false - hasNext = len(eventInfos) > req.Limit || req.Limit == 0 - if len(eventInfos) > req.Limit && req.Limit > 0 { - eventInfos = eventInfos[:len(eventInfos)-1] // Trim last item on first page + if sortOrder == "desc" { + if goingBackward { + orderByClause = "delivery_time ASC, delivery_id ASC" + } else { + orderByClause = "delivery_time DESC, delivery_id DESC" + } + finalOrderByClause = "delivery_time DESC, delivery_id DESC" + } else { + if goingBackward { + orderByClause = "delivery_time DESC, delivery_id DESC" + } else { + orderByClause = "delivery_time ASC, delivery_id ASC" + } + finalOrderByClause = "delivery_time ASC, delivery_id ASC" } } - // Step 2: Get full event data - eventIDs := make([]string, len(eventInfos)) - for i, info := range eventInfos { - eventIDs[i] = info.eventID + // Build cursor conditions - always include both conditions but use empty string check + // This ensures PostgreSQL can infer types for both parameters + // For next cursor (going forward in display order): get items that come AFTER in sort order + // For prev cursor (going backward in display order): get items that come BEFORE in sort order + // Since we flip the query order for prev, we use the same comparison direction + var cursorCondition string + if sortOrder == "desc" { + // DESC: next means smaller values, prev means larger values (but we query with flipped order) + cursorCondition = fmt.Sprintf("AND ($10::text = '' OR %s < $10::text) AND ($11::text = '' OR %s > $11::text)", cursorCol, cursorCol) + } else { + // ASC: next means larger values, prev means smaller values (but we query with flipped order) + cursorCondition = fmt.Sprintf("AND ($10::text = '' OR %s > $10::text) AND ($11::text = '' OR %s < $11::text)", cursorCol, cursorCol) } - eventQuery := ` + query := fmt.Sprintf(` + WITH filtered AS ( + SELECT + idx.event_id, + idx.delivery_id, + idx.tenant_id, + idx.destination_id, + idx.event_time, + idx.delivery_time, + idx.topic, + idx.status, + idx.time_event_id, + idx.time_delivery_id + FROM event_delivery_index idx + WHERE idx.tenant_id = $1 + AND ($2::text = '' OR idx.event_id = $2) + AND (array_length($3::text[], 1) IS NULL OR idx.destination_id = ANY($3)) + AND ($4::text = '' OR idx.status = $4) + AND (array_length($5::text[], 1) IS NULL OR idx.topic = ANY($5)) + AND ($6::timestamptz IS NULL OR idx.event_time >= $6) + AND ($7::timestamptz IS NULL OR idx.event_time <= $7) + AND ($8::timestamptz IS NULL OR idx.delivery_time >= $8) + AND ($9::timestamptz IS NULL OR idx.delivery_time <= $9) + %s + ORDER BY %s + LIMIT $12 + ) SELECT - id, - tenant_id, - time, - topic, - eligible_for_retry, - data, - metadata - FROM events e - WHERE id = ANY($1)` - - eventRows, err := s.db.Query(ctx, eventQuery, eventIDs) + f.event_id, + f.delivery_id, + f.destination_id, + f.event_time, + f.delivery_time, + f.topic, + f.status, + f.time_event_id, + f.time_delivery_id, + e.tenant_id, + e.eligible_for_retry, + e.data, + e.metadata, + d.code, + d.response_data + FROM filtered f + JOIN events e ON e.id = f.event_id AND e.time = f.event_time + JOIN deliveries d ON d.id = f.delivery_id AND d.time = f.delivery_time + ORDER BY %s + `, cursorCondition, orderByClause, finalOrderByClause) + + rows, err := s.db.Query(ctx, query, + req.TenantID, // $1 + req.EventID, // $2 + req.DestinationIDs, // $3 + req.Status, // $4 + req.Topics, // $5 + req.EventStart, // $6 + req.EventEnd, // $7 + req.DeliveryStart, // $8 + req.DeliveryEnd, // $9 + nextCursor.Position, // $10 + prevCursor.Position, // $11 + limit+1, // $12 - fetch one extra to detect if there's more + ) if err != nil { - return driver.ListEventResponse{}, err + return driver.ListDeliveryEventResponse{}, fmt.Errorf("query failed: %w", err) } - defer eventRows.Close() - - // Build map of events - eventMap := make(map[string]*models.Event) - for eventRows.Next() { - event := &models.Event{} - err := eventRows.Scan( - &event.ID, - &event.TenantID, - &event.Time, - &event.Topic, - &event.EligibleForRetry, - &event.Data, - &event.Metadata, + defer rows.Close() + + type rowData struct { + de *models.DeliveryEvent + timeEventID string + timeDeliveryID string + } + var results []rowData + + for rows.Next() { + var ( + eventID string + deliveryID string + destinationID string + eventTime time.Time + deliveryTime time.Time + topic string + status string + timeEventID string + timeDeliveryID string + tenantID string + eligibleForRetry bool + data map[string]interface{} + metadata map[string]string + code string + responseData map[string]interface{} + ) + + err := rows.Scan( + &eventID, + &deliveryID, + &destinationID, + &eventTime, + &deliveryTime, + &topic, + &status, + &timeEventID, + &timeDeliveryID, + &tenantID, + &eligibleForRetry, + &data, + &metadata, + &code, + &responseData, ) if err != nil { - return driver.ListEventResponse{}, err + return driver.ListDeliveryEventResponse{}, fmt.Errorf("scan failed: %w", err) } - eventMap[event.ID] = event - } - // Combine events with their status in correct order - events := make([]*models.Event, 0, len(eventInfos)) - for _, info := range eventInfos { - if baseEvent, ok := eventMap[info.eventID]; ok { - // Create new event for each destination - event := *baseEvent // Make copy - event.DestinationID = info.destinationID - event.Status = info.status - events = append(events, &event) + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, deliveryID), + DestinationID: destinationID, + Event: models.Event{ + ID: eventID, + TenantID: tenantID, + DestinationID: destinationID, + Topic: topic, + EligibleForRetry: eligibleForRetry, + Time: eventTime, + Data: data, + Metadata: metadata, + }, + Delivery: &models.Delivery{ + ID: deliveryID, + EventID: eventID, + DestinationID: destinationID, + Status: status, + Time: deliveryTime, + Code: code, + ResponseData: responseData, + }, } + + results = append(results, rowData{de: de, timeEventID: timeEventID, timeDeliveryID: timeDeliveryID}) } - var nextCursor, prevCursor string - if len(events) > 0 { - lastItem := eventInfos[len(eventInfos)-1].timeEventID - firstItem := eventInfos[0].timeEventID + if err := rows.Err(); err != nil { + return driver.ListDeliveryEventResponse{}, fmt.Errorf("rows error: %w", err) + } - if hasNext { - nextCursor = s.cursorParser.Format(lastItem) - } - if hasPrev { - prevCursor = s.cursorParser.Format(firstItem) + // Handle pagination cursors + // When going backward, the extra item (if any) is at the BEGINNING after re-sort + // When going forward, the extra item is at the END + var hasMore bool + if len(results) > limit { + hasMore = true + if goingBackward { + // Trim from beginning - the extra item is now first after DESC re-sort + results = results[1:] + } else { + // Trim from end - the extra item is last + results = results[:limit] } } - return driver.ListEventResponse{ - Data: events, - Next: nextCursor, - Prev: prevCursor, - Count: totalCount, - }, nil -} + // Build response + data := make([]*models.DeliveryEvent, len(results)) + for i, r := range results { + data[i] = r.de + } -func (s *logStore) RetrieveEvent(ctx context.Context, tenantID, eventID string) (*models.Event, error) { - query := ` - SELECT - id, - tenant_id, - time, - topic, - eligible_for_retry, - data, - metadata, - CASE - WHEN EXISTS (SELECT 1 FROM deliveries d WHERE d.event_id = e.id AND d.status = 'success') THEN 'success' - WHEN EXISTS (SELECT 1 FROM deliveries d WHERE d.event_id = e.id) THEN 'failed' - ELSE 'pending' - END as status - FROM events e - WHERE tenant_id = $1 AND id = $2` - - row := s.db.QueryRow(ctx, query, tenantID, eventID) + var nextEncoded, prevEncoded string + if len(results) > 0 { + // Position value depends on sortBy + // Must match the cursorCol expression used in the query + getPosition := func(r rowData) string { + if sortBy == "event_time" { + // Composite cursor matching the SQL expression + return r.timeEventID + "_" + r.timeDeliveryID + } + return r.timeDeliveryID + } - event := &models.Event{} - err := row.Scan( - &event.ID, - &event.TenantID, - &event.Time, - &event.Topic, - &event.EligibleForRetry, - &event.Data, - &event.Metadata, - &event.Status, - ) - if err == pgx.ErrNoRows { - return nil, nil - } - if err != nil { - return nil, err + encodeCursor := func(position string) string { + return cursor.Encode(cursor.Cursor{ + SortBy: sortBy, + SortOrder: sortOrder, + Position: position, + }) + } + + if !prevCursor.IsEmpty() { + // Came from prev, so there's definitely more "next" + nextEncoded = encodeCursor(getPosition(results[len(results)-1])) + if hasMore { + prevEncoded = encodeCursor(getPosition(results[0])) + } + } else if !nextCursor.IsEmpty() { + // Came from next, so there's definitely more "prev" + prevEncoded = encodeCursor(getPosition(results[0])) + if hasMore { + nextEncoded = encodeCursor(getPosition(results[len(results)-1])) + } + } else { + // First page + if hasMore { + nextEncoded = encodeCursor(getPosition(results[len(results)-1])) + } + // No prev on first page + } } - return event, nil + return driver.ListDeliveryEventResponse{ + Data: data, + Next: nextEncoded, + Prev: prevEncoded, + }, nil } -func (s *logStore) RetrieveEventByDestination(ctx context.Context, tenantID, destinationID, eventID string) (*models.Event, error) { - query := ` - WITH latest_status AS ( - SELECT DISTINCT ON (event_id, destination_id) status - FROM event_delivery_index - WHERE tenant_id = $1 AND destination_id = $2 AND event_id = $3 - ORDER BY event_id, destination_id, delivery_time DESC - ) - SELECT - e.id, - e.tenant_id, - e.time, - e.topic, - e.eligible_for_retry, - e.data, - e.metadata, - $2 as destination_id, - COALESCE(s.status, 'pending') as status - FROM events e - LEFT JOIN latest_status s ON true - WHERE e.tenant_id = $1 AND e.id = $3` +// RetrieveEvent retrieves a single event by ID. +// If DestinationID is provided, it scopes the query to that destination. +func (s *logStore) RetrieveEvent(ctx context.Context, req driver.RetrieveEventRequest) (*models.Event, error) { + var query string + var args []interface{} + + if req.DestinationID != "" { + // Scope to specific destination - get status from index + query = ` + SELECT + e.id, + e.tenant_id, + $3 as destination_id, + e.time, + e.topic, + e.eligible_for_retry, + e.data, + e.metadata + FROM events e + WHERE e.tenant_id = $1 AND e.id = $2 + AND EXISTS ( + SELECT 1 FROM event_delivery_index idx + WHERE idx.tenant_id = $1 AND idx.event_id = $2 AND idx.destination_id = $3 + )` + args = []interface{}{req.TenantID, req.EventID, req.DestinationID} + } else { + query = ` + SELECT + e.id, + e.tenant_id, + e.destination_id, + e.time, + e.topic, + e.eligible_for_retry, + e.data, + e.metadata + FROM events e + WHERE e.tenant_id = $1 AND e.id = $2` + args = []interface{}{req.TenantID, req.EventID} + } - row := s.db.QueryRow(ctx, query, tenantID, destinationID, eventID) + row := s.db.QueryRow(ctx, query, args...) event := &models.Event{} err := row.Scan( &event.ID, &event.TenantID, + &event.DestinationID, &event.Time, &event.Topic, &event.EligibleForRetry, &event.Data, &event.Metadata, - &event.DestinationID, - &event.Status, ) if err == pgx.ErrNoRows { return nil, nil } - if err != nil { return nil, err } @@ -324,44 +397,11 @@ func (s *logStore) RetrieveEventByDestination(ctx context.Context, tenantID, des return event, nil } -func (s *logStore) ListDelivery(ctx context.Context, req driver.ListDeliveryRequest) ([]*models.Delivery, error) { - query := ` - SELECT id, event_id, destination_id, status, time, code, response_data - FROM deliveries - WHERE event_id = $1 - AND ($2 = '' OR destination_id = $2) - ORDER BY time DESC` - - rows, err := s.db.Query(ctx, query, - req.EventID, - req.DestinationID) - if err != nil { - return nil, err - } - defer rows.Close() - - var deliveries []*models.Delivery - for rows.Next() { - delivery := &models.Delivery{} - err := rows.Scan( - &delivery.ID, - &delivery.EventID, - &delivery.DestinationID, - &delivery.Status, - &delivery.Time, - &delivery.Code, - &delivery.ResponseData, - ) - if err != nil { - return nil, err - } - deliveries = append(deliveries, delivery) +func (s *logStore) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents []*models.DeliveryEvent) error { + if len(deliveryEvents) == 0 { + return nil } - return deliveries, nil -} - -func (s *logStore) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents []*models.DeliveryEvent) error { tx, err := s.db.Begin(ctx) if err != nil { return err @@ -413,7 +453,7 @@ func (s *logStore) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents [ // Insert into index _, err = tx.Exec(ctx, ` INSERT INTO event_delivery_index ( - event_id, delivery_id, tenant_id, destination_id, + event_id, delivery_id, tenant_id, destination_id, event_time, delivery_time, topic, status ) SELECT * FROM unnest( @@ -442,7 +482,11 @@ func eventDeliveryIndexArrays(deliveryEvents []*models.DeliveryEvent) []interfac for i, de := range deliveryEvents { eventIDs[i] = de.Event.ID - deliveryIDs[i] = de.ID + if de.Delivery != nil { + deliveryIDs[i] = de.Delivery.ID + } else { + deliveryIDs[i] = de.ID + } tenantIDs[i] = de.Event.TenantID destinationIDs[i] = de.DestinationID eventTimes[i] = de.Event.Time diff --git a/internal/logstore/pglogstore/schema.sql b/internal/logstore/pglogstore/schema.sql deleted file mode 100644 index 16d4b8bf..00000000 --- a/internal/logstore/pglogstore/schema.sql +++ /dev/null @@ -1,25 +0,0 @@ -CREATE TABLE IF NOT EXISTS events ( - id TEXT NOT NULL, - tenant_id TEXT NOT NULL, - destination_id TEXT NOT NULL, - topic TEXT NOT NULL, - eligible_for_retry BOOLEAN NOT NULL, - time TIMESTAMPTZ NOT NULL, - metadata JSONB NOT NULL, - data JSONB NOT NULL, - PRIMARY KEY (id) -); - -CREATE INDEX IF NOT EXISTS events_tenant_time_idx ON events (tenant_id, time DESC); - -CREATE TABLE IF NOT EXISTS deliveries ( - id TEXT NOT NULL, - event_id TEXT NOT NULL, - destination_id TEXT NOT NULL, - status TEXT NOT NULL, - time TIMESTAMPTZ NOT NULL, - PRIMARY KEY (id), - FOREIGN KEY (event_id) REFERENCES events (id) -); - -CREATE INDEX IF NOT EXISTS deliveries_event_time_idx ON deliveries (event_id, time DESC); \ No newline at end of file From 2a2500a7fb168d48beb011c8facc19e3adfb9c10 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Fri, 5 Dec 2025 01:46:18 +0700 Subject: [PATCH 08/19] test: idempotent insert --- internal/logstore/drivertest/drivertest.go | 137 +++++++++++++------ internal/logstore/memlogstore/memlogstore.go | 19 ++- 2 files changed, 111 insertions(+), 45 deletions(-) diff --git a/internal/logstore/drivertest/drivertest.go b/internal/logstore/drivertest/drivertest.go index b2af218f..94ea570e 100644 --- a/internal/logstore/drivertest/drivertest.go +++ b/internal/logstore/drivertest/drivertest.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/google/uuid" + "github.com/hookdeck/outpost/internal/idgen" "github.com/hookdeck/outpost/internal/logstore/cursor" "github.com/hookdeck/outpost/internal/logstore/driver" "github.com/hookdeck/outpost/internal/models" @@ -66,8 +66,8 @@ func testInsertManyDeliveryEvent(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() - destinationID := uuid.New().String() + tenantID := idgen.String() + destinationID := idgen.Destination() startTime := time.Now().Add(-1 * time.Hour) t.Run("insert single delivery event", func(t *testing.T) { @@ -81,7 +81,7 @@ func testInsertManyDeliveryEvent(t *testing.T, newHarness HarnessMaker) { testutil.DeliveryFactory.WithStatus("success"), ) de := &models.DeliveryEvent{ - ID: uuid.New().String(), + ID: idgen.String(), DestinationID: destinationID, Event: *event, Delivery: delivery, @@ -104,7 +104,7 @@ func testInsertManyDeliveryEvent(t *testing.T, newHarness HarnessMaker) { }) t.Run("insert multiple delivery events", func(t *testing.T) { - eventID := uuid.New().String() + eventID := idgen.Event() baseDeliveryTime := time.Now().Truncate(time.Second) event := testutil.EventFactory.AnyPointer( testutil.EventFactory.WithID(eventID), @@ -152,6 +152,57 @@ func testInsertManyDeliveryEvent(t *testing.T, newHarness HarnessMaker) { err := logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{}) require.NoError(t, err) }) + + t.Run("duplicate insert is idempotent", func(t *testing.T) { + // Create unique tenant to isolate this test + idempotentTenantID := idgen.String() + idempotentDestID := idgen.Destination() + eventTime := time.Now().Add(-30 * time.Minute).Truncate(time.Second) + deliveryTime := eventTime.Add(1 * time.Second) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithTenantID(idempotentTenantID), + testutil.EventFactory.WithDestinationID(idempotentDestID), + testutil.EventFactory.WithTime(eventTime), + ) + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithEventID(event.ID), + testutil.DeliveryFactory.WithDestinationID(idempotentDestID), + testutil.DeliveryFactory.WithStatus("success"), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + de := &models.DeliveryEvent{ + ID: idgen.String(), + DestinationID: idempotentDestID, + Event: *event, + Delivery: delivery, + } + batch := []*models.DeliveryEvent{de} + + // First insert + err := logStore.InsertManyDeliveryEvent(ctx, batch) + require.NoError(t, err) + + // Second insert (duplicate) - should not error + err = logStore.InsertManyDeliveryEvent(ctx, batch) + require.NoError(t, err) + + // Third insert (duplicate) - should not error + err = logStore.InsertManyDeliveryEvent(ctx, batch) + require.NoError(t, err) + + // Verify only 1 record exists (no duplicates) + queryStart := eventTime.Add(-1 * time.Hour) + response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ + TenantID: idempotentTenantID, + Limit: 100, + EventStart: &queryStart, + }) + require.NoError(t, err) + require.Len(t, response.Data, 1, "duplicate inserts should not create multiple records") + assert.Equal(t, event.ID, response.Data[0].Event.ID) + assert.Equal(t, delivery.ID, response.Data[0].Delivery.ID) + }) } // testListDeliveryEvent tests the ListDeliveryEvent method with various filters and pagination @@ -166,11 +217,11 @@ func testListDeliveryEvent(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() + tenantID := idgen.String() destinationIDs := []string{ - uuid.New().String(), - uuid.New().String(), - uuid.New().String(), + idgen.Destination(), + idgen.Destination(), + idgen.Destination(), } // Track events by various dimensions for assertions @@ -539,9 +590,9 @@ func testRetrieveEvent(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() - destinationID := uuid.New().String() - eventID := uuid.New().String() + tenantID := idgen.String() + destinationID := idgen.Destination() + eventID := idgen.Event() eventTime := time.Now().Truncate(time.Millisecond) event := testutil.EventFactory.AnyPointer( @@ -568,7 +619,7 @@ func testRetrieveEvent(t *testing.T, newHarness HarnessMaker) { ) de := &models.DeliveryEvent{ - ID: uuid.New().String(), + ID: idgen.String(), DestinationID: destinationID, Event: *event, Delivery: delivery, @@ -650,9 +701,9 @@ func testTenantIsolation(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenant1ID := uuid.New().String() - tenant2ID := uuid.New().String() - destinationID := uuid.New().String() + tenant1ID := idgen.String() + tenant2ID := idgen.String() + destinationID := idgen.Destination() // Create events for tenant1 event1 := testutil.EventFactory.AnyPointer( @@ -677,8 +728,8 @@ func testTenantIsolation(t *testing.T, newHarness HarnessMaker) { ) require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ - {ID: uuid.New().String(), DestinationID: destinationID, Event: *event1, Delivery: delivery1}, - {ID: uuid.New().String(), DestinationID: destinationID, Event: *event2, Delivery: delivery2}, + {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event1, Delivery: delivery1}, + {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event2, Delivery: delivery2}, })) startTime := time.Now().Add(-1 * time.Hour) @@ -759,8 +810,8 @@ func testPaginationSimple(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() - destinationID := uuid.New().String() + tenantID := idgen.String() + destinationID := idgen.Destination() baseTime := time.Now().Truncate(time.Second) // Create 5 delivery events with distinct times @@ -1120,10 +1171,10 @@ func generateRealisticTimestampData(t *testing.T, logStore driver.LogStore) *Pag t.Helper() ctx := context.Background() - tenantID := uuid.New().String() + tenantID := idgen.String() destinationIDs := []string{ - uuid.New().String(), - uuid.New().String(), + idgen.Destination(), + idgen.Destination(), } baseTime := time.Now().Truncate(time.Second) @@ -1316,8 +1367,8 @@ func generateIdenticalTimestampData(t *testing.T, logStore driver.LogStore) *Pag t.Helper() ctx := context.Background() - tenantID := uuid.New().String() - destinationID := uuid.New().String() + tenantID := idgen.String() + destinationID := idgen.Destination() // All events and deliveries share the SAME timestamp sameTime := time.Now().Truncate(time.Second) @@ -1716,8 +1767,8 @@ func testInvalidSortValues(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() - destinationID := uuid.New().String() + tenantID := idgen.String() + destinationID := idgen.Destination() baseTime := time.Now().Truncate(time.Second) // Insert test data with distinct delivery times @@ -1804,8 +1855,8 @@ func testEmptyVsNilFilters(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() - destinationID := uuid.New().String() + tenantID := idgen.String() + destinationID := idgen.Destination() startTime := time.Now().Add(-1 * time.Hour) // Insert test data @@ -1819,7 +1870,7 @@ func testEmptyVsNilFilters(t *testing.T, newHarness HarnessMaker) { testutil.DeliveryFactory.WithDestinationID(destinationID), ) require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{ - {ID: uuid.New().String(), DestinationID: destinationID, Event: *event, Delivery: delivery}, + {ID: idgen.DeliveryEvent(), DestinationID: destinationID, Event: *event, Delivery: delivery}, })) t.Run("nil DestinationIDs equals empty DestinationIDs", func(t *testing.T) { @@ -1902,8 +1953,8 @@ func testTimeBoundaryPrecision(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() - destinationID := uuid.New().String() + tenantID := idgen.String() + destinationID := idgen.Destination() // Create events at precise times boundaryTime := time.Now().Truncate(time.Second) @@ -2052,8 +2103,8 @@ func testEventIDPagination(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() - destinationID := uuid.New().String() + tenantID := idgen.String() + destinationID := idgen.Destination() eventID := "evt_with_many_deliveries" baseTime := time.Now().Truncate(time.Second) @@ -2184,8 +2235,8 @@ func testDataImmutability(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() - destinationID := uuid.New().String() + tenantID := idgen.String() + destinationID := idgen.Destination() eventID := "immutable_event" startTime := time.Now().Add(-1 * time.Hour) @@ -2323,8 +2374,8 @@ func testCursorMismatchedSortBy(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() - destinationID := uuid.New().String() + tenantID := idgen.String() + destinationID := idgen.Destination() baseTime := time.Now().Truncate(time.Second) startTime := baseTime.Add(-48 * time.Hour) @@ -2390,8 +2441,8 @@ func testCursorMismatchedSortOrder(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() - destinationID := uuid.New().String() + tenantID := idgen.String() + destinationID := idgen.Destination() baseTime := time.Now().Truncate(time.Second) startTime := baseTime.Add(-48 * time.Hour) @@ -2456,7 +2507,7 @@ func testMalformedCursor(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() + tenantID := idgen.String() startTime := time.Now().Add(-1 * time.Hour) testCases := []struct { @@ -2502,8 +2553,8 @@ func testCursorMatchingSortParams(t *testing.T, newHarness HarnessMaker) { logStore, err := h.MakeDriver(ctx) require.NoError(t, err) - tenantID := uuid.New().String() - destinationID := uuid.New().String() + tenantID := idgen.String() + destinationID := idgen.Destination() baseTime := time.Now().Truncate(time.Second) startTime := baseTime.Add(-48 * time.Hour) diff --git a/internal/logstore/memlogstore/memlogstore.go b/internal/logstore/memlogstore/memlogstore.go index c402f6f6..5afd521e 100644 --- a/internal/logstore/memlogstore/memlogstore.go +++ b/internal/logstore/memlogstore/memlogstore.go @@ -29,15 +29,30 @@ func (s *memLogStore) InsertManyDeliveryEvent(ctx context.Context, deliveryEvent s.mu.Lock() defer s.mu.Unlock() - // Deep copy to avoid external mutation for _, de := range deliveryEvents { + // Deep copy to avoid external mutation copied := &models.DeliveryEvent{ ID: de.ID, DestinationID: de.DestinationID, Event: de.Event, Delivery: de.Delivery, } - s.deliveryEvents = append(s.deliveryEvents, copied) + + // Check for existing entry and update (idempotent upsert) + found := false + for i, existing := range s.deliveryEvents { + // Match on event_id + delivery_id (same as pglogstore index key) + if existing.Event.ID == de.Event.ID && existing.Delivery != nil && de.Delivery != nil && existing.Delivery.ID == de.Delivery.ID { + // Update existing entry (like ON CONFLICT DO UPDATE) + s.deliveryEvents[i] = copied + found = true + break + } + } + + if !found { + s.deliveryEvents = append(s.deliveryEvents, copied) + } } return nil } From bcc4225fc99d928bff4b7aa4801d2934432ea315 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Fri, 5 Dec 2025 03:20:01 +0700 Subject: [PATCH 09/19] feat: chlogstore implementation --- internal/logstore/chlogstore/README.md | 109 +++ internal/logstore/chlogstore/chlogstore.go | 661 +++++++++++++----- .../logstore/chlogstore/chlogstore_test.go | 16 +- internal/logstore/drivertest/drivertest.go | 9 + .../logstore/memlogstore/memlogstore_test.go | 5 + .../logstore/pglogstore/pglogstore_test.go | 5 + .../clickhouse/000001_init.down.sql | 4 +- .../migrations/clickhouse/000001_init.up.sql | 49 +- 8 files changed, 654 insertions(+), 204 deletions(-) create mode 100644 internal/logstore/chlogstore/README.md diff --git a/internal/logstore/chlogstore/README.md b/internal/logstore/chlogstore/README.md new file mode 100644 index 00000000..360452ce --- /dev/null +++ b/internal/logstore/chlogstore/README.md @@ -0,0 +1,109 @@ +# chlogstore + +ClickHouse implementation of the LogStore interface. + +## Schema + +Single denormalized table - each row represents a delivery attempt for an event. + +| Table | Engine | Partition | Order By | +|-------|--------|-----------|----------| +| `event_log` | ReplacingMergeTree | `toYYYYMMDD(delivery_time)` | `(tenant_id, destination_id, delivery_time, event_id, delivery_id)` | + +**Secondary indexes:** +- `idx_event_id` - bloom_filter for event_id lookups +- `idx_topic` - bloom_filter for topic filtering +- `idx_status` - set index for status filtering + +## Design Principles + +### Stateless Queries + +All queries are designed to be stateless: +- No `GROUP BY`, no aggregation +- Direct row access with `ORDER BY` + `LIMIT` +- O(limit) performance regardless of total data volume + +This avoids the scaling issues of aggregation-based queries that must scan all matching rows before applying LIMIT. + +### Eventual Consistency + +ReplacingMergeTree deduplicates rows asynchronously during background merges. This means: +- Duplicate inserts (retries) may briefly appear as multiple rows +- Background merge consolidates duplicates within seconds/minutes +- Production queries do NOT use `FINAL` to avoid full-scan overhead + +For most use cases (log viewing), brief duplicates are acceptable. + +## Operations + +### ListDeliveryEvent + +Direct index scan with cursor-based pagination. + +```sql +SELECT + event_id, tenant_id, destination_id, topic, eligible_for_retry, + event_time, metadata, data, + delivery_id, delivery_event_id, status, delivery_time, code, response_data +FROM event_log +WHERE tenant_id = ? + AND [optional filters: destination_id, status, topic, time ranges] + AND [cursor condition] +ORDER BY delivery_time DESC, delivery_id DESC +LIMIT 101 +``` + +**Cursor design:** +- Format: `v1:{sortBy}:{sortOrder}:{position}` (base62 encoded) +- Position for delivery_time sort: `{timestamp}::{delivery_id}` +- Position for event_time sort: `{event_timestamp}::{event_id}::{delivery_timestamp}::{delivery_id}` +- Validates sort params match - rejects mismatched cursors + +**Backward pagination:** +- Reverses ORDER BY direction +- Reverses comparison operator in cursor condition +- Reverses results after fetching + +### RetrieveEvent + +Direct lookup by tenant_id and event_id. + +```sql +SELECT + event_id, tenant_id, destination_id, topic, eligible_for_retry, + event_time, metadata, data +FROM event_log +WHERE tenant_id = ? AND event_id = ? +LIMIT 1 +``` + +With destination filter, adds `AND destination_id = ?`. + +### InsertManyDeliveryEvent + +Batch insert using ClickHouse's native batch API. + +```go +batch, _ := conn.PrepareBatch(ctx, ` + INSERT INTO event_log ( + event_id, tenant_id, destination_id, topic, eligible_for_retry, + event_time, metadata, data, + delivery_id, delivery_event_id, status, delivery_time, code, response_data + ) +`) +for _, de := range deliveryEvents { + batch.Append(...) +} +batch.Send() +``` + +**Idempotency:** ReplacingMergeTree deduplicates rows with identical ORDER BY keys during background merges. + +## Performance Characteristics + +| Operation | Complexity | Notes | +|-----------|------------|-------| +| ListDeliveryEvent | O(limit) | Index scan, stops at LIMIT | +| RetrieveEvent | O(1) | Single row lookup via bloom filter | +| InsertManyDeliveryEvent | O(batch) | Batch insert, async dedup | diff --git a/internal/logstore/chlogstore/chlogstore.go b/internal/logstore/chlogstore/chlogstore.go index f347ba48..f586c39f 100644 --- a/internal/logstore/chlogstore/chlogstore.go +++ b/internal/logstore/chlogstore/chlogstore.go @@ -2,8 +2,13 @@ package chlogstore import ( "context" + "encoding/json" + "fmt" + "strings" + "time" "github.com/hookdeck/outpost/internal/clickhouse" + "github.com/hookdeck/outpost/internal/logstore/cursor" "github.com/hookdeck/outpost/internal/logstore/driver" "github.com/hookdeck/outpost/internal/models" ) @@ -18,109 +23,355 @@ func NewLogStore(chDB clickhouse.DB) driver.LogStore { return &logStoreImpl{chDB: chDB} } -func (s *logStoreImpl) ListEvent(ctx context.Context, request driver.ListEventRequest) (driver.ListEventResponse, error) { - // TODO: implement - return driver.ListEventResponse{}, nil - - // var ( - // query string - // queryOpts []any - // ) - - // var cursor string - // if cursorTime, err := time.Parse(time.RFC3339, request.Cursor); err == nil { - // cursor = cursorTime.Format("2006-01-02T15:04:05") // RFC3339 without timezone - // } - - // if cursor == "" { - // query = ` - // SELECT - // id, - // tenant_id, - // destination_id, - // time, - // topic, - // eligible_for_retry, - // data, - // metadata - // FROM events - // WHERE tenant_id = ? - // AND (? = 0 OR destination_id IN ?) - // ORDER BY time DESC - // LIMIT ? - // ` - // queryOpts = []any{request.TenantID, len(request.DestinationIDs), request.DestinationIDs, request.Limit} - // } else { - // query = ` - // SELECT - // id, - // tenant_id, - // destination_id, - // time, - // topic, - // eligible_for_retry, - // data, - // metadata - // FROM events - // WHERE tenant_id = ? AND time < ? - // AND (? = 0 OR destination_id IN ?) - // ORDER BY time DESC - // LIMIT ? - // ` - // queryOpts = []any{request.TenantID, cursor, len(request.DestinationIDs), request.DestinationIDs, request.Limit} - // } - // rows, err := s.chDB.Query(ctx, query, queryOpts...) - // if err != nil { - // return driver.ListEventResponse{}, err - // } - // defer rows.Close() - - // var events []*models.Event - // for rows.Next() { - // event := &models.Event{} - // if err := rows.Scan( - // &event.ID, - // &event.TenantID, - // &event.DestinationID, - // &event.Time, - // &event.Topic, - // &event.EligibleForRetry, - // &event.Data, - // &event.Metadata, - // ); err != nil { - // return driver.ListEventResponse{}, err - // } - // events = append(events, event) - // } - // var nextCursor string - // if len(events) > 0 { - // nextCursor = events[len(events)-1].Time.Format(time.RFC3339) - // } - - // return driver.ListEventResponse{ - // Data: events, - // Next: nextCursor, - // Count: int64(len(events)), - // }, nil -} +func (s *logStoreImpl) ListDeliveryEvent(ctx context.Context, req driver.ListDeliveryEventRequest) (driver.ListDeliveryEventResponse, error) { + // Validate and set defaults + sortBy := req.SortBy + if sortBy != "event_time" && sortBy != "delivery_time" { + sortBy = "delivery_time" + } + sortOrder := req.SortOrder + if sortOrder != "asc" && sortOrder != "desc" { + sortOrder = "desc" + } + + limit := req.Limit + if limit <= 0 { + limit = 100 + } + + // Decode and validate cursors + nextCursor, prevCursor, err := cursor.DecodeAndValidate(req.Next, req.Prev, sortBy, sortOrder) + if err != nil { + return driver.ListDeliveryEventResponse{}, err + } + + // Determine if we're going backward (using prev cursor) + goingBackward := !prevCursor.IsEmpty() + + // Build ORDER BY clause + // For delivery_time sort: ORDER BY delivery_time, delivery_id + // For event_time sort: ORDER BY event_time, event_id, delivery_time, delivery_id + var orderByClause string + if sortBy == "event_time" { + if sortOrder == "desc" { + if goingBackward { + orderByClause = "ORDER BY event_time ASC, event_id ASC, delivery_time ASC, delivery_id ASC" + } else { + orderByClause = "ORDER BY event_time DESC, event_id DESC, delivery_time DESC, delivery_id DESC" + } + } else { + if goingBackward { + orderByClause = "ORDER BY event_time DESC, event_id DESC, delivery_time DESC, delivery_id DESC" + } else { + orderByClause = "ORDER BY event_time ASC, event_id ASC, delivery_time ASC, delivery_id ASC" + } + } + } else { + if sortOrder == "desc" { + if goingBackward { + orderByClause = "ORDER BY delivery_time ASC, delivery_id ASC" + } else { + orderByClause = "ORDER BY delivery_time DESC, delivery_id DESC" + } + } else { + if goingBackward { + orderByClause = "ORDER BY delivery_time DESC, delivery_id DESC" + } else { + orderByClause = "ORDER BY delivery_time ASC, delivery_id ASC" + } + } + } + + // Build query with filters + var conditions []string + var args []interface{} + + // Required: tenant_id + conditions = append(conditions, "tenant_id = ?") + args = append(args, req.TenantID) + + // Optional filters + if req.EventID != "" { + conditions = append(conditions, "event_id = ?") + args = append(args, req.EventID) + } + + if len(req.DestinationIDs) > 0 { + conditions = append(conditions, "destination_id IN ?") + args = append(args, req.DestinationIDs) + } + + if req.Status != "" { + conditions = append(conditions, "status = ?") + args = append(args, req.Status) + } + + if len(req.Topics) > 0 { + conditions = append(conditions, "topic IN ?") + args = append(args, req.Topics) + } + + // Time filters + if req.EventStart != nil { + conditions = append(conditions, "event_time >= ?") + args = append(args, *req.EventStart) + } + if req.EventEnd != nil { + conditions = append(conditions, "event_time <= ?") + args = append(args, *req.EventEnd) + } + if req.DeliveryStart != nil { + conditions = append(conditions, "delivery_time >= ?") + args = append(args, *req.DeliveryStart) + } + if req.DeliveryEnd != nil { + conditions = append(conditions, "delivery_time <= ?") + args = append(args, *req.DeliveryEnd) + } + + // Cursor conditions + if !nextCursor.IsEmpty() { + cursorCond := buildCursorCondition(sortBy, sortOrder, nextCursor.Position, false) + conditions = append(conditions, cursorCond) + } else if !prevCursor.IsEmpty() { + cursorCond := buildCursorCondition(sortBy, sortOrder, prevCursor.Position, true) + conditions = append(conditions, cursorCond) + } + + whereClause := strings.Join(conditions, " AND ") -func (s *logStoreImpl) RetrieveEvent(ctx context.Context, tenantID, eventID string) (*models.Event, error) { - rows, err := s.chDB.Query(ctx, ` + query := fmt.Sprintf(` SELECT - id, + event_id, tenant_id, destination_id, - time, topic, eligible_for_retry, + event_time, + metadata, data, - metadata - FROM events - WHERE tenant_id = ? AND id = ? - `, tenantID, eventID, - ) + delivery_id, + delivery_event_id, + status, + delivery_time, + code, + response_data + FROM event_log + WHERE %s + %s + LIMIT %d + `, whereClause, orderByClause, limit+1) + + rows, err := s.chDB.Query(ctx, query, args...) if err != nil { - return nil, err + return driver.ListDeliveryEventResponse{}, fmt.Errorf("query failed: %w", err) + } + defer rows.Close() + + type rowData struct { + de *models.DeliveryEvent + eventTime time.Time + deliveryTime time.Time + } + var results []rowData + + for rows.Next() { + var ( + eventID string + tenantID string + destinationID string + topic string + eligibleForRetry bool + eventTime time.Time + metadataStr string + dataStr string + deliveryID string + deliveryEventID string + status string + deliveryTime time.Time + code string + responseDataStr string + ) + + err := rows.Scan( + &eventID, + &tenantID, + &destinationID, + &topic, + &eligibleForRetry, + &eventTime, + &metadataStr, + &dataStr, + &deliveryID, + &deliveryEventID, + &status, + &deliveryTime, + &code, + &responseDataStr, + ) + if err != nil { + return driver.ListDeliveryEventResponse{}, fmt.Errorf("scan failed: %w", err) + } + + // Parse JSON fields + var metadata map[string]string + var data map[string]interface{} + var responseData map[string]interface{} + + if metadataStr != "" { + if err := json.Unmarshal([]byte(metadataStr), &metadata); err != nil { + return driver.ListDeliveryEventResponse{}, fmt.Errorf("failed to unmarshal metadata: %w", err) + } + } + if dataStr != "" { + if err := json.Unmarshal([]byte(dataStr), &data); err != nil { + return driver.ListDeliveryEventResponse{}, fmt.Errorf("failed to unmarshal data: %w", err) + } + } + if responseDataStr != "" { + if err := json.Unmarshal([]byte(responseDataStr), &responseData); err != nil { + return driver.ListDeliveryEventResponse{}, fmt.Errorf("failed to unmarshal response_data: %w", err) + } + } + + de := &models.DeliveryEvent{ + ID: deliveryEventID, + DestinationID: destinationID, + Event: models.Event{ + ID: eventID, + TenantID: tenantID, + DestinationID: destinationID, + Topic: topic, + EligibleForRetry: eligibleForRetry, + Time: eventTime, + Data: data, + Metadata: metadata, + }, + Delivery: &models.Delivery{ + ID: deliveryID, + EventID: eventID, + DestinationID: destinationID, + Status: status, + Time: deliveryTime, + Code: code, + ResponseData: responseData, + }, + } + + results = append(results, rowData{de: de, eventTime: eventTime, deliveryTime: deliveryTime}) + } + + if err := rows.Err(); err != nil { + return driver.ListDeliveryEventResponse{}, fmt.Errorf("rows error: %w", err) + } + + // Handle pagination + var hasMore bool + if len(results) > limit { + hasMore = true + results = results[:limit] // Always keep the first `limit` items, remove the extra + } + + // When going backward, we queried in reverse order, so reverse results back to normal order + if goingBackward { + for i, j := 0, len(results)-1; i < j; i, j = i+1, j-1 { + results[i], results[j] = results[j], results[i] + } + } + + // Build response + data := make([]*models.DeliveryEvent, len(results)) + for i, r := range results { + data[i] = r.de + } + + var nextEncoded, prevEncoded string + if len(results) > 0 { + getPosition := func(r rowData) string { + if sortBy == "event_time" { + // Composite cursor: eventTime::eventID::deliveryTime::deliveryID + return fmt.Sprintf("%d::%s::%d::%s", + r.eventTime.UnixMilli(), + r.de.Event.ID, + r.deliveryTime.UnixMilli(), + r.de.Delivery.ID, + ) + } + // delivery_time cursor: deliveryTime::deliveryID + return fmt.Sprintf("%d::%s", r.deliveryTime.UnixMilli(), r.de.Delivery.ID) + } + + encodeCursor := func(position string) string { + return cursor.Encode(cursor.Cursor{ + SortBy: sortBy, + SortOrder: sortOrder, + Position: position, + }) + } + + if !prevCursor.IsEmpty() { + nextEncoded = encodeCursor(getPosition(results[len(results)-1])) + if hasMore { + prevEncoded = encodeCursor(getPosition(results[0])) + } + } else if !nextCursor.IsEmpty() { + prevEncoded = encodeCursor(getPosition(results[0])) + if hasMore { + nextEncoded = encodeCursor(getPosition(results[len(results)-1])) + } + } else { + if hasMore { + nextEncoded = encodeCursor(getPosition(results[len(results)-1])) + } + } + } + + return driver.ListDeliveryEventResponse{ + Data: data, + Next: nextEncoded, + Prev: prevEncoded, + }, nil +} + +func (s *logStoreImpl) RetrieveEvent(ctx context.Context, req driver.RetrieveEventRequest) (*models.Event, error) { + var query string + var args []interface{} + + if req.DestinationID != "" { + query = ` + SELECT + event_id, + tenant_id, + destination_id, + topic, + eligible_for_retry, + event_time, + metadata, + data + FROM event_log + WHERE tenant_id = ? AND event_id = ? AND destination_id = ? + LIMIT 1` + args = []interface{}{req.TenantID, req.EventID, req.DestinationID} + } else { + query = ` + SELECT + event_id, + tenant_id, + destination_id, + topic, + eligible_for_retry, + event_time, + metadata, + data + FROM event_log + WHERE tenant_id = ? AND event_id = ? + LIMIT 1` + args = []interface{}{req.TenantID, req.EventID} + } + + rows, err := s.chDB.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("query failed: %w", err) } defer rows.Close() @@ -128,123 +379,183 @@ func (s *logStoreImpl) RetrieveEvent(ctx context.Context, tenantID, eventID stri return nil, nil } + var metadataStr, dataStr string event := &models.Event{} + if err := rows.Scan( &event.ID, &event.TenantID, &event.DestinationID, - &event.Time, &event.Topic, &event.EligibleForRetry, - &event.Data, - &event.Metadata, + &event.Time, + &metadataStr, + &dataStr, ); err != nil { - return nil, err + return nil, fmt.Errorf("scan failed: %w", err) } - return event, nil -} - -func (s *logStoreImpl) ListDelivery(ctx context.Context, request driver.ListDeliveryRequest) ([]*models.Delivery, error) { - query := ` - SELECT - id, - event_id, - destination_id, - status, - time - FROM deliveries - WHERE event_id = ? - ORDER BY time DESC - ` - rows, err := s.chDB.Query(ctx, query, request.EventID) - if err != nil { - return nil, err + // Parse JSON fields + if metadataStr != "" { + if err := json.Unmarshal([]byte(metadataStr), &event.Metadata); err != nil { + return nil, fmt.Errorf("failed to unmarshal metadata: %w", err) + } } - defer rows.Close() - - var deliveries []*models.Delivery - for rows.Next() { - delivery := &models.Delivery{} - if err := rows.Scan( - &delivery.ID, - &delivery.EventID, - &delivery.DestinationID, - &delivery.Status, - &delivery.Time, - ); err != nil { - return nil, err + if dataStr != "" { + if err := json.Unmarshal([]byte(dataStr), &event.Data); err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %w", err) } - deliveries = append(deliveries, delivery) } - return deliveries, nil + return event, nil } -func (s *logStoreImpl) InsertManyEvent(ctx context.Context, events []*models.Event) error { +func (s *logStoreImpl) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents []*models.DeliveryEvent) error { + if len(deliveryEvents) == 0 { + return nil + } + batch, err := s.chDB.PrepareBatch(ctx, - "INSERT INTO events (id, tenant_id, destination_id, time, topic, eligible_for_retry, metadata, data) VALUES (?, ?, ?, ?, ?, ?)", + `INSERT INTO event_log ( + event_id, tenant_id, destination_id, topic, eligible_for_retry, event_time, metadata, data, + delivery_id, delivery_event_id, status, delivery_time, code, response_data + )`, ) if err != nil { - return err + return fmt.Errorf("prepare batch failed: %w", err) } - for _, event := range events { + for _, de := range deliveryEvents { + metadataJSON, err := json.Marshal(de.Event.Metadata) + if err != nil { + return fmt.Errorf("failed to marshal metadata: %w", err) + } + dataJSON, err := json.Marshal(de.Event.Data) + if err != nil { + return fmt.Errorf("failed to marshal data: %w", err) + } + + var deliveryID, status, code string + var deliveryTime time.Time + var responseDataJSON []byte + + if de.Delivery != nil { + deliveryID = de.Delivery.ID + status = de.Delivery.Status + deliveryTime = de.Delivery.Time + code = de.Delivery.Code + responseDataJSON, err = json.Marshal(de.Delivery.ResponseData) + if err != nil { + return fmt.Errorf("failed to marshal response_data: %w", err) + } + } else { + deliveryID = de.ID + status = "pending" + deliveryTime = de.Event.Time + code = "" + responseDataJSON = []byte("{}") + } + if err := batch.Append( - &event.ID, - &event.TenantID, - &event.DestinationID, - &event.Time, - &event.Topic, - &event.EligibleForRetry, - &event.Metadata, - &event.Data, + de.Event.ID, + de.Event.TenantID, + de.DestinationID, + de.Event.Topic, + de.Event.EligibleForRetry, + de.Event.Time, + string(metadataJSON), + string(dataJSON), + deliveryID, + de.ID, + status, + deliveryTime, + code, + string(responseDataJSON), ); err != nil { - return err + return fmt.Errorf("batch append failed: %w", err) } } if err := batch.Send(); err != nil { - return err + return fmt.Errorf("batch send failed: %w", err) } return nil } -func (s *logStoreImpl) InsertManyDelivery(ctx context.Context, deliveries []*models.Delivery) error { - batch, err := s.chDB.PrepareBatch(ctx, - "INSERT INTO deliveries (id, delivery_event_id, event_id, destination_id, status, time) VALUES (?, ?, ?, ?, ?, ?)", - ) - if err != nil { - return err - } +// buildCursorCondition builds a SQL condition for cursor-based pagination +func buildCursorCondition(sortBy, sortOrder, position string, isBackward bool) string { + // Parse position based on sortBy + // For delivery_time: "timestamp::deliveryID" + // For event_time: "timestamp::eventID::timestamp::deliveryID" - for _, delivery := range deliveries { - if err := batch.Append( - &delivery.ID, - &delivery.DeliveryEventID, - &delivery.EventID, - &delivery.DestinationID, - &delivery.Status, - &delivery.Time, - ); err != nil { - return err + if sortBy == "event_time" { + // Parse: eventTimeMs::eventID::deliveryTimeMs::deliveryID + parts := strings.SplitN(position, "::", 4) + if len(parts) != 4 { + return "1=1" // invalid cursor, return always true } - } + eventTimeMs := parts[0] + eventID := parts[1] + deliveryTimeMs := parts[2] + deliveryID := parts[3] - if err := batch.Send(); err != nil { - return err + // Determine comparison direction + var cmp string + if sortOrder == "desc" { + if isBackward { + cmp = ">" + } else { + cmp = "<" + } + } else { + if isBackward { + cmp = "<" + } else { + cmp = ">" + } + } + + // Build multi-column comparison + // (event_time, event_id, delivery_time, delivery_id) < (cursor_values) + return fmt.Sprintf(`( + event_time %s fromUnixTimestamp64Milli(%s) + OR (event_time = fromUnixTimestamp64Milli(%s) AND event_id %s '%s') + OR (event_time = fromUnixTimestamp64Milli(%s) AND event_id = '%s' AND delivery_time %s fromUnixTimestamp64Milli(%s)) + OR (event_time = fromUnixTimestamp64Milli(%s) AND event_id = '%s' AND delivery_time = fromUnixTimestamp64Milli(%s) AND delivery_id %s '%s') + )`, + cmp, eventTimeMs, + eventTimeMs, cmp, eventID, + eventTimeMs, eventID, cmp, deliveryTimeMs, + eventTimeMs, eventID, deliveryTimeMs, cmp, deliveryID, + ) } - return nil -} + // delivery_time sort: "timestamp::deliveryID" + parts := strings.SplitN(position, "::", 2) + if len(parts) != 2 { + return "1=1" + } + deliveryTimeMs := parts[0] + deliveryID := parts[1] -func (s *logStoreImpl) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents []*models.DeliveryEvent) error { - // TODO: implement - return nil -} + var cmp string + if sortOrder == "desc" { + if isBackward { + cmp = ">" + } else { + cmp = "<" + } + } else { + if isBackward { + cmp = "<" + } else { + cmp = ">" + } + } -func (s *logStoreImpl) RetrieveEventByDestination(ctx context.Context, tenantID, destinationID, eventID string) (*models.Event, error) { - // TODO: implement - return nil, nil + return fmt.Sprintf(`( + delivery_time %s fromUnixTimestamp64Milli(%s) + OR (delivery_time = fromUnixTimestamp64Milli(%s) AND delivery_id %s '%s') + )`, cmp, deliveryTimeMs, deliveryTimeMs, cmp, deliveryID) } diff --git a/internal/logstore/chlogstore/chlogstore_test.go b/internal/logstore/chlogstore/chlogstore_test.go index 194dc50c..5f298a7a 100644 --- a/internal/logstore/chlogstore/chlogstore_test.go +++ b/internal/logstore/chlogstore/chlogstore_test.go @@ -9,15 +9,16 @@ import ( "github.com/hookdeck/outpost/internal/logstore/drivertest" "github.com/hookdeck/outpost/internal/migrator" "github.com/hookdeck/outpost/internal/util/testinfra" + "github.com/hookdeck/outpost/internal/util/testutil" "github.com/stretchr/testify/require" ) -// func TestConformance(t *testing.T) { -// testutil.CheckIntegrationTest(t) -// t.Parallel() +func TestConformance(t *testing.T) { + testutil.CheckIntegrationTest(t) + t.Parallel() -// drivertest.RunConformanceTests(t, newHarness) -// } + drivertest.RunConformanceTests(t, newHarness) +} type harness struct { chDB clickhouse.DB @@ -72,6 +73,11 @@ func (h *harness) Close() { h.closer() } +func (h *harness) FlushWrites(ctx context.Context) error { + // Force ClickHouse to merge parts and deduplicate rows + return h.chDB.Exec(ctx, "OPTIMIZE TABLE event_log FINAL") +} + func (h *harness) MakeDriver(ctx context.Context) (driver.LogStore, error) { return NewLogStore(h.chDB), nil } diff --git a/internal/logstore/drivertest/drivertest.go b/internal/logstore/drivertest/drivertest.go index 94ea570e..3bcecb14 100644 --- a/internal/logstore/drivertest/drivertest.go +++ b/internal/logstore/drivertest/drivertest.go @@ -20,6 +20,11 @@ import ( type Harness interface { MakeDriver(ctx context.Context) (driver.LogStore, error) + // FlushWrites ensures all writes are fully persisted and visible. + // For eventually consistent stores (e.g., ClickHouse ReplacingMergeTree), + // this forces merge/compaction. For immediately consistent stores (e.g., PostgreSQL), + // this is a no-op. + FlushWrites(ctx context.Context) error Close() } @@ -191,6 +196,10 @@ func testInsertManyDeliveryEvent(t *testing.T, newHarness HarnessMaker) { err = logStore.InsertManyDeliveryEvent(ctx, batch) require.NoError(t, err) + // Flush writes to ensure deduplication is visible (for eventually consistent stores) + err = h.FlushWrites(ctx) + require.NoError(t, err) + // Verify only 1 record exists (no duplicates) queryStart := eventTime.Add(-1 * time.Hour) response, err := logStore.ListDeliveryEvent(ctx, driver.ListDeliveryEventRequest{ diff --git a/internal/logstore/memlogstore/memlogstore_test.go b/internal/logstore/memlogstore/memlogstore_test.go index c79394fb..1451e836 100644 --- a/internal/logstore/memlogstore/memlogstore_test.go +++ b/internal/logstore/memlogstore/memlogstore_test.go @@ -20,6 +20,11 @@ func (h *memLogStoreHarness) Close() { // No-op for in-memory store } +func (h *memLogStoreHarness) FlushWrites(ctx context.Context) error { + // In-memory store is immediately consistent + return nil +} + func newHarness(ctx context.Context, t *testing.T) (drivertest.Harness, error) { return &memLogStoreHarness{ logStore: NewLogStore(), diff --git a/internal/logstore/pglogstore/pglogstore_test.go b/internal/logstore/pglogstore/pglogstore_test.go index 7dcfe0de..bb3e39b4 100644 --- a/internal/logstore/pglogstore/pglogstore_test.go +++ b/internal/logstore/pglogstore/pglogstore_test.go @@ -70,6 +70,11 @@ func (h *harness) Close() { h.closer() } +func (h *harness) FlushWrites(ctx context.Context) error { + // PostgreSQL is immediately consistent, no flush needed + return nil +} + func (h *harness) MakeDriver(ctx context.Context) (driver.LogStore, error) { return NewLogStore(h.db), nil } diff --git a/internal/migrator/migrations/clickhouse/000001_init.down.sql b/internal/migrator/migrations/clickhouse/000001_init.down.sql index 1bc93eb7..f3db6fbf 100644 --- a/internal/migrator/migrations/clickhouse/000001_init.down.sql +++ b/internal/migrator/migrations/clickhouse/000001_init.down.sql @@ -1,3 +1 @@ -DROP TABLE IF EXISTS deliveries; - -DROP TABLE IF EXISTS events; \ No newline at end of file +DROP TABLE IF EXISTS event_log; diff --git a/internal/migrator/migrations/clickhouse/000001_init.up.sql b/internal/migrator/migrations/clickhouse/000001_init.up.sql index ab5e207c..f12b7f78 100644 --- a/internal/migrator/migrations/clickhouse/000001_init.up.sql +++ b/internal/migrator/migrations/clickhouse/000001_init.up.sql @@ -1,23 +1,30 @@ -CREATE TABLE IF NOT EXISTS events ( - id String, - tenant_id String, - destination_id String, - topic String, - eligible_for_retry Bool, - time DateTime, - metadata String, - data String -) ENGINE = MergeTree -ORDER BY - (id, time); +-- Single denormalized table for events and deliveries +-- Each row represents a delivery attempt for an event +-- Stateless queries: no GROUP BY, no aggregation, direct row access -CREATE TABLE IF NOT EXISTS deliveries ( - id String, - delivery_event_id String, - event_id String, - destination_id String, - status String, - time DateTime +CREATE TABLE IF NOT EXISTS event_log ( + -- Event fields + event_id String, + tenant_id String, + destination_id String, + topic String, + eligible_for_retry Bool, + event_time DateTime64(3), + metadata String, -- JSON serialized + data String, -- JSON serialized + + -- Delivery fields + delivery_id String, + delivery_event_id String, + status String, -- 'success', 'failed' + delivery_time DateTime64(3), + code String, + response_data String, -- JSON serialized + + -- Indexes for filtering (bloom filters help skip granules) + INDEX idx_event_id event_id TYPE bloom_filter GRANULARITY 4, + INDEX idx_topic topic TYPE bloom_filter GRANULARITY 4, + INDEX idx_status status TYPE set(100) GRANULARITY 4 ) ENGINE = ReplacingMergeTree -ORDER BY - (id, time); \ No newline at end of file +PARTITION BY toYYYYMMDD(delivery_time) +ORDER BY (tenant_id, destination_id, delivery_time, event_id, delivery_id); From b4245237a5a601efb5922c9ccf007e2d9d99fede Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Fri, 5 Dec 2025 19:44:16 +0700 Subject: [PATCH 10/19] chore: update logstore usage with updated interface --- internal/apirouter/retry_handlers.go | 5 ++++- internal/deliverymq/messagehandler.go | 8 ++++++-- internal/deliverymq/mock_test.go | 7 ++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/internal/apirouter/retry_handlers.go b/internal/apirouter/retry_handlers.go index d6035e4b..ca357a86 100644 --- a/internal/apirouter/retry_handlers.go +++ b/internal/apirouter/retry_handlers.go @@ -52,7 +52,10 @@ func (h *RetryHandlers) Retry(c *gin.Context) { return } - event, err := h.logStore.RetrieveEvent(c, tenantID, eventID) + event, err := h.logStore.RetrieveEvent(c, logstore.RetrieveEventRequest{ + TenantID: tenantID, + EventID: eventID, + }) if err != nil { AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return diff --git a/internal/deliverymq/messagehandler.go b/internal/deliverymq/messagehandler.go index 73390aab..bc939cb9 100644 --- a/internal/deliverymq/messagehandler.go +++ b/internal/deliverymq/messagehandler.go @@ -12,6 +12,7 @@ import ( "github.com/hookdeck/outpost/internal/destregistry" "github.com/hookdeck/outpost/internal/idempotence" "github.com/hookdeck/outpost/internal/logging" + "github.com/hookdeck/outpost/internal/logstore" "github.com/hookdeck/outpost/internal/models" "github.com/hookdeck/outpost/internal/mqs" "github.com/hookdeck/outpost/internal/scheduler" @@ -96,7 +97,7 @@ type DestinationGetter interface { } type EventGetter interface { - RetrieveEvent(ctx context.Context, tenantID, eventID string) (*models.Event, error) + RetrieveEvent(ctx context.Context, request logstore.RetrieveEventRequest) (*models.Event, error) } type DeliveryTracer interface { @@ -417,7 +418,10 @@ func (h *messageHandler) ensureDeliveryEvent(ctx context.Context, deliveryEvent return nil } - event, err := h.logStore.RetrieveEvent(ctx, deliveryEvent.Event.TenantID, deliveryEvent.Event.ID) + event, err := h.logStore.RetrieveEvent(ctx, logstore.RetrieveEventRequest{ + TenantID: deliveryEvent.Event.TenantID, + EventID: deliveryEvent.Event.ID, + }) if err != nil { return err } diff --git a/internal/deliverymq/mock_test.go b/internal/deliverymq/mock_test.go index c627ce08..78c2ac6f 100644 --- a/internal/deliverymq/mock_test.go +++ b/internal/deliverymq/mock_test.go @@ -9,6 +9,7 @@ import ( "github.com/hookdeck/outpost/internal/alert" "github.com/hookdeck/outpost/internal/idgen" + "github.com/hookdeck/outpost/internal/logstore" "github.com/hookdeck/outpost/internal/models" mqs "github.com/hookdeck/outpost/internal/mqs" "github.com/hookdeck/outpost/internal/scheduler" @@ -111,12 +112,12 @@ func (m *mockEventGetter) clearError() { m.err = nil } -func (m *mockEventGetter) RetrieveEvent(ctx context.Context, tenantID, eventID string) (*models.Event, error) { +func (m *mockEventGetter) RetrieveEvent(ctx context.Context, req logstore.RetrieveEventRequest) (*models.Event, error) { if m.err != nil { return nil, m.err } - m.lastRetrievedID = eventID - event, ok := m.events[eventID] + m.lastRetrievedID = req.EventID + event, ok := m.events[req.EventID] if !ok { return nil, errors.New("event not found") } From e7a349362c93b1485ac0849f54e612e2cecae1c1 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Tue, 2 Dec 2025 12:52:44 +0700 Subject: [PATCH 11/19] chore: configure logstore with ch driver --- internal/config/config.go | 71 ++++++++++++++++++----------------- internal/config/validation.go | 9 +---- internal/logstore/logstore.go | 18 ++++----- internal/services/builder.go | 3 +- 4 files changed, 48 insertions(+), 53 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index b31aec01..79901389 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,6 +9,7 @@ import ( "github.com/caarlos0/env/v9" "github.com/hookdeck/outpost/internal/backoff" + "github.com/hookdeck/outpost/internal/clickhouse" "github.com/hookdeck/outpost/internal/migrator" "github.com/hookdeck/outpost/internal/redis" "github.com/hookdeck/outpost/internal/telemetry" @@ -61,10 +62,10 @@ type Config struct { HTTPUserAgent string `yaml:"http_user_agent" env:"HTTP_USER_AGENT" desc:"Custom HTTP User-Agent string for outgoing webhook deliveries. If unset, a default (OrganizationName/Version) is used." required:"N"` // Infrastructure - Redis RedisConfig `yaml:"redis"` - // ClickHouse ClickHouseConfig `yaml:"clickhouse"` - PostgresURL string `yaml:"postgres" env:"POSTGRES_URL" desc:"Connection URL for PostgreSQL, used for log storage. Example: 'postgres://user:pass@host:port/dbname?sslmode=disable'." required:"Y"` - MQs *MQsConfig `yaml:"mqs"` + Redis RedisConfig `yaml:"redis"` + ClickHouse ClickHouseConfig `yaml:"clickhouse"` + PostgresURL string `yaml:"postgres" env:"POSTGRES_URL" desc:"Connection URL for PostgreSQL, used for log storage. Example: 'postgres://user:pass@host:port/dbname?sslmode=disable'." required:"N"` + MQs *MQsConfig `yaml:"mqs"` // PublishMQ PublishMQ PublishMQConfig `yaml:"publishmq"` @@ -131,9 +132,9 @@ func (c *Config) InitDefaults() { Host: "127.0.0.1", Port: 6379, } - // c.ClickHouse = ClickHouseConfig{ - // Database: "outpost", - // } + c.ClickHouse = ClickHouseConfig{ + Database: "outpost", + } c.MQs = &MQsConfig{ RabbitMQ: RabbitMQConfig{ Exchange: "outpost", @@ -378,24 +379,24 @@ func (c *RedisConfig) ToConfig() *redis.RedisConfig { } } -// type ClickHouseConfig struct { -// Addr string `yaml:"addr" env:"CLICKHOUSE_ADDR" desc:"Address (host:port) of the ClickHouse server. Example: 'localhost:9000'. Required if ClickHouse is used for log storage." required:"C"` -// Username string `yaml:"username" env:"CLICKHOUSE_USERNAME" desc:"Username for ClickHouse authentication." required:"N"` -// Password string `yaml:"password" env:"CLICKHOUSE_PASSWORD" desc:"Password for ClickHouse authentication." required:"N"` -// Database string `yaml:"database" env:"CLICKHOUSE_DATABASE" desc:"Database name in ClickHouse to use." required:"N"` -// } - -// func (c *ClickHouseConfig) ToConfig() *clickhouse.ClickHouseConfig { -// if c.Addr == "" { -// return nil -// } -// return &clickhouse.ClickHouseConfig{ -// Addr: c.Addr, -// Username: c.Username, -// Password: c.Password, -// Database: c.Database, -// } -// } +type ClickHouseConfig struct { + Addr string `yaml:"addr" env:"CLICKHOUSE_ADDR" desc:"Address (host:port) of the ClickHouse server. Example: 'localhost:9000'." required:"N"` + Username string `yaml:"username" env:"CLICKHOUSE_USERNAME" desc:"Username for ClickHouse authentication." required:"N"` + Password string `yaml:"password" env:"CLICKHOUSE_PASSWORD" desc:"Password for ClickHouse authentication." required:"N"` + Database string `yaml:"database" env:"CLICKHOUSE_DATABASE" desc:"Database name in ClickHouse to use." required:"N"` +} + +func (c *ClickHouseConfig) ToConfig() *clickhouse.ClickHouseConfig { + if c.Addr == "" { + return nil + } + return &clickhouse.ClickHouseConfig{ + Addr: c.Addr, + Username: c.Username, + Password: c.Password, + Database: c.Database, + } +} type AlertConfig struct { CallbackURL string `yaml:"callback_url" env:"ALERT_CALLBACK_URL" desc:"URL to which Outpost will send a POST request when an alert is triggered (e.g., for destination failures)." required:"N"` @@ -447,10 +448,10 @@ func (c *Config) ToTelemetryApplicationInfo() telemetry.ApplicationInfo { portalEnabled := c.APIKey != "" && c.APIJWTSecret != "" entityStore := "redis" - logStore := "TODO" - // if c.ClickHouse.Addr != "" { - // logStore = "clickhouse" - // } + logStore := "" + if c.ClickHouse.Addr != "" { + logStore = "clickhouse" + } if c.PostgresURL != "" { logStore = "postgres" } @@ -471,11 +472,11 @@ func (c *Config) ToMigratorOpts() migrator.MigrationOpts { PG: migrator.MigrationOptsPG{ URL: c.PostgresURL, }, - // CH: migrator.MigrationOptsCH{ - // Addr: c.ClickHouse.Addr, - // Username: c.ClickHouse.Username, - // Password: c.ClickHouse.Password, - // Database: c.ClickHouse.Database, - // }, + CH: migrator.MigrationOptsCH{ + Addr: c.ClickHouse.Addr, + Username: c.ClickHouse.Username, + Password: c.ClickHouse.Password, + Database: c.ClickHouse.Database, + }, } } diff --git a/internal/config/validation.go b/internal/config/validation.go index 705ef3e8..5b9a7c5b 100644 --- a/internal/config/validation.go +++ b/internal/config/validation.go @@ -89,17 +89,10 @@ func (c *Config) validateRedis() error { } // validateLogStorage validates the ClickHouse / PG configuration -// Temporary: disable CH as it's not fully supported yet func (c *Config) validateLogStorage() error { - // if c.ClickHouse.Addr == "" && c.PostgresURL == "" { - // return ErrMissingLogStorage - // } - if c.PostgresURL == "" { + if c.ClickHouse.Addr == "" && c.PostgresURL == "" { return ErrMissingLogStorage } - // if c.ClickHouse.Addr != "" { - // return fmt.Errorf("ClickHouse is not currently supported") - // } return nil } diff --git a/internal/logstore/logstore.go b/internal/logstore/logstore.go index 63d0211d..cd4b5ace 100644 --- a/internal/logstore/logstore.go +++ b/internal/logstore/logstore.go @@ -49,20 +49,20 @@ func NewLogStore(ctx context.Context, driverOpts DriverOpts) (LogStore, error) { } type Config struct { - // ClickHouse *clickhouse.ClickHouseConfig - Postgres *string + ClickHouse *clickhouse.ClickHouseConfig + Postgres *string } func MakeDriverOpts(cfg Config) (DriverOpts, error) { driverOpts := DriverOpts{} - // if cfg.ClickHouse != nil { - // chDB, err := clickhouse.New(cfg.ClickHouse) - // if err != nil { - // return DriverOpts{}, err - // } - // driverOpts.CH = chDB - // } + if cfg.ClickHouse != nil { + chDB, err := clickhouse.New(cfg.ClickHouse) + if err != nil { + return DriverOpts{}, err + } + driverOpts.CH = chDB + } if cfg.Postgres != nil && *cfg.Postgres != "" { pgDB, err := pgxpool.New(context.Background(), *cfg.Postgres) diff --git a/internal/services/builder.go b/internal/services/builder.go index 0594fcbd..ac570ddd 100644 --- a/internal/services/builder.go +++ b/internal/services/builder.go @@ -502,7 +502,8 @@ func (s *serviceInstance) initRedis(ctx context.Context, cfg *config.Config, log func (s *serviceInstance) initLogStore(ctx context.Context, cfg *config.Config, logger *logging.Logger) error { logger.Debug("configuring log store driver", zap.String("service", s.name)) logStoreDriverOpts, err := logstore.MakeDriverOpts(logstore.Config{ - Postgres: &cfg.PostgresURL, + ClickHouse: cfg.ClickHouse.ToConfig(), + Postgres: &cfg.PostgresURL, }) if err != nil { logger.Error("log store driver configuration failed", zap.String("service", s.name), zap.Error(err)) From 65c6a0386408e6094255e185bd4b75ffded8439d Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Tue, 2 Dec 2025 12:52:58 +0700 Subject: [PATCH 12/19] test: e2e suite with ch --- cmd/e2e/configs/basic.go | 12 ++++++------ cmd/e2e/suites_test.go | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/e2e/configs/basic.go b/cmd/e2e/configs/basic.go index 65054bec..db90677e 100644 --- a/cmd/e2e/configs/basic.go +++ b/cmd/e2e/configs/basic.go @@ -148,12 +148,12 @@ func setLogStorage(t *testing.T, c *config.Config, logStorage LogStorageType) er case LogStorageTypePostgres: postgresURL := testinfra.NewPostgresConfig(t) c.PostgresURL = postgresURL - // case LogStorageTypeClickHouse: - // clickHouseConfig := testinfra.NewClickHouseConfig(t) - // c.ClickHouse.Addr = clickHouseConfig.Addr - // c.ClickHouse.Username = clickHouseConfig.Username - // c.ClickHouse.Password = clickHouseConfig.Password - // c.ClickHouse.Database = clickHouseConfig.Database + case LogStorageTypeClickHouse: + clickHouseConfig := testinfra.NewClickHouseConfig(t) + c.ClickHouse.Addr = clickHouseConfig.Addr + c.ClickHouse.Username = clickHouseConfig.Username + c.ClickHouse.Password = clickHouseConfig.Password + c.ClickHouse.Database = clickHouseConfig.Database default: return fmt.Errorf("invalid log storage type: %s", logStorage) } diff --git a/cmd/e2e/suites_test.go b/cmd/e2e/suites_test.go index 926f8da1..7209d7d8 100644 --- a/cmd/e2e/suites_test.go +++ b/cmd/e2e/suites_test.go @@ -178,13 +178,13 @@ func (s *basicSuite) TearDownSuite() { s.e2eSuite.TearDownSuite() } -// func TestCHBasicSuite(t *testing.T) { -// t.Parallel() -// if testing.Short() { -// t.Skip("skipping e2e test") -// } -// suite.Run(t, &basicSuite{logStorageType: configs.LogStorageTypeClickHouse}) -// } +func TestBasicSuiteWithCH(t *testing.T) { + t.Parallel() + if testing.Short() { + t.Skip("skipping e2e test") + } + suite.Run(t, &basicSuite{logStorageType: configs.LogStorageTypeClickHouse}) +} func TestPGBasicSuite(t *testing.T) { t.Parallel() From 0f6d3dae729107cde657547ef039d2eb6c7624de Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Mon, 8 Dec 2025 18:03:16 +0700 Subject: [PATCH 13/19] feat: implement delivery retrieve --- internal/logstore/chlogstore/chlogstore.go | 115 ++++++++++++ internal/logstore/driver/driver.go | 6 + internal/logstore/drivertest/drivertest.go | 166 ++++++++++++++++++ internal/logstore/logstore.go | 2 + internal/logstore/memlogstore/memlogstore.go | 13 ++ internal/logstore/noop.go | 4 + internal/logstore/pglogstore/pglogstore.go | 88 ++++++++++ .../migrations/clickhouse/000001_init.up.sql | 1 + 8 files changed, 395 insertions(+) diff --git a/internal/logstore/chlogstore/chlogstore.go b/internal/logstore/chlogstore/chlogstore.go index f586c39f..f28bf927 100644 --- a/internal/logstore/chlogstore/chlogstore.go +++ b/internal/logstore/chlogstore/chlogstore.go @@ -410,6 +410,121 @@ func (s *logStoreImpl) RetrieveEvent(ctx context.Context, req driver.RetrieveEve return event, nil } +// RetrieveDeliveryEvent retrieves a single delivery event by delivery ID. +func (s *logStoreImpl) RetrieveDeliveryEvent(ctx context.Context, req driver.RetrieveDeliveryEventRequest) (*models.DeliveryEvent, error) { + query := ` + SELECT + event_id, + tenant_id, + destination_id, + topic, + eligible_for_retry, + event_time, + metadata, + data, + delivery_id, + delivery_event_id, + status, + delivery_time, + code, + response_data + FROM event_log + WHERE tenant_id = ? AND delivery_id = ? + LIMIT 1` + + rows, err := s.chDB.Query(ctx, query, req.TenantID, req.DeliveryID) + if err != nil { + return nil, fmt.Errorf("query failed: %w", err) + } + defer rows.Close() + + if !rows.Next() { + return nil, nil + } + + var ( + eventID string + tenantID string + destinationID string + topic string + eligibleForRetry bool + eventTime time.Time + metadataStr string + dataStr string + deliveryID string + deliveryEventID string + status string + deliveryTime time.Time + code string + responseDataStr string + ) + + err = rows.Scan( + &eventID, + &tenantID, + &destinationID, + &topic, + &eligibleForRetry, + &eventTime, + &metadataStr, + &dataStr, + &deliveryID, + &deliveryEventID, + &status, + &deliveryTime, + &code, + &responseDataStr, + ) + if err != nil { + return nil, fmt.Errorf("scan failed: %w", err) + } + + // Parse JSON fields + var metadata map[string]string + var data map[string]interface{} + var responseData map[string]interface{} + + if metadataStr != "" { + if err := json.Unmarshal([]byte(metadataStr), &metadata); err != nil { + return nil, fmt.Errorf("failed to unmarshal metadata: %w", err) + } + } + if dataStr != "" { + if err := json.Unmarshal([]byte(dataStr), &data); err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %w", err) + } + } + if responseDataStr != "" { + if err := json.Unmarshal([]byte(responseDataStr), &responseData); err != nil { + return nil, fmt.Errorf("failed to unmarshal response_data: %w", err) + } + } + + return &models.DeliveryEvent{ + ID: deliveryEventID, + DestinationID: destinationID, + Event: models.Event{ + ID: eventID, + TenantID: tenantID, + DestinationID: destinationID, + Topic: topic, + EligibleForRetry: eligibleForRetry, + Time: eventTime, + Data: data, + Metadata: metadata, + }, + Delivery: &models.Delivery{ + ID: deliveryID, + EventID: eventID, + DestinationID: destinationID, + Status: status, + Time: deliveryTime, + Code: code, + ResponseData: responseData, + }, + }, nil +} + func (s *logStoreImpl) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents []*models.DeliveryEvent) error { if len(deliveryEvents) == 0 { return nil diff --git a/internal/logstore/driver/driver.go b/internal/logstore/driver/driver.go index 1eebd9d4..3418a1c4 100644 --- a/internal/logstore/driver/driver.go +++ b/internal/logstore/driver/driver.go @@ -15,6 +15,7 @@ var ErrInvalidCursor = errors.New("invalid cursor") type LogStore interface { ListDeliveryEvent(context.Context, ListDeliveryEventRequest) (ListDeliveryEventResponse, error) RetrieveEvent(ctx context.Context, request RetrieveEventRequest) (*models.Event, error) + RetrieveDeliveryEvent(ctx context.Context, request RetrieveDeliveryEventRequest) (*models.DeliveryEvent, error) InsertManyDeliveryEvent(context.Context, []*models.DeliveryEvent) error } @@ -46,3 +47,8 @@ type RetrieveEventRequest struct { EventID string // required DestinationID string // optional - if provided, scopes to that destination } + +type RetrieveDeliveryEventRequest struct { + TenantID string // required + DeliveryID string // required +} diff --git a/internal/logstore/drivertest/drivertest.go b/internal/logstore/drivertest/drivertest.go index 3bcecb14..e81d9ee0 100644 --- a/internal/logstore/drivertest/drivertest.go +++ b/internal/logstore/drivertest/drivertest.go @@ -42,6 +42,9 @@ func RunConformanceTests(t *testing.T, newHarness HarnessMaker) { t.Run("TestRetrieveEvent", func(t *testing.T) { testRetrieveEvent(t, newHarness) }) + t.Run("TestRetrieveDeliveryEvent", func(t *testing.T) { + testRetrieveDeliveryEvent(t, newHarness) + }) t.Run("TestTenantIsolation", func(t *testing.T) { testTenantIsolation(t, newHarness) }) @@ -698,6 +701,169 @@ func testRetrieveEvent(t *testing.T, newHarness HarnessMaker) { }) } +// testRetrieveDeliveryEvent tests the RetrieveDeliveryEvent method +func testRetrieveDeliveryEvent(t *testing.T, newHarness HarnessMaker) { + t.Helper() + + ctx := context.Background() + h, err := newHarness(ctx, t) + require.NoError(t, err) + t.Cleanup(h.Close) + + logStore, err := h.MakeDriver(ctx) + require.NoError(t, err) + + tenantID := idgen.String() + destinationID := idgen.Destination() + eventID := idgen.Event() + deliveryID := idgen.Delivery() + eventTime := time.Now().Truncate(time.Millisecond) + deliveryTime := eventTime.Add(100 * time.Millisecond) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("order.created"), + testutil.EventFactory.WithTime(eventTime), + testutil.EventFactory.WithEligibleForRetry(true), + testutil.EventFactory.WithMetadata(map[string]string{ + "source": "api", + }), + testutil.EventFactory.WithData(map[string]interface{}{ + "order_id": "ord_456", + "amount": 99.99, + }), + ) + + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(deliveryID), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("success"), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + delivery.Code = "200" + delivery.ResponseData = map[string]interface{}{ + "latency_ms": 42, + } + + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, deliveryID), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{de})) + require.NoError(t, h.FlushWrites(ctx)) + + t.Run("retrieve existing delivery with all fields", func(t *testing.T) { + retrieved, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ + TenantID: tenantID, + DeliveryID: deliveryID, + }) + require.NoError(t, err) + require.NotNil(t, retrieved) + + // Verify delivery event ID + assert.Equal(t, de.ID, retrieved.ID) + assert.Equal(t, destinationID, retrieved.DestinationID) + + // Verify event fields + assert.Equal(t, eventID, retrieved.Event.ID) + assert.Equal(t, tenantID, retrieved.Event.TenantID) + assert.Equal(t, destinationID, retrieved.Event.DestinationID) + assert.Equal(t, "order.created", retrieved.Event.Topic) + assert.Equal(t, true, retrieved.Event.EligibleForRetry) + assert.WithinDuration(t, eventTime, retrieved.Event.Time, time.Second) + assert.Equal(t, "api", retrieved.Event.Metadata["source"]) + assert.Equal(t, "ord_456", retrieved.Event.Data["order_id"]) + + // Verify delivery fields + require.NotNil(t, retrieved.Delivery) + assert.Equal(t, deliveryID, retrieved.Delivery.ID) + assert.Equal(t, eventID, retrieved.Delivery.EventID) + assert.Equal(t, destinationID, retrieved.Delivery.DestinationID) + assert.Equal(t, "success", retrieved.Delivery.Status) + assert.WithinDuration(t, deliveryTime, retrieved.Delivery.Time, time.Second) + assert.Equal(t, "200", retrieved.Delivery.Code) + // Note: JSON unmarshaling converts integers to float64, but in-memory stores keep them as int + latencyMs := retrieved.Delivery.ResponseData["latency_ms"] + switch v := latencyMs.(type) { + case int: + assert.Equal(t, 42, v) + case float64: + assert.Equal(t, float64(42), v) + default: + t.Errorf("unexpected type for latency_ms: %T", latencyMs) + } + }) + + t.Run("retrieve non-existent delivery", func(t *testing.T) { + retrieved, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ + TenantID: tenantID, + DeliveryID: "non-existent", + }) + require.NoError(t, err) + assert.Nil(t, retrieved) + }) + + t.Run("retrieve with wrong tenant", func(t *testing.T) { + retrieved, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ + TenantID: "wrong-tenant", + DeliveryID: deliveryID, + }) + require.NoError(t, err) + assert.Nil(t, retrieved, "should not return delivery for wrong tenant") + }) + + t.Run("retrieve multiple deliveries for same event", func(t *testing.T) { + // Insert another delivery for the same event (simulating a retry) + secondDeliveryID := idgen.Delivery() + secondDeliveryTime := deliveryTime.Add(time.Second) + + secondDelivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(secondDeliveryID), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("failed"), + testutil.DeliveryFactory.WithTime(secondDeliveryTime), + ) + secondDelivery.Code = "500" + + secondDE := &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, secondDeliveryID), + DestinationID: destinationID, + Event: *event, + Delivery: secondDelivery, + } + + require.NoError(t, logStore.InsertManyDeliveryEvent(ctx, []*models.DeliveryEvent{secondDE})) + require.NoError(t, h.FlushWrites(ctx)) + + // Retrieve first delivery - should get first delivery + first, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ + TenantID: tenantID, + DeliveryID: deliveryID, + }) + require.NoError(t, err) + require.NotNil(t, first) + assert.Equal(t, deliveryID, first.Delivery.ID) + assert.Equal(t, "success", first.Delivery.Status) + + // Retrieve second delivery - should get second delivery + second, err := logStore.RetrieveDeliveryEvent(ctx, driver.RetrieveDeliveryEventRequest{ + TenantID: tenantID, + DeliveryID: secondDeliveryID, + }) + require.NoError(t, err) + require.NotNil(t, second) + assert.Equal(t, secondDeliveryID, second.Delivery.ID) + assert.Equal(t, "failed", second.Delivery.Status) + }) +} + // testTenantIsolation ensures data from one tenant cannot be accessed by another func testTenantIsolation(t *testing.T, newHarness HarnessMaker) { t.Helper() diff --git a/internal/logstore/logstore.go b/internal/logstore/logstore.go index cd4b5ace..e9008ed6 100644 --- a/internal/logstore/logstore.go +++ b/internal/logstore/logstore.go @@ -15,10 +15,12 @@ import ( type ListDeliveryEventRequest = driver.ListDeliveryEventRequest type ListDeliveryEventResponse = driver.ListDeliveryEventResponse type RetrieveEventRequest = driver.RetrieveEventRequest +type RetrieveDeliveryEventRequest = driver.RetrieveDeliveryEventRequest type LogStore interface { ListDeliveryEvent(context.Context, ListDeliveryEventRequest) (ListDeliveryEventResponse, error) RetrieveEvent(ctx context.Context, request RetrieveEventRequest) (*models.Event, error) + RetrieveDeliveryEvent(ctx context.Context, request RetrieveDeliveryEventRequest) (*models.DeliveryEvent, error) InsertManyDeliveryEvent(context.Context, []*models.DeliveryEvent) error } diff --git a/internal/logstore/memlogstore/memlogstore.go b/internal/logstore/memlogstore/memlogstore.go index 5afd521e..778f6117 100644 --- a/internal/logstore/memlogstore/memlogstore.go +++ b/internal/logstore/memlogstore/memlogstore.go @@ -224,6 +224,19 @@ func (s *memLogStore) RetrieveEvent(ctx context.Context, req driver.RetrieveEven return nil, nil } +// RetrieveDeliveryEvent retrieves a single delivery event by delivery ID. +func (s *memLogStore) RetrieveDeliveryEvent(ctx context.Context, req driver.RetrieveDeliveryEventRequest) (*models.DeliveryEvent, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + for _, de := range s.deliveryEvents { + if de.Event.TenantID == req.TenantID && de.Delivery != nil && de.Delivery.ID == req.DeliveryID { + return copyDeliveryEvent(de), nil + } + } + return nil, nil +} + func (s *memLogStore) matchesFilter(de *models.DeliveryEvent, req driver.ListDeliveryEventRequest) bool { // Tenant filter (required) if de.Event.TenantID != req.TenantID { diff --git a/internal/logstore/noop.go b/internal/logstore/noop.go index af084c62..da944e0d 100644 --- a/internal/logstore/noop.go +++ b/internal/logstore/noop.go @@ -23,6 +23,10 @@ func (l *noopLogStore) RetrieveEvent(ctx context.Context, request driver.Retriev return nil, nil } +func (l *noopLogStore) RetrieveDeliveryEvent(ctx context.Context, request driver.RetrieveDeliveryEventRequest) (*models.DeliveryEvent, error) { + return nil, nil +} + func (l *noopLogStore) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents []*models.DeliveryEvent) error { return nil } diff --git a/internal/logstore/pglogstore/pglogstore.go b/internal/logstore/pglogstore/pglogstore.go index fe11f2db..5e22ea7a 100644 --- a/internal/logstore/pglogstore/pglogstore.go +++ b/internal/logstore/pglogstore/pglogstore.go @@ -397,6 +397,94 @@ func (s *logStore) RetrieveEvent(ctx context.Context, req driver.RetrieveEventRe return event, nil } +// RetrieveDeliveryEvent retrieves a single delivery event by delivery ID. +func (s *logStore) RetrieveDeliveryEvent(ctx context.Context, req driver.RetrieveDeliveryEventRequest) (*models.DeliveryEvent, error) { + query := ` + SELECT + idx.event_id, + idx.delivery_id, + idx.destination_id, + idx.event_time, + idx.delivery_time, + idx.topic, + idx.status, + e.tenant_id, + e.eligible_for_retry, + e.data, + e.metadata, + d.code, + d.response_data + FROM event_delivery_index idx + JOIN events e ON e.id = idx.event_id AND e.time = idx.event_time + JOIN deliveries d ON d.id = idx.delivery_id AND d.time = idx.delivery_time + WHERE idx.tenant_id = $1 AND idx.delivery_id = $2 + LIMIT 1` + + row := s.db.QueryRow(ctx, query, req.TenantID, req.DeliveryID) + + var ( + eventID string + deliveryID string + destinationID string + eventTime time.Time + deliveryTime time.Time + topic string + status string + tenantID string + eligibleForRetry bool + data map[string]interface{} + metadata map[string]string + code string + responseData map[string]interface{} + ) + + err := row.Scan( + &eventID, + &deliveryID, + &destinationID, + &eventTime, + &deliveryTime, + &topic, + &status, + &tenantID, + &eligibleForRetry, + &data, + &metadata, + &code, + &responseData, + ) + if err == pgx.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("scan failed: %w", err) + } + + return &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, deliveryID), + DestinationID: destinationID, + Event: models.Event{ + ID: eventID, + TenantID: tenantID, + DestinationID: destinationID, + Topic: topic, + EligibleForRetry: eligibleForRetry, + Time: eventTime, + Data: data, + Metadata: metadata, + }, + Delivery: &models.Delivery{ + ID: deliveryID, + EventID: eventID, + DestinationID: destinationID, + Status: status, + Time: deliveryTime, + Code: code, + ResponseData: responseData, + }, + }, nil +} + func (s *logStore) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents []*models.DeliveryEvent) error { if len(deliveryEvents) == 0 { return nil diff --git a/internal/migrator/migrations/clickhouse/000001_init.up.sql b/internal/migrator/migrations/clickhouse/000001_init.up.sql index f12b7f78..45712966 100644 --- a/internal/migrator/migrations/clickhouse/000001_init.up.sql +++ b/internal/migrator/migrations/clickhouse/000001_init.up.sql @@ -23,6 +23,7 @@ CREATE TABLE IF NOT EXISTS event_log ( -- Indexes for filtering (bloom filters help skip granules) INDEX idx_event_id event_id TYPE bloom_filter GRANULARITY 4, + INDEX idx_delivery_id delivery_id TYPE bloom_filter GRANULARITY 4, INDEX idx_topic topic TYPE bloom_filter GRANULARITY 4, INDEX idx_status status TYPE set(100) GRANULARITY 4 ) ENGINE = ReplacingMergeTree From 1fac7235519fd8bee151f1ea7a8deb7218787b20 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Mon, 8 Dec 2025 18:45:21 +0700 Subject: [PATCH 14/19] feat: log api & retry implementation --- internal/apirouter/log_handlers.go | 265 +++++++++------ internal/apirouter/log_handlers_test.go | 386 ++++++++++++++++++++++ internal/apirouter/retry_handlers.go | 74 +++-- internal/apirouter/retry_handlers_test.go | 150 +++++++++ internal/apirouter/router.go | 45 +-- internal/apirouter/router_test.go | 23 +- internal/logstore/logstore.go | 6 + internal/logstore/noop.go | 32 -- 8 files changed, 788 insertions(+), 193 deletions(-) create mode 100644 internal/apirouter/log_handlers_test.go create mode 100644 internal/apirouter/retry_handlers_test.go delete mode 100644 internal/logstore/noop.go diff --git a/internal/apirouter/log_handlers.go b/internal/apirouter/log_handlers.go index e827021b..adf39018 100644 --- a/internal/apirouter/log_handlers.go +++ b/internal/apirouter/log_handlers.go @@ -26,20 +26,126 @@ func NewLogHandlers( } } -func (h *LogHandlers) ListEvent(c *gin.Context) { - h.listEvent(c, c.QueryArray("destination_id")) +// ExpandOptions represents which fields to expand in the response +type ExpandOptions struct { + Event bool + EventData bool + Destination bool } -func (h *LogHandlers) ListEventByDestination(c *gin.Context) { - h.listEvent(c, []string{c.Param("destinationID")}) +func parseExpandOptions(c *gin.Context) ExpandOptions { + opts := ExpandOptions{} + for _, e := range c.QueryArray("expand") { + switch e { + case "event": + opts.Event = true + case "event.data": + opts.Event = true + opts.EventData = true + case "destination": + opts.Destination = true + } + } + return opts +} + +// API Response types + +// APIDelivery is the API response for a delivery +type APIDelivery struct { + ID string `json:"id"` + Status string `json:"status"` + DeliveredAt time.Time `json:"delivered_at"` + Code string `json:"code,omitempty"` + ResponseData map[string]interface{} `json:"response_data,omitempty"` + Attempt int `json:"attempt"` + + // Expandable fields - string (ID) or object depending on expand + Event interface{} `json:"event"` + Destination string `json:"destination"` +} + +// APIEventSummary is the event object when expand=event (without data) +type APIEventSummary struct { + ID string `json:"id"` + Topic string `json:"topic"` + Time time.Time `json:"time"` + EligibleForRetry bool `json:"eligible_for_retry"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// APIEventFull is the event object when expand=event.data +type APIEventFull struct { + ID string `json:"id"` + Topic string `json:"topic"` + Time time.Time `json:"time"` + EligibleForRetry bool `json:"eligible_for_retry"` + Metadata map[string]string `json:"metadata,omitempty"` + Data map[string]interface{} `json:"data,omitempty"` +} + +// ListDeliveriesResponse is the response for ListDeliveries +type ListDeliveriesResponse struct { + Data []APIDelivery `json:"data"` + Next string `json:"next,omitempty"` + Prev string `json:"prev,omitempty"` +} + +// toAPIDelivery converts a DeliveryEvent to APIDelivery with expand options +func toAPIDelivery(de *models.DeliveryEvent, opts ExpandOptions) APIDelivery { + api := APIDelivery{ + ID: de.Delivery.ID, + Attempt: de.Attempt, + Destination: de.DestinationID, + } + + // Set delivery fields if delivery exists + if de.Delivery != nil { + api.Status = de.Delivery.Status + api.DeliveredAt = de.Delivery.Time + api.Code = de.Delivery.Code + api.ResponseData = de.Delivery.ResponseData + } + + // Handle event expansion + if opts.EventData { + api.Event = APIEventFull{ + ID: de.Event.ID, + Topic: de.Event.Topic, + Time: de.Event.Time, + EligibleForRetry: de.Event.EligibleForRetry, + Metadata: de.Event.Metadata, + Data: de.Event.Data, + } + } else if opts.Event { + api.Event = APIEventSummary{ + ID: de.Event.ID, + Topic: de.Event.Topic, + Time: de.Event.Time, + EligibleForRetry: de.Event.EligibleForRetry, + Metadata: de.Event.Metadata, + } + } else { + api.Event = de.Event.ID + } + + // TODO: Handle destination expansion + // This would require injecting EntityStore into LogHandlers and batch-fetching + // destinations by ID. Consider if this is needed - clients can fetch destination + // details separately via GET /destinations/:id if needed. + + return api } -func (h *LogHandlers) listEvent(c *gin.Context, destinationIDs []string) { +// ListDeliveries handles GET /:tenantID/deliveries +// Query params: event_id, destination_id, status, topic[], start, end, limit, next, prev, expand[] +func (h *LogHandlers) ListDeliveries(c *gin.Context) { tenant := mustTenantFromContext(c) if tenant == nil { return } + // Parse time filters var start, end *time.Time if startStr := c.Query("start"); startStr != "" { t, err := time.Parse(time.RFC3339, startStr) @@ -70,137 +176,102 @@ func (h *LogHandlers) listEvent(c *gin.Context, destinationIDs []string) { end = &t } - limitStr := c.Query("limit") - limit, err := strconv.Atoi(limitStr) - if err != nil { - limit = 100 + // Parse limit + limit := 100 + if limitStr := c.Query("limit"); limitStr != "" { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 { + limit = l + } } - response, err := h.logStore.ListEvent(c.Request.Context(), logstore.ListEventRequest{ - Next: c.Query("next"), - Prev: c.Query("prev"), - Limit: limit, - Start: start, - End: end, + + // Parse destination_id (single value for now) + var destinationIDs []string + if destID := c.Query("destination_id"); destID != "" { + destinationIDs = []string{destID} + } + + // Build request + req := logstore.ListDeliveryEventRequest{ TenantID: tenant.ID, + EventID: c.Query("event_id"), DestinationIDs: destinationIDs, - Topics: c.QueryArray("topic"), Status: c.Query("status"), - }) + Topics: c.QueryArray("topic"), + DeliveryStart: start, + DeliveryEnd: end, + Limit: limit, + Next: c.Query("next"), + Prev: c.Query("prev"), + } + + // Call logstore + response, err := h.logStore.ListDeliveryEvent(c.Request.Context(), req) if err != nil { AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return } - if len(response.Data) == 0 { - // Return an empty array instead of null - c.JSON(http.StatusOK, gin.H{ - "data": []models.Event{}, - "next": "", - "prev": "", - "count": 0, - }) - return + + // Parse expand options + expandOpts := parseExpandOptions(c) + + // Transform to API response + apiDeliveries := make([]APIDelivery, len(response.Data)) + for i, de := range response.Data { + apiDeliveries[i] = toAPIDelivery(de, expandOpts) } - c.JSON(http.StatusOK, gin.H{ - "data": response.Data, - "next": response.Next, - "prev": response.Prev, - "count": response.Count, + + c.JSON(http.StatusOK, ListDeliveriesResponse{ + Data: apiDeliveries, + Next: response.Next, + Prev: response.Prev, }) } +// RetrieveEvent handles GET /:tenantID/events/:eventID func (h *LogHandlers) RetrieveEvent(c *gin.Context) { tenant := mustTenantFromContext(c) if tenant == nil { return } eventID := c.Param("eventID") - event, err := h.logStore.RetrieveEvent(c.Request.Context(), tenant.ID, eventID) + event, err := h.logStore.RetrieveEvent(c.Request.Context(), logstore.RetrieveEventRequest{ + TenantID: tenant.ID, + EventID: eventID, + }) if err != nil { AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return } if event == nil { - c.Status(http.StatusNotFound) + AbortWithError(c, http.StatusNotFound, NewErrNotFound("event")) return } c.JSON(http.StatusOK, event) } -func (h *LogHandlers) RetrieveEventByDestination(c *gin.Context) { +// RetrieveDelivery handles GET /:tenantID/deliveries/:deliveryID +func (h *LogHandlers) RetrieveDelivery(c *gin.Context) { tenant := mustTenantFromContext(c) if tenant == nil { return } - destinationID := c.Param("destinationID") - eventID := c.Param("eventID") - event, err := h.logStore.RetrieveEventByDestination(c.Request.Context(), tenant.ID, destinationID, eventID) - if err != nil { - AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) - return - } - if event == nil { - c.Status(http.StatusNotFound) - return - } - c.JSON(http.StatusOK, event) -} - -type DeliveryResponse struct { - ID string `json:"id"` - DeliveredAt string `json:"delivered_at"` - Status string `json:"status"` - Code string `json:"code"` - ResponseData map[string]interface{} `json:"response_data"` -} + deliveryID := c.Param("deliveryID") -func (h *LogHandlers) ListDeliveryByEvent(c *gin.Context) { - event := h.mustEventWithTenant(c, c.Param("eventID")) - if event == nil { - return - } - deliveries, err := h.logStore.ListDelivery(c.Request.Context(), logstore.ListDeliveryRequest{ - EventID: event.ID, - DestinationID: c.Query("destination_id"), + deliveryEvent, err := h.logStore.RetrieveDeliveryEvent(c.Request.Context(), logstore.RetrieveDeliveryEventRequest{ + TenantID: tenant.ID, + DeliveryID: deliveryID, }) if err != nil { AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return } - if len(deliveries) == 0 { - // Return an empty array instead of null - c.JSON(http.StatusOK, []DeliveryResponse{}) + if deliveryEvent == nil { + AbortWithError(c, http.StatusNotFound, NewErrNotFound("delivery")) return } - deliveryData := make([]DeliveryResponse, len(deliveries)) - for i, delivery := range deliveries { - deliveryData[i] = DeliveryResponse{ - ID: delivery.ID, - DeliveredAt: delivery.Time.UTC().Format(time.RFC3339), - Status: delivery.Status, - Code: delivery.Code, - ResponseData: delivery.ResponseData, - } - } - c.JSON(http.StatusOK, deliveryData) -} -func (h *LogHandlers) mustEventWithTenant(c *gin.Context, eventID string) *models.Event { - tenant := mustTenantFromContext(c) - if tenant == nil { - return nil - } - event, err := h.logStore.RetrieveEvent(c.Request.Context(), tenant.ID, eventID) - if err != nil { - AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) - return nil - } - if event == nil { - c.Status(http.StatusNotFound) - return nil - } - if event.TenantID != tenant.ID { - c.Status(http.StatusForbidden) - return nil - } - return event + // Parse expand options + expandOpts := parseExpandOptions(c) + + c.JSON(http.StatusOK, toAPIDelivery(deliveryEvent, expandOpts)) } diff --git a/internal/apirouter/log_handlers_test.go b/internal/apirouter/log_handlers_test.go new file mode 100644 index 00000000..f0be8e66 --- /dev/null +++ b/internal/apirouter/log_handlers_test.go @@ -0,0 +1,386 @@ +package apirouter_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/hookdeck/outpost/internal/idgen" + "github.com/hookdeck/outpost/internal/models" + "github.com/hookdeck/outpost/internal/util/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestListDeliveries(t *testing.T) { + t.Parallel() + + result := setupTestRouterFull(t, "", "") + + // Create a tenant + tenantID := idgen.String() + destinationID := idgen.Destination() + require.NoError(t, result.entityStore.UpsertTenant(context.Background(), models.Tenant{ + ID: tenantID, + CreatedAt: time.Now(), + })) + require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ + ID: destinationID, + TenantID: tenantID, + Type: "webhook", + Topics: []string{"*"}, + CreatedAt: time.Now(), + })) + + t.Run("should return empty list when no deliveries", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/deliveries", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + + data := response["data"].([]interface{}) + assert.Len(t, data, 0) + }) + + t.Run("should list deliveries", func(t *testing.T) { + // Seed delivery events + eventID := idgen.Event() + deliveryID := idgen.Delivery() + eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) + deliveryTime := eventTime.Add(100 * time.Millisecond) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("user.created"), + testutil.EventFactory.WithTime(eventTime), + ) + + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(deliveryID), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("success"), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, deliveryID), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + + require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/deliveries", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + + data := response["data"].([]interface{}) + assert.Len(t, data, 1) + + firstDelivery := data[0].(map[string]interface{}) + assert.Equal(t, deliveryID, firstDelivery["id"]) + assert.Equal(t, "success", firstDelivery["status"]) + assert.Equal(t, eventID, firstDelivery["event"]) // Not expanded + assert.Equal(t, destinationID, firstDelivery["destination"]) + }) + + t.Run("should expand event when expand=event", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/deliveries?expand=event", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + + data := response["data"].([]interface{}) + require.Len(t, data, 1) + + firstDelivery := data[0].(map[string]interface{}) + event := firstDelivery["event"].(map[string]interface{}) + assert.NotNil(t, event["id"]) + assert.Equal(t, "user.created", event["topic"]) + // data should not be present without expand=event.data + assert.Nil(t, event["data"]) + }) + + t.Run("should expand event.data when expand=event.data", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/deliveries?expand=event.data", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + + data := response["data"].([]interface{}) + require.Len(t, data, 1) + + firstDelivery := data[0].(map[string]interface{}) + event := firstDelivery["event"].(map[string]interface{}) + assert.NotNil(t, event["id"]) + assert.NotNil(t, event["data"]) // data should be present + }) + + t.Run("should filter by destination_id", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/deliveries?destination_id="+destinationID, nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + + data := response["data"].([]interface{}) + assert.Len(t, data, 1) + }) + + t.Run("should filter by non-existent destination_id", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/deliveries?destination_id=nonexistent", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + + data := response["data"].([]interface{}) + assert.Len(t, data, 0) + }) + + t.Run("should return 404 for non-existent tenant", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/nonexistent/deliveries", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + }) +} + +func TestRetrieveDelivery(t *testing.T) { + t.Parallel() + + result := setupTestRouterFull(t, "", "") + + // Create a tenant + tenantID := idgen.String() + destinationID := idgen.Destination() + require.NoError(t, result.entityStore.UpsertTenant(context.Background(), models.Tenant{ + ID: tenantID, + CreatedAt: time.Now(), + })) + require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ + ID: destinationID, + TenantID: tenantID, + Type: "webhook", + Topics: []string{"*"}, + CreatedAt: time.Now(), + })) + + // Seed a delivery event + eventID := idgen.Event() + deliveryID := idgen.Delivery() + eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) + deliveryTime := eventTime.Add(100 * time.Millisecond) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("order.created"), + testutil.EventFactory.WithTime(eventTime), + ) + + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(deliveryID), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("failed"), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, deliveryID), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + + require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + + t.Run("should retrieve delivery by ID", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/deliveries/"+deliveryID, nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + + assert.Equal(t, deliveryID, response["id"]) + assert.Equal(t, "failed", response["status"]) + assert.Equal(t, eventID, response["event"]) // Not expanded + assert.Equal(t, destinationID, response["destination"]) + }) + + t.Run("should expand event when expand=event", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/deliveries/"+deliveryID+"?expand=event", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + + event := response["event"].(map[string]interface{}) + assert.Equal(t, eventID, event["id"]) + assert.Equal(t, "order.created", event["topic"]) + // data should not be present without expand=event.data + assert.Nil(t, event["data"]) + }) + + t.Run("should expand event.data when expand=event.data", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/deliveries/"+deliveryID+"?expand=event.data", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + + event := response["event"].(map[string]interface{}) + assert.Equal(t, eventID, event["id"]) + assert.NotNil(t, event["data"]) // data should be present + }) + + t.Run("should return 404 for non-existent delivery", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/deliveries/nonexistent", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + }) + + t.Run("should return 404 for non-existent tenant", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/nonexistent/deliveries/"+deliveryID, nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + }) +} + +func TestRetrieveEvent(t *testing.T) { + t.Parallel() + + result := setupTestRouterFull(t, "", "") + + // Create a tenant + tenantID := idgen.String() + destinationID := idgen.Destination() + require.NoError(t, result.entityStore.UpsertTenant(context.Background(), models.Tenant{ + ID: tenantID, + CreatedAt: time.Now(), + })) + require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ + ID: destinationID, + TenantID: tenantID, + Type: "webhook", + Topics: []string{"*"}, + CreatedAt: time.Now(), + })) + + // Seed a delivery event + eventID := idgen.Event() + deliveryID := idgen.Delivery() + eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) + deliveryTime := eventTime.Add(100 * time.Millisecond) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("payment.processed"), + testutil.EventFactory.WithTime(eventTime), + testutil.EventFactory.WithData(map[string]interface{}{ + "amount": 100.50, + }), + testutil.EventFactory.WithMetadata(map[string]string{ + "source": "stripe", + }), + ) + + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(deliveryID), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("success"), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, deliveryID), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + + require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + + t.Run("should retrieve event by ID", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/events/"+eventID, nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + + assert.Equal(t, eventID, response["id"]) + assert.Equal(t, tenantID, response["tenant_id"]) + assert.Equal(t, "payment.processed", response["topic"]) + assert.Equal(t, "stripe", response["metadata"].(map[string]interface{})["source"]) + assert.Equal(t, 100.50, response["data"].(map[string]interface{})["amount"]) + }) + + t.Run("should return 404 for non-existent event", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/events/nonexistent", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + }) + + t.Run("should return 404 for non-existent tenant", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/nonexistent/events/"+eventID, nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + }) +} diff --git a/internal/apirouter/retry_handlers.go b/internal/apirouter/retry_handlers.go index ca357a86..e9fe821c 100644 --- a/internal/apirouter/retry_handlers.go +++ b/internal/apirouter/retry_handlers.go @@ -1,7 +1,6 @@ package apirouter import ( - "errors" "net/http" "github.com/gin-gonic/gin" @@ -12,10 +11,6 @@ import ( "go.uber.org/zap" ) -var ( - ErrDestinationDisabled = errors.New("destination is disabled") -) - type RetryHandlers struct { logger *logging.Logger entityStore models.EntityStore @@ -23,7 +18,12 @@ type RetryHandlers struct { deliveryMQ *deliverymq.DeliveryMQ } -func NewRetryHandlers(logger *logging.Logger, entityStore models.EntityStore, logStore logstore.LogStore, deliveryMQ *deliverymq.DeliveryMQ) *RetryHandlers { +func NewRetryHandlers( + logger *logging.Logger, + entityStore models.EntityStore, + logStore logstore.LogStore, + deliveryMQ *deliverymq.DeliveryMQ, +) *RetryHandlers { return &RetryHandlers{ logger: logger, entityStore: entityStore, @@ -32,50 +32,64 @@ func NewRetryHandlers(logger *logging.Logger, entityStore models.EntityStore, lo } } -func (h *RetryHandlers) Retry(c *gin.Context) { - tenantID := c.Param("tenantID") - destinationID := c.Param("destinationID") - eventID := c.Param("eventID") +// RetryDelivery handles POST /:tenantID/deliveries/:deliveryID/retry +// Constraints: +// - Only the latest delivery for an event+destination pair can be retried +// - Destination must exist and be enabled +func (h *RetryHandlers) RetryDelivery(c *gin.Context) { + tenant := mustTenantFromContext(c) + if tenant == nil { + return + } + deliveryID := c.Param("deliveryID") - // 1. Retrieve destination & event data - destination, err := h.entityStore.RetrieveDestination(c, tenantID, destinationID) + // 1. Look up delivery by ID + deliveryEvent, err := h.logStore.RetrieveDeliveryEvent(c.Request.Context(), logstore.RetrieveDeliveryEventRequest{ + TenantID: tenant.ID, + DeliveryID: deliveryID, + }) if err != nil { AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return } - if destination == nil { - AbortWithError(c, http.StatusNotFound, NewErrNotFound("destination")) - return - } - if destination.DisabledAt != nil { - AbortWithError(c, http.StatusBadRequest, NewErrBadRequest(ErrDestinationDisabled)) + if deliveryEvent == nil { + AbortWithError(c, http.StatusNotFound, NewErrNotFound("delivery")) return } - event, err := h.logStore.RetrieveEvent(c, logstore.RetrieveEventRequest{ - TenantID: tenantID, - EventID: eventID, - }) + // 2. Check destination exists and is enabled + destination, err := h.entityStore.RetrieveDestination(c.Request.Context(), tenant.ID, deliveryEvent.DestinationID) if err != nil { AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return } - if event == nil { - AbortWithError(c, http.StatusNotFound, NewErrNotFound("event")) + if destination == nil { + AbortWithError(c, http.StatusNotFound, NewErrNotFound("destination")) + return + } + if destination.DisabledAt != nil { + AbortWithError(c, http.StatusBadRequest, ErrorResponse{ + Code: http.StatusBadRequest, + Message: "Destination is disabled", + Data: map[string]string{ + "error": "destination_disabled", + }, + }) return } - // 2. Initiate redelivery - deliveryEvent := models.NewManualDeliveryEvent(*event, destination.ID) + // 3. Create and publish retry delivery event + retryDeliveryEvent := models.NewManualDeliveryEvent(deliveryEvent.Event, deliveryEvent.DestinationID) - if err := h.deliveryMQ.Publish(c, deliveryEvent); err != nil { + if err := h.deliveryMQ.Publish(c.Request.Context(), retryDeliveryEvent); err != nil { AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) return } - h.logger.Ctx(c).Audit("manual retry initiated", - zap.String("event_id", event.ID), - zap.String("destination_id", destination.ID)) + h.logger.Ctx(c.Request.Context()).Audit("manual retry initiated", + zap.String("delivery_id", deliveryID), + zap.String("event_id", deliveryEvent.Event.ID), + zap.String("destination_id", deliveryEvent.DestinationID)) c.JSON(http.StatusAccepted, gin.H{ "success": true, diff --git a/internal/apirouter/retry_handlers_test.go b/internal/apirouter/retry_handlers_test.go new file mode 100644 index 00000000..0f75fa76 --- /dev/null +++ b/internal/apirouter/retry_handlers_test.go @@ -0,0 +1,150 @@ +package apirouter_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/hookdeck/outpost/internal/idgen" + "github.com/hookdeck/outpost/internal/models" + "github.com/hookdeck/outpost/internal/util/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRetryDelivery(t *testing.T) { + t.Parallel() + + result := setupTestRouterFull(t, "", "") + + // Create a tenant and destination + tenantID := idgen.String() + destinationID := idgen.Destination() + require.NoError(t, result.entityStore.UpsertTenant(context.Background(), models.Tenant{ + ID: tenantID, + CreatedAt: time.Now(), + })) + require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ + ID: destinationID, + TenantID: tenantID, + Type: "webhook", + Topics: []string{"*"}, + CreatedAt: time.Now(), + })) + + // Seed a delivery event + eventID := idgen.Event() + deliveryID := idgen.Delivery() + eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) + deliveryTime := eventTime.Add(100 * time.Millisecond) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("order.created"), + testutil.EventFactory.WithTime(eventTime), + ) + + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(deliveryID), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("failed"), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, deliveryID), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + + require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + + t.Run("should retry delivery successfully", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", baseAPIPath+"/"+tenantID+"/deliveries/"+deliveryID+"/retry", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusAccepted, w.Code) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + assert.Equal(t, true, response["success"]) + }) + + t.Run("should return 404 for non-existent delivery", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", baseAPIPath+"/"+tenantID+"/deliveries/nonexistent/retry", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + }) + + t.Run("should return 404 for non-existent tenant", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", baseAPIPath+"/nonexistent/deliveries/"+deliveryID+"/retry", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + }) + + t.Run("should return 400 when destination is disabled", func(t *testing.T) { + // Create a new destination that's disabled + disabledDestinationID := idgen.Destination() + disabledAt := time.Now() + require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ + ID: disabledDestinationID, + TenantID: tenantID, + Type: "webhook", + Topics: []string{"*"}, + CreatedAt: time.Now(), + DisabledAt: &disabledAt, + })) + + // Create a delivery for the disabled destination + disabledEventID := idgen.Event() + disabledDeliveryID := idgen.Delivery() + + disabledEvent := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(disabledEventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(disabledDestinationID), + testutil.EventFactory.WithTopic("order.created"), + testutil.EventFactory.WithTime(eventTime), + ) + + disabledDelivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(disabledDeliveryID), + testutil.DeliveryFactory.WithEventID(disabledEventID), + testutil.DeliveryFactory.WithDestinationID(disabledDestinationID), + testutil.DeliveryFactory.WithStatus("failed"), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + + disabledDE := &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", disabledEventID, disabledDeliveryID), + DestinationID: disabledDestinationID, + Event: *disabledEvent, + Delivery: disabledDelivery, + } + + require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{disabledDE})) + + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", baseAPIPath+"/"+tenantID+"/deliveries/"+disabledDeliveryID+"/retry", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + assert.Equal(t, "Destination is disabled", response["message"]) + }) +} diff --git a/internal/apirouter/router.go b/internal/apirouter/router.go index 3f96092e..ae4cb8be 100644 --- a/internal/apirouter/router.go +++ b/internal/apirouter/router.go @@ -151,8 +151,8 @@ func NewRouter( tenantHandlers := NewTenantHandlers(logger, telemetry, cfg.JWTSecret, entityStore) destinationHandlers := NewDestinationHandlers(logger, telemetry, entityStore, cfg.Topics, cfg.Registry) publishHandlers := NewPublishHandlers(logger, publishmqEventHandler) - retryHandlers := NewRetryHandlers(logger, entityStore, logStore, deliveryMQ) logHandlers := NewLogHandlers(logger, logStore) + retryHandlers := NewRetryHandlers(logger, entityStore, logStore, deliveryMQ) topicHandlers := NewTopicHandlers(logger, cfg.Topics) // Admin routes @@ -334,22 +334,11 @@ func NewRouter( }, }, - // Event routes - { - Method: http.MethodGet, - Path: "/:tenantID/events", - Handler: logHandlers.ListEvent, - AuthScope: AuthScopeAdminOrTenant, - Mode: RouteModeAlways, - AllowTenantFromJWT: true, - Middlewares: []gin.HandlerFunc{ - RequireTenantMiddleware(entityStore), - }, - }, + // Delivery routes (new API) { Method: http.MethodGet, - Path: "/:tenantID/events/:eventID", - Handler: logHandlers.RetrieveEvent, + Path: "/:tenantID/deliveries", + Handler: logHandlers.ListDeliveries, AuthScope: AuthScopeAdminOrTenant, Mode: RouteModeAlways, AllowTenantFromJWT: true, @@ -359,8 +348,8 @@ func NewRouter( }, { Method: http.MethodGet, - Path: "/:tenantID/events/:eventID/deliveries", - Handler: logHandlers.ListDeliveryByEvent, + Path: "/:tenantID/deliveries/:deliveryID", + Handler: logHandlers.RetrieveDelivery, AuthScope: AuthScopeAdminOrTenant, Mode: RouteModeAlways, AllowTenantFromJWT: true, @@ -369,9 +358,9 @@ func NewRouter( }, }, { - Method: http.MethodGet, - Path: "/:tenantID/destinations/:destinationID/events", - Handler: logHandlers.ListEventByDestination, + Method: http.MethodPost, + Path: "/:tenantID/deliveries/:deliveryID/retry", + Handler: retryHandlers.RetryDelivery, AuthScope: AuthScopeAdminOrTenant, Mode: RouteModeAlways, AllowTenantFromJWT: true, @@ -379,10 +368,12 @@ func NewRouter( RequireTenantMiddleware(entityStore), }, }, + + // Event routes { Method: http.MethodGet, - Path: "/:tenantID/destinations/:destinationID/events/:eventID", - Handler: logHandlers.RetrieveEventByDestination, + Path: "/:tenantID/events/:eventID", + Handler: logHandlers.RetrieveEvent, AuthScope: AuthScopeAdminOrTenant, Mode: RouteModeAlways, AllowTenantFromJWT: true, @@ -390,16 +381,6 @@ func NewRouter( RequireTenantMiddleware(entityStore), }, }, - - // Retry routes - { - Method: http.MethodPost, - Path: "/:tenantID/destinations/:destinationID/events/:eventID/retry", - Handler: retryHandlers.Retry, - AuthScope: AuthScopeAdminOrTenant, - Mode: RouteModeAlways, - AllowTenantFromJWT: true, - }, } // Register all routes to a single router diff --git a/internal/apirouter/router_test.go b/internal/apirouter/router_test.go index fd2511eb..b5f2c54e 100644 --- a/internal/apirouter/router_test.go +++ b/internal/apirouter/router_test.go @@ -28,7 +28,20 @@ import ( const baseAPIPath = "/api/v1" +type testRouterResult struct { + router http.Handler + logger *logging.Logger + redisClient redis.Client + entityStore models.EntityStore + logStore logstore.LogStore +} + func setupTestRouter(t *testing.T, apiKey, jwtSecret string, funcs ...func(t *testing.T) clickhouse.DB) (http.Handler, *logging.Logger, redis.Client) { + result := setupTestRouterFull(t, apiKey, jwtSecret, funcs...) + return result.router, result.logger, result.redisClient +} + +func setupTestRouterFull(t *testing.T, apiKey, jwtSecret string, funcs ...func(t *testing.T) clickhouse.DB) testRouterResult { gin.SetMode(gin.TestMode) logger := testutil.CreateTestLogger(t) redisClient := testutil.CreateTestRedisClient(t) @@ -53,7 +66,13 @@ func setupTestRouter(t *testing.T, apiKey, jwtSecret string, funcs ...func(t *te eventHandler, &telemetry.NoopTelemetry{}, ) - return router, logger, redisClient + return testRouterResult{ + router: router, + logger: logger, + redisClient: redisClient, + entityStore: entityStore, + logStore: logStore, + } } func setupTestLogStore(t *testing.T, funcs ...func(t *testing.T) clickhouse.DB) logstore.LogStore { @@ -62,7 +81,7 @@ func setupTestLogStore(t *testing.T, funcs ...func(t *testing.T) clickhouse.DB) chDB = f(t) } if chDB == nil { - return logstore.NewNoopLogStore() + return logstore.NewMemLogStore() } logStore, err := logstore.NewLogStore(context.Background(), logstore.DriverOpts{ CH: chDB, diff --git a/internal/logstore/logstore.go b/internal/logstore/logstore.go index e9008ed6..e35d4266 100644 --- a/internal/logstore/logstore.go +++ b/internal/logstore/logstore.go @@ -7,6 +7,7 @@ import ( "github.com/hookdeck/outpost/internal/clickhouse" "github.com/hookdeck/outpost/internal/logstore/chlogstore" "github.com/hookdeck/outpost/internal/logstore/driver" + "github.com/hookdeck/outpost/internal/logstore/memlogstore" "github.com/hookdeck/outpost/internal/logstore/pglogstore" "github.com/hookdeck/outpost/internal/models" "github.com/jackc/pgx/v5/pgxpool" @@ -50,6 +51,11 @@ func NewLogStore(ctx context.Context, driverOpts DriverOpts) (LogStore, error) { return nil, errors.New("no driver provided") } +// NewMemLogStore returns an in-memory log store for testing. +func NewMemLogStore() LogStore { + return memlogstore.NewLogStore() +} + type Config struct { ClickHouse *clickhouse.ClickHouseConfig Postgres *string diff --git a/internal/logstore/noop.go b/internal/logstore/noop.go deleted file mode 100644 index da944e0d..00000000 --- a/internal/logstore/noop.go +++ /dev/null @@ -1,32 +0,0 @@ -package logstore - -import ( - "context" - - "github.com/hookdeck/outpost/internal/logstore/driver" - "github.com/hookdeck/outpost/internal/models" -) - -func NewNoopLogStore() LogStore { - return &noopLogStore{} -} - -type noopLogStore struct{} - -var _ LogStore = (*noopLogStore)(nil) - -func (l *noopLogStore) ListDeliveryEvent(ctx context.Context, request driver.ListDeliveryEventRequest) (driver.ListDeliveryEventResponse, error) { - return driver.ListDeliveryEventResponse{}, nil -} - -func (l *noopLogStore) RetrieveEvent(ctx context.Context, request driver.RetrieveEventRequest) (*models.Event, error) { - return nil, nil -} - -func (l *noopLogStore) RetrieveDeliveryEvent(ctx context.Context, request driver.RetrieveDeliveryEventRequest) (*models.DeliveryEvent, error) { - return nil, nil -} - -func (l *noopLogStore) InsertManyDeliveryEvent(ctx context.Context, deliveryEvents []*models.DeliveryEvent) error { - return nil -} From b600fb8b7b5672af201fde1f360baee47b7c1002 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Mon, 8 Dec 2025 19:05:06 +0700 Subject: [PATCH 15/19] chore: backward-compat support --- internal/apirouter/legacy_handlers.go | 260 +++++++++++++++ internal/apirouter/legacy_handlers_test.go | 348 +++++++++++++++++++++ internal/apirouter/router.go | 50 +++ 3 files changed, 658 insertions(+) create mode 100644 internal/apirouter/legacy_handlers.go create mode 100644 internal/apirouter/legacy_handlers_test.go diff --git a/internal/apirouter/legacy_handlers.go b/internal/apirouter/legacy_handlers.go new file mode 100644 index 00000000..9b00adcc --- /dev/null +++ b/internal/apirouter/legacy_handlers.go @@ -0,0 +1,260 @@ +package apirouter + +import ( + "errors" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/hookdeck/outpost/internal/deliverymq" + "github.com/hookdeck/outpost/internal/logging" + "github.com/hookdeck/outpost/internal/logstore" + "github.com/hookdeck/outpost/internal/models" + "go.uber.org/zap" +) + +var ( + ErrDestinationDisabled = errors.New("destination is disabled") +) + +// LegacyHandlers provides backward-compatible endpoints for the old API. +// These handlers are deprecated and will be removed in a future version. +type LegacyHandlers struct { + logger *logging.Logger + entityStore models.EntityStore + logStore logstore.LogStore + deliveryMQ *deliverymq.DeliveryMQ +} + +func NewLegacyHandlers( + logger *logging.Logger, + entityStore models.EntityStore, + logStore logstore.LogStore, + deliveryMQ *deliverymq.DeliveryMQ, +) *LegacyHandlers { + return &LegacyHandlers{ + logger: logger, + entityStore: entityStore, + logStore: logStore, + deliveryMQ: deliveryMQ, + } +} + +// setDeprecationHeader adds deprecation warning headers to the response. +func setDeprecationHeader(c *gin.Context, newEndpoint string) { + c.Header("Deprecation", "true") + c.Header("X-Deprecated-Message", "This endpoint is deprecated. Use "+newEndpoint+" instead.") +} + +// RetryByEventDestination handles the legacy retry endpoint: +// POST /:tenantID/destinations/:destinationID/events/:eventID/retry +// +// This shim finds the latest delivery for the event+destination pair and retries it. +// Deprecated: Use POST /:tenantID/deliveries/:deliveryID/retry instead. +func (h *LegacyHandlers) RetryByEventDestination(c *gin.Context) { + setDeprecationHeader(c, "POST /:tenantID/deliveries/:deliveryID/retry") + + tenant := mustTenantFromContext(c) + if tenant == nil { + return + } + destinationID := c.Param("destinationID") + eventID := c.Param("eventID") + + // 1. Check destination exists and is enabled + destination, err := h.entityStore.RetrieveDestination(c.Request.Context(), tenant.ID, destinationID) + if err != nil { + AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) + return + } + if destination == nil { + AbortWithError(c, http.StatusNotFound, NewErrNotFound("destination")) + return + } + if destination.DisabledAt != nil { + AbortWithError(c, http.StatusBadRequest, NewErrBadRequest(ErrDestinationDisabled)) + return + } + + // 2. Retrieve event + event, err := h.logStore.RetrieveEvent(c.Request.Context(), logstore.RetrieveEventRequest{ + TenantID: tenant.ID, + EventID: eventID, + }) + if err != nil { + AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) + return + } + if event == nil { + AbortWithError(c, http.StatusNotFound, NewErrNotFound("event")) + return + } + + // 3. Create and publish retry delivery event + deliveryEvent := models.NewManualDeliveryEvent(*event, destination.ID) + + if err := h.deliveryMQ.Publish(c.Request.Context(), deliveryEvent); err != nil { + AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) + return + } + + h.logger.Ctx(c.Request.Context()).Audit("manual retry initiated (legacy)", + zap.String("event_id", event.ID), + zap.String("destination_id", destination.ID)) + + c.JSON(http.StatusAccepted, gin.H{ + "success": true, + }) +} + +// ListEventsByDestination handles the legacy endpoint: +// GET /:tenantID/destinations/:destinationID/events +// +// This shim queries deliveries filtered by destination and returns unique events. +// Deprecated: Use GET /:tenantID/deliveries?destination_id=X&expand=event instead. +func (h *LegacyHandlers) ListEventsByDestination(c *gin.Context) { + setDeprecationHeader(c, "GET /:tenantID/deliveries?destination_id=X&expand=event") + + tenant := mustTenantFromContext(c) + if tenant == nil { + return + } + destinationID := c.Param("destinationID") + + // Parse pagination params + limit := 100 + if limitStr := c.Query("limit"); limitStr != "" { + if parsed, err := strconv.Atoi(limitStr); err == nil && parsed > 0 { + limit = parsed + } + } + + // Query deliveries for this destination with pagination + response, err := h.logStore.ListDeliveryEvent(c.Request.Context(), logstore.ListDeliveryEventRequest{ + TenantID: tenant.ID, + DestinationIDs: []string{destinationID}, + Limit: limit, + Next: c.Query("next"), + Prev: c.Query("prev"), + SortBy: "event_time", + SortOrder: "desc", + }) + if err != nil { + AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) + return + } + + // Extract unique events (by event ID, keep first occurrence) + seen := make(map[string]bool) + events := []models.Event{} + for _, de := range response.Data { + if !seen[de.Event.ID] { + seen[de.Event.ID] = true + events = append(events, de.Event) + } + } + + // Return empty array (not null) if no events + if len(events) == 0 { + c.JSON(http.StatusOK, gin.H{ + "data": []models.Event{}, + "next": "", + "prev": "", + "count": 0, + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "data": events, + "next": response.Next, + "prev": response.Prev, + "count": len(events), + }) +} + +// RetrieveEventByDestination handles the legacy endpoint: +// GET /:tenantID/destinations/:destinationID/events/:eventID +// +// Deprecated: Use GET /:tenantID/events/:eventID instead. +func (h *LegacyHandlers) RetrieveEventByDestination(c *gin.Context) { + setDeprecationHeader(c, "GET /:tenantID/events/:eventID") + + tenant := mustTenantFromContext(c) + if tenant == nil { + return + } + eventID := c.Param("eventID") + // destinationID is available but not strictly needed for retrieval + + event, err := h.logStore.RetrieveEvent(c.Request.Context(), logstore.RetrieveEventRequest{ + TenantID: tenant.ID, + EventID: eventID, + }) + if err != nil { + AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) + return + } + if event == nil { + AbortWithError(c, http.StatusNotFound, NewErrNotFound("event")) + return + } + + c.JSON(http.StatusOK, event) +} + +// LegacyDeliveryResponse matches the old delivery response format. +type LegacyDeliveryResponse struct { + ID string `json:"id"` + DeliveredAt string `json:"delivered_at"` + Status string `json:"status"` + Code string `json:"code"` + ResponseData map[string]interface{} `json:"response_data"` +} + +// ListDeliveriesByEvent handles the legacy endpoint: +// GET /:tenantID/events/:eventID/deliveries +// +// Deprecated: Use GET /:tenantID/deliveries?event_id=X instead. +func (h *LegacyHandlers) ListDeliveriesByEvent(c *gin.Context) { + setDeprecationHeader(c, "GET /:tenantID/deliveries?event_id=X") + + tenant := mustTenantFromContext(c) + if tenant == nil { + return + } + eventID := c.Param("eventID") + + // Query deliveries for this event + response, err := h.logStore.ListDeliveryEvent(c.Request.Context(), logstore.ListDeliveryEventRequest{ + TenantID: tenant.ID, + EventID: eventID, + Limit: 100, + SortBy: "delivery_time", + SortOrder: "desc", + }) + if err != nil { + AbortWithError(c, http.StatusInternalServerError, NewErrInternalServer(err)) + return + } + + // Return empty array (not null) if no deliveries + if len(response.Data) == 0 { + c.JSON(http.StatusOK, []LegacyDeliveryResponse{}) + return + } + + // Transform to legacy delivery response format (bare array) + deliveries := make([]LegacyDeliveryResponse, len(response.Data)) + for i, de := range response.Data { + deliveries[i] = LegacyDeliveryResponse{ + ID: de.Delivery.ID, + DeliveredAt: de.Delivery.Time.UTC().Format("2006-01-02T15:04:05Z07:00"), + Status: de.Delivery.Status, + Code: de.Delivery.Code, + ResponseData: de.Delivery.ResponseData, + } + } + + c.JSON(http.StatusOK, deliveries) +} diff --git a/internal/apirouter/legacy_handlers_test.go b/internal/apirouter/legacy_handlers_test.go new file mode 100644 index 00000000..14e1880e --- /dev/null +++ b/internal/apirouter/legacy_handlers_test.go @@ -0,0 +1,348 @@ +package apirouter_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/hookdeck/outpost/internal/idgen" + "github.com/hookdeck/outpost/internal/models" + "github.com/hookdeck/outpost/internal/util/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLegacyRetryByEventDestination(t *testing.T) { + t.Parallel() + + result := setupTestRouterFull(t, "", "") + + // Create a tenant and destination + tenantID := idgen.String() + destinationID := idgen.Destination() + require.NoError(t, result.entityStore.UpsertTenant(context.Background(), models.Tenant{ + ID: tenantID, + CreatedAt: time.Now(), + })) + require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ + ID: destinationID, + TenantID: tenantID, + Type: "webhook", + Topics: []string{"*"}, + CreatedAt: time.Now(), + })) + + // Seed a delivery event + eventID := idgen.Event() + deliveryID := idgen.Delivery() + eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) + deliveryTime := eventTime.Add(100 * time.Millisecond) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("order.created"), + testutil.EventFactory.WithTime(eventTime), + ) + + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(deliveryID), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("failed"), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, deliveryID), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + + require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + + t.Run("should retry via legacy endpoint and return deprecation header", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", baseAPIPath+"/"+tenantID+"/destinations/"+destinationID+"/events/"+eventID+"/retry", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusAccepted, w.Code) + assert.Equal(t, "true", w.Header().Get("Deprecation")) + assert.Contains(t, w.Header().Get("X-Deprecated-Message"), "POST /:tenantID/deliveries/:deliveryID/retry") + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + assert.Equal(t, true, response["success"]) + }) + + t.Run("should return 404 for non-existent event", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", baseAPIPath+"/"+tenantID+"/destinations/"+destinationID+"/events/nonexistent/retry", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + }) + + t.Run("should return 404 for non-existent destination", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", baseAPIPath+"/"+tenantID+"/destinations/nonexistent/events/"+eventID+"/retry", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + }) + + t.Run("should return 400 when destination is disabled", func(t *testing.T) { + // Create a disabled destination + disabledDestinationID := idgen.Destination() + disabledAt := time.Now() + require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ + ID: disabledDestinationID, + TenantID: tenantID, + Type: "webhook", + Topics: []string{"*"}, + CreatedAt: time.Now(), + DisabledAt: &disabledAt, + })) + + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", baseAPIPath+"/"+tenantID+"/destinations/"+disabledDestinationID+"/events/"+eventID+"/retry", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + }) +} + +func TestLegacyListEventsByDestination(t *testing.T) { + t.Parallel() + + result := setupTestRouterFull(t, "", "") + + // Create a tenant and destination + tenantID := idgen.String() + destinationID := idgen.Destination() + require.NoError(t, result.entityStore.UpsertTenant(context.Background(), models.Tenant{ + ID: tenantID, + CreatedAt: time.Now(), + })) + require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ + ID: destinationID, + TenantID: tenantID, + Type: "webhook", + Topics: []string{"*"}, + CreatedAt: time.Now(), + })) + + // Seed delivery events + eventID := idgen.Event() + deliveryID := idgen.Delivery() + eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) + deliveryTime := eventTime.Add(100 * time.Millisecond) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("order.created"), + testutil.EventFactory.WithTime(eventTime), + ) + + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(deliveryID), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("success"), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, deliveryID), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + + require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + + t.Run("should list events for destination with deprecation header", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/destinations/"+destinationID+"/events", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "true", w.Header().Get("Deprecation")) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + + events := response["data"].([]interface{}) + assert.Len(t, events, 1) + + firstEvent := events[0].(map[string]interface{}) + assert.Equal(t, eventID, firstEvent["id"]) + assert.Equal(t, "order.created", firstEvent["topic"]) + }) +} + +func TestLegacyRetrieveEventByDestination(t *testing.T) { + t.Parallel() + + result := setupTestRouterFull(t, "", "") + + // Create a tenant and destination + tenantID := idgen.String() + destinationID := idgen.Destination() + require.NoError(t, result.entityStore.UpsertTenant(context.Background(), models.Tenant{ + ID: tenantID, + CreatedAt: time.Now(), + })) + require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ + ID: destinationID, + TenantID: tenantID, + Type: "webhook", + Topics: []string{"*"}, + CreatedAt: time.Now(), + })) + + // Seed a delivery event + eventID := idgen.Event() + deliveryID := idgen.Delivery() + eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) + deliveryTime := eventTime.Add(100 * time.Millisecond) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("order.created"), + testutil.EventFactory.WithTime(eventTime), + ) + + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(deliveryID), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("success"), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, deliveryID), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + + require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + + t.Run("should retrieve event by destination with deprecation header", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/destinations/"+destinationID+"/events/"+eventID, nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "true", w.Header().Get("Deprecation")) + + var response map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + + assert.Equal(t, eventID, response["id"]) + assert.Equal(t, "order.created", response["topic"]) + }) + + t.Run("should return 404 for non-existent event", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/destinations/"+destinationID+"/events/nonexistent", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + }) +} + +func TestLegacyListDeliveriesByEvent(t *testing.T) { + t.Parallel() + + result := setupTestRouterFull(t, "", "") + + // Create a tenant and destination + tenantID := idgen.String() + destinationID := idgen.Destination() + require.NoError(t, result.entityStore.UpsertTenant(context.Background(), models.Tenant{ + ID: tenantID, + CreatedAt: time.Now(), + })) + require.NoError(t, result.entityStore.UpsertDestination(context.Background(), models.Destination{ + ID: destinationID, + TenantID: tenantID, + Type: "webhook", + Topics: []string{"*"}, + CreatedAt: time.Now(), + })) + + // Seed a delivery event + eventID := idgen.Event() + deliveryID := idgen.Delivery() + eventTime := time.Now().Add(-1 * time.Hour).Truncate(time.Millisecond) + deliveryTime := eventTime.Add(100 * time.Millisecond) + + event := testutil.EventFactory.AnyPointer( + testutil.EventFactory.WithID(eventID), + testutil.EventFactory.WithTenantID(tenantID), + testutil.EventFactory.WithDestinationID(destinationID), + testutil.EventFactory.WithTopic("order.created"), + testutil.EventFactory.WithTime(eventTime), + ) + + delivery := testutil.DeliveryFactory.AnyPointer( + testutil.DeliveryFactory.WithID(deliveryID), + testutil.DeliveryFactory.WithEventID(eventID), + testutil.DeliveryFactory.WithDestinationID(destinationID), + testutil.DeliveryFactory.WithStatus("success"), + testutil.DeliveryFactory.WithTime(deliveryTime), + ) + + de := &models.DeliveryEvent{ + ID: fmt.Sprintf("%s_%s", eventID, deliveryID), + DestinationID: destinationID, + Event: *event, + Delivery: delivery, + } + + require.NoError(t, result.logStore.InsertManyDeliveryEvent(context.Background(), []*models.DeliveryEvent{de})) + + t.Run("should list deliveries for event with deprecation header", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/events/"+eventID+"/deliveries", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "true", w.Header().Get("Deprecation")) + + // Old format returns bare array, not {data: [...]} + var deliveries []map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &deliveries)) + + assert.Len(t, deliveries, 1) + assert.Equal(t, deliveryID, deliveries[0]["id"]) + assert.Equal(t, "success", deliveries[0]["status"]) + }) + + t.Run("should return empty list for non-existent event", func(t *testing.T) { + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", baseAPIPath+"/"+tenantID+"/events/nonexistent/deliveries", nil) + result.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + // Old format returns bare array + var deliveries []map[string]interface{} + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &deliveries)) + + assert.Len(t, deliveries, 0) + }) +} diff --git a/internal/apirouter/router.go b/internal/apirouter/router.go index ae4cb8be..9f9be79c 100644 --- a/internal/apirouter/router.go +++ b/internal/apirouter/router.go @@ -154,6 +154,7 @@ func NewRouter( logHandlers := NewLogHandlers(logger, logStore) retryHandlers := NewRetryHandlers(logger, entityStore, logStore, deliveryMQ) topicHandlers := NewTopicHandlers(logger, cfg.Topics) + legacyHandlers := NewLegacyHandlers(logger, entityStore, logStore, deliveryMQ) // Admin routes adminRoutes := []RouteDefinition{ @@ -383,12 +384,61 @@ func NewRouter( }, } + // Legacy routes (deprecated, for backward compatibility) + legacyRoutes := []RouteDefinition{ + { + Method: http.MethodPost, + Path: "/:tenantID/destinations/:destinationID/events/:eventID/retry", + Handler: legacyHandlers.RetryByEventDestination, + AuthScope: AuthScopeAdminOrTenant, + Mode: RouteModeAlways, + AllowTenantFromJWT: true, + Middlewares: []gin.HandlerFunc{ + RequireTenantMiddleware(entityStore), + }, + }, + { + Method: http.MethodGet, + Path: "/:tenantID/destinations/:destinationID/events", + Handler: legacyHandlers.ListEventsByDestination, + AuthScope: AuthScopeAdminOrTenant, + Mode: RouteModeAlways, + AllowTenantFromJWT: true, + Middlewares: []gin.HandlerFunc{ + RequireTenantMiddleware(entityStore), + }, + }, + { + Method: http.MethodGet, + Path: "/:tenantID/destinations/:destinationID/events/:eventID", + Handler: legacyHandlers.RetrieveEventByDestination, + AuthScope: AuthScopeAdminOrTenant, + Mode: RouteModeAlways, + AllowTenantFromJWT: true, + Middlewares: []gin.HandlerFunc{ + RequireTenantMiddleware(entityStore), + }, + }, + { + Method: http.MethodGet, + Path: "/:tenantID/events/:eventID/deliveries", + Handler: legacyHandlers.ListDeliveriesByEvent, + AuthScope: AuthScopeAdminOrTenant, + Mode: RouteModeAlways, + AllowTenantFromJWT: true, + Middlewares: []gin.HandlerFunc{ + RequireTenantMiddleware(entityStore), + }, + }, + } + // Register all routes to a single router apiRoutes := []RouteDefinition{} // combine all routes apiRoutes = append(apiRoutes, adminRoutes...) apiRoutes = append(apiRoutes, portalRoutes...) apiRoutes = append(apiRoutes, tenantAgnosticRoutes...) apiRoutes = append(apiRoutes, tenantSpecificRoutes...) + apiRoutes = append(apiRoutes, legacyRoutes...) registerRoutes(apiRouter, cfg, apiRoutes) From f9238441f94c08c74382ddc270f0a538c7a83e3c Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Mon, 8 Dec 2025 19:20:27 +0700 Subject: [PATCH 16/19] test: e2e log api test cases --- cmd/e2e/log_test.go | 932 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 932 insertions(+) create mode 100644 cmd/e2e/log_test.go diff --git a/cmd/e2e/log_test.go b/cmd/e2e/log_test.go new file mode 100644 index 00000000..6779ce35 --- /dev/null +++ b/cmd/e2e/log_test.go @@ -0,0 +1,932 @@ +package e2e_test + +import ( + "fmt" + "net/http" + "time" + + "github.com/hookdeck/outpost/cmd/e2e/httpclient" + "github.com/hookdeck/outpost/internal/idgen" +) + +// TestLogAPI tests the new Log API endpoints (deliveries, events). +// +// Setup: +// 1. Create a tenant +// 2. Configure mock webhook server to accept deliveries +// 3. Create a destination pointing to the mock server +// 4. Publish an event and wait for delivery to complete +// +// Test Cases: +// - GET /:tenantID/deliveries - List all deliveries with proper response structure +// - GET /:tenantID/deliveries?destination_id=X - Filter deliveries by destination +// - GET /:tenantID/deliveries?event_id=X - Filter deliveries by event +// - GET /:tenantID/deliveries?expand=event - Expand event summary (without data) +// - GET /:tenantID/deliveries?expand=event.data - Expand full event with payload data +// - GET /:tenantID/events/:eventID - Retrieve a single event with full details +// - GET /:tenantID/events/:eventID (non-existent) - Returns 404 +func (suite *basicSuite) TestLogAPI() { + tenantID := idgen.String() + destinationID := idgen.Destination() + eventID := idgen.Event() + + // Setup: Create tenant, destination, and publish an event + setupTests := []APITest{ + { + Name: "PUT /:tenantID - create tenant", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPUT, + Path: "/" + tenantID, + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusCreated, + }, + }, + }, + { + Name: "PUT mockserver/destinations - setup mock", + Request: httpclient.Request{ + Method: httpclient.MethodPUT, + BaseURL: suite.mockServerBaseURL, + Path: "/destinations", + Body: map[string]interface{}{ + "id": destinationID, + "type": "webhook", + "config": map[string]interface{}{ + "url": fmt.Sprintf("%s/webhook/%s", suite.mockServerBaseURL, destinationID), + }, + }, + }, + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusOK, + }, + }, + }, + { + Name: "POST /:tenantID/destinations - create destination", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPOST, + Path: "/" + tenantID + "/destinations", + Body: map[string]interface{}{ + "id": destinationID, + "type": "webhook", + "topics": "*", + "config": map[string]interface{}{ + "url": fmt.Sprintf("%s/webhook/%s", suite.mockServerBaseURL, destinationID), + }, + }, + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusCreated, + }, + }, + }, + { + Name: "POST /publish - publish event", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPOST, + Path: "/publish", + Body: map[string]interface{}{ + "id": eventID, + "tenant_id": tenantID, + "topic": "user.created", + "eligible_for_retry": true, + "data": map[string]interface{}{ + "user_id": "123", + }, + }, + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusAccepted, + }, + }, + }, + } + suite.RunAPITests(suite.T(), setupTests) + + // Wait for delivery to complete + time.Sleep(2 * time.Second) + + // Test the new Log API endpoints + logAPITests := []APITest{ + // GET /:tenantID/deliveries - list deliveries + { + Name: "GET /:tenantID/deliveries - list all deliveries", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/" + tenantID + "/deliveries", + }), + Expected: APITestExpectation{ + Validate: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "statusCode": map[string]interface{}{"const": 200}, + "body": map[string]interface{}{ + "type": "object", + "required": []interface{}{"data"}, + "properties": map[string]interface{}{ + "data": map[string]interface{}{ + "type": "array", + "minItems": 1, + "items": map[string]interface{}{ + "type": "object", + "required": []interface{}{"id", "event", "destination", "status", "delivered_at"}, + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "string"}, + "event": map[string]interface{}{"type": "string"}, // Event ID when not expanded + "destination": map[string]interface{}{"const": destinationID}, + "status": map[string]interface{}{"type": "string"}, + "delivered_at": map[string]interface{}{"type": "string"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + // GET /:tenantID/deliveries?destination_id=X - filter by destination + { + Name: "GET /:tenantID/deliveries?destination_id=X - filter by destination", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/" + tenantID + "/deliveries?destination_id=" + destinationID, + }), + Expected: APITestExpectation{ + Validate: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "statusCode": map[string]interface{}{"const": 200}, + "body": map[string]interface{}{ + "type": "object", + "required": []interface{}{"data"}, + "properties": map[string]interface{}{ + "data": map[string]interface{}{ + "type": "array", + "minItems": 1, + }, + }, + }, + }, + }, + }, + }, + // GET /:tenantID/deliveries?event_id=X - filter by event + { + Name: "GET /:tenantID/deliveries?event_id=X - filter by event", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/" + tenantID + "/deliveries?event_id=" + eventID, + }), + Expected: APITestExpectation{ + Validate: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "statusCode": map[string]interface{}{"const": 200}, + "body": map[string]interface{}{ + "type": "object", + "required": []interface{}{"data"}, + "properties": map[string]interface{}{ + "data": map[string]interface{}{ + "type": "array", + "minItems": 1, + }, + }, + }, + }, + }, + }, + }, + // GET /:tenantID/deliveries?expand=event - expand event (without data) + { + Name: "GET /:tenantID/deliveries?expand=event - expand event summary", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/" + tenantID + "/deliveries?expand=event", + }), + Expected: APITestExpectation{ + Validate: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "statusCode": map[string]interface{}{"const": 200}, + "body": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "data": map[string]interface{}{ + "type": "array", + "minItems": 1, + "items": map[string]interface{}{ + "type": "object", + "required": []interface{}{"event"}, + "properties": map[string]interface{}{ + "event": map[string]interface{}{ + "type": "object", + "required": []interface{}{"id", "topic", "time"}, + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "string"}, + "topic": map[string]interface{}{"type": "string"}, + "time": map[string]interface{}{"type": "string"}, + }, + // expand=event should NOT include data + "not": map[string]interface{}{ + "required": []interface{}{"data"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + // GET /:tenantID/deliveries?expand=event.data - expand full event with data + { + Name: "GET /:tenantID/deliveries?expand=event.data - expand full event", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/" + tenantID + "/deliveries?expand=event.data", + }), + Expected: APITestExpectation{ + Validate: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "statusCode": map[string]interface{}{"const": 200}, + "body": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "data": map[string]interface{}{ + "type": "array", + "minItems": 1, + "items": map[string]interface{}{ + "type": "object", + "required": []interface{}{"event"}, + "properties": map[string]interface{}{ + "event": map[string]interface{}{ + "type": "object", + "required": []interface{}{"id", "topic", "time", "data"}, + "properties": map[string]interface{}{ + "id": map[string]interface{}{"const": eventID}, + "topic": map[string]interface{}{"const": "user.created"}, + "time": map[string]interface{}{"type": "string"}, + "data": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "user_id": map[string]interface{}{"const": "123"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + // GET /:tenantID/events/:eventID - retrieve single event + { + Name: "GET /:tenantID/events/:eventID - retrieve event", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/" + tenantID + "/events/" + eventID, + }), + Expected: APITestExpectation{ + Validate: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "statusCode": map[string]interface{}{"const": 200}, + "body": map[string]interface{}{ + "type": "object", + "required": []interface{}{"id", "topic", "time", "data"}, + "properties": map[string]interface{}{ + "id": map[string]interface{}{"const": eventID}, + "topic": map[string]interface{}{"const": "user.created"}, + "data": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "user_id": map[string]interface{}{"const": "123"}, + }, + }, + }, + }, + }, + }, + }, + }, + // GET /:tenantID/events/:eventID - non-existent event + { + Name: "GET /:tenantID/events/:eventID - not found", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/" + tenantID + "/events/" + idgen.Event(), + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusNotFound, + }, + }, + }, + } + suite.RunAPITests(suite.T(), logAPITests) + + // Cleanup + cleanupTests := []APITest{ + { + Name: "DELETE mockserver/destinations/:destinationID", + Request: httpclient.Request{ + Method: httpclient.MethodDELETE, + BaseURL: suite.mockServerBaseURL, + Path: "/destinations/" + destinationID, + }, + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusOK, + }, + }, + }, + { + Name: "DELETE /:tenantID", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodDELETE, + Path: "/" + tenantID, + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusOK, + }, + }, + }, + } + suite.RunAPITests(suite.T(), cleanupTests) +} + +// TestRetryAPI tests the retry endpoint. +// +// Setup: +// 1. Create a tenant +// 2. Configure mock webhook server to FAIL (return 500) +// 3. Create a destination pointing to the mock server +// 4. Publish an event with eligible_for_retry=false (fails once, no auto-retry) +// 5. Wait for delivery to fail, then fetch the delivery ID +// 6. Update mock server to SUCCEED (return 200) +// +// Test Cases: +// - POST /:tenantID/deliveries/:deliveryID/retry - Successful retry returns 202 Accepted +// - POST /:tenantID/deliveries/:deliveryID/retry (non-existent) - Returns 404 +// - Verify retry created new delivery - Event now has 2+ deliveries +// - POST /:tenantID/deliveries/:deliveryID/retry (disabled destination) - Returns 400 +func (suite *basicSuite) TestRetryAPI() { + tenantID := idgen.String() + destinationID := idgen.Destination() + eventID := idgen.Event() + + // Setup: create tenant, destination with failing webhook, and publish event + setupTests := []APITest{ + { + Name: "PUT /:tenantID - create tenant", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPUT, + Path: "/" + tenantID, + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusCreated, + }, + }, + }, + { + Name: "PUT mockserver/destinations - setup mock to fail", + Request: httpclient.Request{ + Method: httpclient.MethodPUT, + BaseURL: suite.mockServerBaseURL, + Path: "/destinations", + Body: map[string]interface{}{ + "id": destinationID, + "type": "webhook", + "config": map[string]interface{}{ + "url": fmt.Sprintf("%s/webhook/%s", suite.mockServerBaseURL, destinationID), + }, + "response": map[string]interface{}{ + "status": 500, // Fail deliveries + }, + }, + }, + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusOK, + }, + }, + }, + { + Name: "POST /:tenantID/destinations - create destination", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPOST, + Path: "/" + tenantID + "/destinations", + Body: map[string]interface{}{ + "id": destinationID, + "type": "webhook", + "topics": "*", + "config": map[string]interface{}{ + "url": fmt.Sprintf("%s/webhook/%s", suite.mockServerBaseURL, destinationID), + }, + }, + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusCreated, + }, + }, + }, + { + Name: "POST /publish - publish event (will fail)", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPOST, + Path: "/publish", + Body: map[string]interface{}{ + "id": eventID, + "tenant_id": tenantID, + "topic": "user.created", + "eligible_for_retry": false, // Disable auto-retry + "data": map[string]interface{}{ + "user_id": "456", + }, + }, + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusAccepted, + }, + }, + }, + } + suite.RunAPITests(suite.T(), setupTests) + + // Wait for delivery to complete (and fail) + time.Sleep(2 * time.Second) + + // Get the delivery ID + deliveriesResp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/" + tenantID + "/deliveries?event_id=" + eventID, + })) + suite.Require().NoError(err) + suite.Require().Equal(http.StatusOK, deliveriesResp.StatusCode) + + body := deliveriesResp.Body.(map[string]interface{}) + data := body["data"].([]interface{}) + suite.Require().NotEmpty(data, "should have at least one delivery") + firstDelivery := data[0].(map[string]interface{}) + deliveryID := firstDelivery["id"].(string) + + // Update mock to succeed for retry + updateMockTests := []APITest{ + { + Name: "PUT mockserver/destinations - setup mock to succeed", + Request: httpclient.Request{ + Method: httpclient.MethodPUT, + BaseURL: suite.mockServerBaseURL, + Path: "/destinations", + Body: map[string]interface{}{ + "id": destinationID, + "type": "webhook", + "config": map[string]interface{}{ + "url": fmt.Sprintf("%s/webhook/%s", suite.mockServerBaseURL, destinationID), + }, + "response": map[string]interface{}{ + "status": 200, // Now succeed + }, + }, + }, + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusOK, + }, + }, + }, + } + suite.RunAPITests(suite.T(), updateMockTests) + + // Test retry endpoint + retryTests := []APITest{ + // POST /:tenantID/deliveries/:deliveryID/retry - successful retry + { + Name: "POST /:tenantID/deliveries/:deliveryID/retry - retry delivery", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPOST, + Path: "/" + tenantID + "/deliveries/" + deliveryID + "/retry", + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusAccepted, + Body: map[string]interface{}{ + "success": true, + }, + }, + }, + }, + // POST /:tenantID/deliveries/:deliveryID/retry - non-existent delivery + { + Name: "POST /:tenantID/deliveries/:deliveryID/retry - not found", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPOST, + Path: "/" + tenantID + "/deliveries/" + idgen.Delivery() + "/retry", + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusNotFound, + }, + }, + }, + } + suite.RunAPITests(suite.T(), retryTests) + + // Wait for retry delivery to complete + time.Sleep(2 * time.Second) + + // Verify we have more deliveries after retry + verifyTests := []APITest{ + { + Name: "GET /:tenantID/deliveries?event_id=X - verify retry created new delivery", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/" + tenantID + "/deliveries?event_id=" + eventID, + }), + Expected: APITestExpectation{ + Validate: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "statusCode": map[string]interface{}{"const": 200}, + "body": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "data": map[string]interface{}{ + "type": "array", + "minItems": 2, // Original + retry + }, + }, + }, + }, + }, + }, + }, + } + suite.RunAPITests(suite.T(), verifyTests) + + // Test retry on disabled destination + disableTests := []APITest{ + { + Name: "PUT /:tenantID/destinations/:destinationID/disable", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPUT, + Path: "/" + tenantID + "/destinations/" + destinationID + "/disable", + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusOK, + }, + }, + }, + { + Name: "POST /:tenantID/deliveries/:deliveryID/retry - disabled destination", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPOST, + Path: "/" + tenantID + "/deliveries/" + deliveryID + "/retry", + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]interface{}{ + "message": "Destination is disabled", + }, + }, + }, + }, + } + suite.RunAPITests(suite.T(), disableTests) + + // Cleanup + cleanupTests := []APITest{ + { + Name: "DELETE mockserver/destinations/:destinationID", + Request: httpclient.Request{ + Method: httpclient.MethodDELETE, + BaseURL: suite.mockServerBaseURL, + Path: "/destinations/" + destinationID, + }, + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusOK, + }, + }, + }, + { + Name: "DELETE /:tenantID", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodDELETE, + Path: "/" + tenantID, + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusOK, + }, + }, + }, + } + suite.RunAPITests(suite.T(), cleanupTests) +} + +// TestLegacyLogAPI tests the deprecated legacy endpoints for backward compatibility. +// All legacy endpoints return "Deprecation: true" header to signal migration. +// +// Setup: +// 1. Create a tenant +// 2. Configure mock webhook server to accept deliveries +// 3. Create a destination pointing to the mock server +// 4. Publish an event and wait for delivery to complete +// +// Test Cases: +// - GET /:tenantID/destinations/:destID/events - Legacy list events (returns {data, count}) +// - GET /:tenantID/destinations/:destID/events/:eventID - Legacy retrieve event +// - GET /:tenantID/events/:eventID/deliveries - Legacy list deliveries (returns bare array, not {data}) +// - POST /:tenantID/destinations/:destID/events/:eventID/retry - Legacy retry endpoint +// +// All responses include: +// - Deprecation: true header +// - X-Deprecated-Message header with migration guidance +func (suite *basicSuite) TestLegacyLogAPI() { + tenantID := idgen.String() + destinationID := idgen.Destination() + eventID := idgen.Event() + + // Setup + setupTests := []APITest{ + { + Name: "PUT /:tenantID - create tenant", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPUT, + Path: "/" + tenantID, + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusCreated, + }, + }, + }, + { + Name: "PUT mockserver/destinations - setup mock", + Request: httpclient.Request{ + Method: httpclient.MethodPUT, + BaseURL: suite.mockServerBaseURL, + Path: "/destinations", + Body: map[string]interface{}{ + "id": destinationID, + "type": "webhook", + "config": map[string]interface{}{ + "url": fmt.Sprintf("%s/webhook/%s", suite.mockServerBaseURL, destinationID), + }, + }, + }, + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusOK, + }, + }, + }, + { + Name: "POST /:tenantID/destinations - create destination", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPOST, + Path: "/" + tenantID + "/destinations", + Body: map[string]interface{}{ + "id": destinationID, + "type": "webhook", + "topics": "*", + "config": map[string]interface{}{ + "url": fmt.Sprintf("%s/webhook/%s", suite.mockServerBaseURL, destinationID), + }, + }, + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusCreated, + }, + }, + }, + { + Name: "POST /publish - publish event", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPOST, + Path: "/publish", + Body: map[string]interface{}{ + "id": eventID, + "tenant_id": tenantID, + "topic": "user.created", + "eligible_for_retry": true, + "data": map[string]interface{}{ + "user_id": "789", + }, + }, + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusAccepted, + }, + }, + }, + } + suite.RunAPITests(suite.T(), setupTests) + + // Wait for delivery + time.Sleep(2 * time.Second) + + // Test legacy endpoints - all should return deprecation headers + legacyTests := []APITest{ + // GET /:tenantID/destinations/:destinationID/events - legacy list events by destination + { + Name: "GET /:tenantID/destinations/:destinationID/events - legacy endpoint", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/" + tenantID + "/destinations/" + destinationID + "/events", + }), + Expected: APITestExpectation{ + Validate: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "statusCode": map[string]interface{}{"const": 200}, + "headers": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "Deprecation": map[string]interface{}{ + "type": "array", + "items": map[string]interface{}{ + "const": "true", + }, + }, + }, + }, + "body": map[string]interface{}{ + "type": "object", + "required": []interface{}{"data", "count"}, + "properties": map[string]interface{}{ + "data": map[string]interface{}{ + "type": "array", + "minItems": 1, + }, + "count": map[string]interface{}{"type": "number"}, + }, + }, + }, + }, + }, + }, + // GET /:tenantID/destinations/:destinationID/events/:eventID - legacy retrieve event + { + Name: "GET /:tenantID/destinations/:destinationID/events/:eventID - legacy endpoint", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/" + tenantID + "/destinations/" + destinationID + "/events/" + eventID, + }), + Expected: APITestExpectation{ + Validate: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "statusCode": map[string]interface{}{"const": 200}, + "headers": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "Deprecation": map[string]interface{}{ + "type": "array", + "items": map[string]interface{}{ + "const": "true", + }, + }, + }, + }, + "body": map[string]interface{}{ + "type": "object", + "required": []interface{}{"id", "topic"}, + "properties": map[string]interface{}{ + "id": map[string]interface{}{"const": eventID}, + "topic": map[string]interface{}{"const": "user.created"}, + }, + }, + }, + }, + }, + }, + // GET /:tenantID/events/:eventID/deliveries - legacy list deliveries by event + { + Name: "GET /:tenantID/events/:eventID/deliveries - legacy endpoint (returns bare array)", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodGET, + Path: "/" + tenantID + "/events/" + eventID + "/deliveries", + }), + Expected: APITestExpectation{ + Validate: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "statusCode": map[string]interface{}{"const": 200}, + "headers": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "Deprecation": map[string]interface{}{ + "type": "array", + "items": map[string]interface{}{ + "const": "true", + }, + }, + }, + }, + // Legacy endpoint returns bare array, not {data: [...]} + "body": map[string]interface{}{ + "type": "array", + "minItems": 1, + "items": map[string]interface{}{ + "type": "object", + "required": []interface{}{"id", "status", "delivered_at"}, + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "string"}, + "status": map[string]interface{}{"type": "string"}, + "delivered_at": map[string]interface{}{"type": "string"}, + }, + }, + }, + }, + }, + }, + }, + // POST /:tenantID/destinations/:destinationID/events/:eventID/retry - legacy retry + { + Name: "POST /:tenantID/destinations/:destinationID/events/:eventID/retry - legacy endpoint", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodPOST, + Path: "/" + tenantID + "/destinations/" + destinationID + "/events/" + eventID + "/retry", + }), + Expected: APITestExpectation{ + Validate: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "statusCode": map[string]interface{}{"const": 202}, + "headers": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "Deprecation": map[string]interface{}{ + "type": "array", + "items": map[string]interface{}{ + "const": "true", + }, + }, + }, + }, + "body": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "success": map[string]interface{}{"const": true}, + }, + }, + }, + }, + }, + }, + } + suite.RunAPITests(suite.T(), legacyTests) + + // Cleanup + cleanupTests := []APITest{ + { + Name: "DELETE mockserver/destinations/:destinationID", + Request: httpclient.Request{ + Method: httpclient.MethodDELETE, + BaseURL: suite.mockServerBaseURL, + Path: "/destinations/" + destinationID, + }, + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusOK, + }, + }, + }, + { + Name: "DELETE /:tenantID", + Request: suite.AuthRequest(httpclient.Request{ + Method: httpclient.MethodDELETE, + Path: "/" + tenantID, + }), + Expected: APITestExpectation{ + Match: &httpclient.Response{ + StatusCode: http.StatusOK, + }, + }, + }, + } + suite.RunAPITests(suite.T(), cleanupTests) +} From 6e1b140eb9ef64cdaabcf9e00827fbb828f5d9c1 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Tue, 2 Dec 2025 18:32:59 +0700 Subject: [PATCH 17/19] build: clickhouse deps in docker compose --- build/dev/deps/compose.yml | 70 +++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/build/dev/deps/compose.yml b/build/dev/deps/compose.yml index 944348d1..6b486502 100644 --- a/build/dev/deps/compose.yml +++ b/build/dev/deps/compose.yml @@ -14,39 +14,39 @@ services: - redis:/data # ============================== Log Store ============================== - # clickhouse: - # image: clickhouse/clickhouse-server:24-alpine - # environment: - # CLICKHOUSE_DB: outpost - # ports: - # # tcp - # - 9000:9000 - # # # http - # # - 8123:8123 - # # # postgresql - # # - 9005:9005 - # volumes: - # # optional to persist data locally - # - clickhouse:/var/lib/clickhouse/ - # # optional to add own config - # # - ./extra-config.xml:/etc/clickhouse-server/config.d/extra-config.xml - # # optional to add users or enable settings for a default user - # # - ./user.xml:/etc/clickhouse-server/users.d/user.xml - # # qol to mount own sql scripts to run them from inside container with - # # clickhouse client < /sql/myquery.sql - # # - ./sql:/sql - # # adjust mem_limit and cpus to machine - # # mem_limit: 12G - # # cpus: 4 - # ulimits: - # nofile: - # soft: 262144 - # hard: 262144 - # healthcheck: - # test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:8123/ping"] - # interval: 1s - # timeout: 1s - # retries: 30 + clickhouse: + image: clickhouse/clickhouse-server:24-alpine + environment: + CLICKHOUSE_DB: outpost + ports: + # tcp + - 9000:9000 + # # http + # - 8123:8123 + # # postgresql + # - 9005:9005 + volumes: + # optional to persist data locally + - clickhouse:/var/lib/clickhouse/ + # optional to add own config + # - ./extra-config.xml:/etc/clickhouse-server/config.d/extra-config.xml + # optional to add users or enable settings for a default user + # - ./user.xml:/etc/clickhouse-server/users.d/user.xml + # qol to mount own sql scripts to run them from inside container with + # clickhouse client < /sql/myquery.sql + # - ./sql:/sql + # adjust mem_limit and cpus to machine + # mem_limit: 12G + # cpus: 4 + ulimits: + nofile: + soft: 262144 + hard: 262144 + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:8123/ping"] + interval: 1s + timeout: 1s + retries: 30 postgres: image: postgres:16-alpine @@ -101,8 +101,8 @@ services: volumes: redis: driver: local - # clickhouse: - # driver: local + clickhouse: + driver: local postgres: driver: local rabbitmq: From 47d39f20429a979690d8d09e47114fb3780578df Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Mon, 8 Dec 2025 19:26:38 +0700 Subject: [PATCH 18/19] chore: gofmt --- cmd/e2e/log_test.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/e2e/log_test.go b/cmd/e2e/log_test.go index 6779ce35..90a95cf3 100644 --- a/cmd/e2e/log_test.go +++ b/cmd/e2e/log_test.go @@ -90,9 +90,9 @@ func (suite *basicSuite) TestLogAPI() { Method: httpclient.MethodPOST, Path: "/publish", Body: map[string]interface{}{ - "id": eventID, - "tenant_id": tenantID, - "topic": "user.created", + "id": eventID, + "tenant_id": tenantID, + "topic": "user.created", "eligible_for_retry": true, "data": map[string]interface{}{ "user_id": "123", @@ -136,10 +136,10 @@ func (suite *basicSuite) TestLogAPI() { "type": "object", "required": []interface{}{"id", "event", "destination", "status", "delivered_at"}, "properties": map[string]interface{}{ - "id": map[string]interface{}{"type": "string"}, - "event": map[string]interface{}{"type": "string"}, // Event ID when not expanded - "destination": map[string]interface{}{"const": destinationID}, - "status": map[string]interface{}{"type": "string"}, + "id": map[string]interface{}{"type": "string"}, + "event": map[string]interface{}{"type": "string"}, // Event ID when not expanded + "destination": map[string]interface{}{"const": destinationID}, + "status": map[string]interface{}{"type": "string"}, "delivered_at": map[string]interface{}{"type": "string"}, }, }, @@ -452,9 +452,9 @@ func (suite *basicSuite) TestRetryAPI() { Method: httpclient.MethodPOST, Path: "/publish", Body: map[string]interface{}{ - "id": eventID, - "tenant_id": tenantID, - "topic": "user.created", + "id": eventID, + "tenant_id": tenantID, + "topic": "user.created", "eligible_for_retry": false, // Disable auto-retry "data": map[string]interface{}{ "user_id": "456", @@ -727,9 +727,9 @@ func (suite *basicSuite) TestLegacyLogAPI() { Method: httpclient.MethodPOST, Path: "/publish", Body: map[string]interface{}{ - "id": eventID, - "tenant_id": tenantID, - "topic": "user.created", + "id": eventID, + "tenant_id": tenantID, + "topic": "user.created", "eligible_for_retry": true, "data": map[string]interface{}{ "user_id": "789", From 89fda09d15e1ca56e13a12d8d71afd70d7490080 Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Tue, 9 Dec 2025 20:25:30 +0700 Subject: [PATCH 19/19] chore: api event --- internal/apirouter/log_handlers.go | 19 ++++++++++++++++++- internal/apirouter/log_handlers_test.go | 3 ++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/internal/apirouter/log_handlers.go b/internal/apirouter/log_handlers.go index adf39018..39b5d0c1 100644 --- a/internal/apirouter/log_handlers.go +++ b/internal/apirouter/log_handlers.go @@ -84,6 +84,16 @@ type APIEventFull struct { Data map[string]interface{} `json:"data,omitempty"` } +// APIEvent is the API response for retrieving a single event +type APIEvent struct { + ID string `json:"id"` + Topic string `json:"topic"` + Time time.Time `json:"time"` + EligibleForRetry bool `json:"eligible_for_retry"` + Metadata map[string]string `json:"metadata,omitempty"` + Data map[string]interface{} `json:"data,omitempty"` +} + // ListDeliveriesResponse is the response for ListDeliveries type ListDeliveriesResponse struct { Data []APIDelivery `json:"data"` @@ -246,7 +256,14 @@ func (h *LogHandlers) RetrieveEvent(c *gin.Context) { AbortWithError(c, http.StatusNotFound, NewErrNotFound("event")) return } - c.JSON(http.StatusOK, event) + c.JSON(http.StatusOK, APIEvent{ + ID: event.ID, + Topic: event.Topic, + Time: event.Time, + EligibleForRetry: event.EligibleForRetry, + Metadata: event.Metadata, + Data: event.Data, + }) } // RetrieveDelivery handles GET /:tenantID/deliveries/:deliveryID diff --git a/internal/apirouter/log_handlers_test.go b/internal/apirouter/log_handlers_test.go index f0be8e66..d2b36707 100644 --- a/internal/apirouter/log_handlers_test.go +++ b/internal/apirouter/log_handlers_test.go @@ -362,10 +362,11 @@ func TestRetrieveEvent(t *testing.T) { require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) assert.Equal(t, eventID, response["id"]) - assert.Equal(t, tenantID, response["tenant_id"]) assert.Equal(t, "payment.processed", response["topic"]) assert.Equal(t, "stripe", response["metadata"].(map[string]interface{})["source"]) assert.Equal(t, 100.50, response["data"].(map[string]interface{})["amount"]) + // tenant_id is not included in API response (tenant-scoped via URL) + assert.Nil(t, response["tenant_id"]) }) t.Run("should return 404 for non-existent event", func(t *testing.T) {