chore: add metrics to count lookup failures per event for braze (rudderlabs#4875)

ItsSudip · web-flow · commit bf0bc5c2bbd0 · 2025-12-23T12:52:29.000+05:30
diff --git a/src/util/prometheus.js b/src/util/prometheus.js
@@ -635,13 +635,6 @@ class Prometheus {
         type: 'counter',
         labelNames: ['identifier_type', 'destination_id'],
       },
-      {
-        name: 'braze_lookup_failure_count',
-        help: 'braze look-up failure count',
-        type: 'counter',
-        labelNames: ['http_status', 'destination_id'],
-      },
-
       {
         name: 'braze_lookup_time',
         help: 'braze look-up time',
@@ -665,6 +658,20 @@ class Prometheus {
           500, 600, 700, 800, 900, 1000,
         ],
       },
+      {
+        name: 'braze_lookup_failure_identifiers',
+        help: 'Count of identifiers that failed to lookup due to API failure',
+        type: 'histogram',
+        labelNames: ['destination_id', 'http_status'],
+        buckets: [0, 10, 20, 30, 40, 50],
+      },
+      {
+        name: 'braze_lookup_success_identifiers',
+        help: 'Count of identifiers that successfully looked up due to API failure',
+        type: 'histogram',
+        labelNames: ['destination_id'],
+        buckets: [0, 10, 20, 30, 40, 50],
+      },
       {
         name: 'fb_custom_audience_event_having_all_null_field_values_for_a_user',
         help: 'fbcustomaudience event having all null field values for a user',
diff --git a/src/v0/destinations/braze/braze.util.test.js b/src/v0/destinations/braze/braze.util.test.js
@@ -205,6 +205,7 @@ describe('dedup utility tests', () => {
       // Mock the response from handleHttpRequest
       handleHttpRequest.mockResolvedValueOnce({
         processedResponse: {
+          status: 200,
           response: {
             users: [
               {
@@ -272,31 +273,34 @@ describe('dedup utility tests', () => {
       // Call the function
       const users = await BrazeDedupUtility.doApiLookup(identfierChunks, { destination });
 
-      // Check the result
+      // Check the result - now returns object with users and failedIdentifiers
       expect(users).toEqual([
-        [
-          {
-            external_id: 'user1',
-            email: 'user1@example.com',
-            custom_attributes: {
-              key1: 'value1',
+        {
+          users: [
+            {
+              external_id: 'user1',
+              email: 'user1@example.com',
+              custom_attributes: {
+                key1: 'value1',
+              },
             },
-          },
-          {
-            external_id: 'user2',
-            email: 'user2@example.com',
-            custom_attributes: {
-              key2: 'value2',
+            {
+              external_id: 'user2',
+              email: 'user2@example.com',
+              custom_attributes: {
+                key2: 'value2',
+              },
             },
-          },
-          {
-            user_aliases: [{ alias_name: 'user3', alias_label: 'rudder_id' }],
-            email: 'user3@example.com',
-            custom_attributes: {
-              key2: 'value3',
+            {
+              user_aliases: [{ alias_name: 'user3', alias_label: 'rudder_id' }],
+              email: 'user3@example.com',
+              custom_attributes: {
+                key2: 'value3',
+              },
             },
-          },
-        ],
+          ],
+          failedIdentifiers: [],
+        },
       ]);
 
       // Check that handleHttpRequest was called once with the correct arguments
@@ -365,6 +369,7 @@ describe('dedup utility tests', () => {
       // Mock the handleHttpRequest function to return the same data every time it's called
       handleHttpRequest.mockImplementationOnce(() => ({
         processedResponse: {
+          status: 200,
           response: {
             users: Array.from({ length: 50 }, (_, i) =>
               removeUndefinedAndNullAndEmptyValues({
@@ -385,6 +390,7 @@ describe('dedup utility tests', () => {
 
       handleHttpRequest.mockImplementationOnce(() => ({
         processedResponse: {
+          status: 200,
           response: {
             users: Array.from({ length: 50 }, (_, i) =>
               removeUndefinedAndNullAndEmptyValues({
@@ -405,6 +411,7 @@ describe('dedup utility tests', () => {
 
       handleHttpRequest.mockImplementationOnce(() => ({
         processedResponse: {
+          status: 200,
           response: {
             users: Array.from({ length: 10 }, (_, i) =>
               removeUndefinedAndNullAndEmptyValues({
@@ -426,8 +433,10 @@ describe('dedup utility tests', () => {
       const chunkedUserData = await BrazeDedupUtility.doApiLookup(identifierChunks, {
         destination,
       });
-      const result = _.flatMap(chunkedUserData);
-      expect(result).toHaveLength(110);
+      // Each chunk now returns { users: [...], failedIdentifiers: [] }
+      // So we need to extract users from each chunk and flatten
+      const allUsers = chunkedUserData.flatMap((chunk) => chunk.users);
+      expect(allUsers).toHaveLength(110);
       expect(handleHttpRequest).toHaveBeenCalledTimes(3);
     });
 
@@ -486,14 +495,20 @@ describe('dedup utility tests', () => {
 
       expect(handleHttpRequest).toHaveBeenCalledTimes(2);
       // Assert that the first chunk was successful and the second failed
-      // The failed chunked will be returned as undefined
+      // The failed chunk will return empty users array with failedIdentifiers
       expect(users).toEqual([
-        [
-          { external_id: 'user1', email: 'user1@example.com' },
-          { alias_name: 'alias1', alias_label: 'rudder_id', email: 'alias1@example.com' },
-          { external_id: 'user2', email: 'user2@example.com' },
-        ],
-        undefined,
+        {
+          users: [
+            { external_id: 'user1', email: 'user1@example.com' },
+            { alias_name: 'alias1', alias_label: 'rudder_id', email: 'alias1@example.com' },
+            { external_id: 'user2', email: 'user2@example.com' },
+          ],
+          failedIdentifiers: [],
+        },
+        {
+          users: [],
+          failedIdentifiers: ['user3', 'alias2'],
+        },
       ]);
     });
   });
@@ -515,11 +530,21 @@ describe('dedup utility tests', () => {
           [{ alias_name: 'alias1', alias_label: 'rudder_id' }],
           [{ alias_name: 'alias2', alias_label: 'rudder_id' }],
         ]);
+      // doApiLookup now returns { users: [...], failedIdentifiers: [...] } for each chunk
       const doApiLookupMock = jest.spyOn(BrazeDedupUtility, 'doApiLookup').mockResolvedValue([
-        [{ external_id: '123', custom_attributes: { key1: 'value1' } }],
-        [{ external_id: '456', custom_attributes: { key2: 'value2' } }],
-        undefined, // simulate failed api call
-        [{ alias_name: 'alias2', custom_attributes: { key3: 'value3' } }],
+        {
+          users: [{ external_id: '123', custom_attributes: { key1: 'value1' } }],
+          failedIdentifiers: [],
+        },
+        {
+          users: [{ external_id: '456', custom_attributes: { key2: 'value2' } }],
+          failedIdentifiers: [],
+        },
+        { users: [], failedIdentifiers: ['alias1'] }, // simulate failed api call
+        {
+          users: [{ alias_name: 'alias2', custom_attributes: { key3: 'value3' } }],
+          failedIdentifiers: [],
+        },
       ]);
 
       // create input data for doLookup
@@ -532,12 +557,13 @@ describe('dedup utility tests', () => {
 
       // call doLookup and verify the output
       const result = await BrazeDedupUtility.doLookup(inputs);
-      expect(result).toEqual([
+      // doLookup now returns { users: [...], failedIdentifiers: Set }
+      expect(result.users).toEqual([
         { external_id: '123', custom_attributes: { key1: 'value1' } },
         { external_id: '456', custom_attributes: { key2: 'value2' } },
-        undefined, // response of failed api call
         { alias_name: 'alias2', custom_attributes: { key3: 'value3' } },
       ]);
+      expect(result.failedIdentifiers).toEqual(new Set(['alias1']));
 
       // verify that the mocked functions were called with correct arguments
       expect(prepareInputForDedupMock).toHaveBeenCalledWith(inputs);
diff --git a/src/v0/destinations/braze/transform.js b/src/v0/destinations/braze/transform.js
@@ -251,6 +251,7 @@ function processTrackWithUserAttributes(
         processParams.userStore,
         payload,
         destination.ID,
+        processParams.failedLookupIdentifiers,
       );
       if (dedupedAttributePayload) {
         requestJson.attributes = [dedupedAttributePayload];
@@ -300,6 +301,7 @@ function processTrackEvent(messageType, message, destination, mappingJson, proce
         processParams.userStore,
         attributePayload,
         destination.ID,
+        processParams.failedLookupIdentifiers,
       );
       if (dedupedAttributePayload) {
         requestJson.attributes = [dedupedAttributePayload];
@@ -534,16 +536,20 @@ async function process(event, processParams = { userStore: new Map() }, reqMetad
 
 const processRouterDest = async (inputs, reqMetadata) => {
   const userStore = new Map();
+  let failedLookupIdentifiers = new Set();
   const { destination } = inputs[0];
   if (destination.Config.supportDedup) {
-    let lookedUpUsers;
+    let lookupResult;
     try {
-      lookedUpUsers = await BrazeDedupUtility.doLookup(inputs);
+      lookupResult = await BrazeDedupUtility.doLookup(inputs);
     } catch (error) {
       logger.error('Error while fetching user store', error);
     }
 
-    BrazeDedupUtility.updateUserStore(userStore, lookedUpUsers, destination.ID);
+    if (lookupResult) {
+      BrazeDedupUtility.updateUserStore(userStore, lookupResult.users, destination.ID);
+      failedLookupIdentifiers = lookupResult.failedIdentifiers || new Set();
+    }
   }
   // group events by userId or anonymousId and then call process
   const groupedInputs = lodash.groupBy(
@@ -563,6 +569,7 @@ const processRouterDest = async (inputs, reqMetadata) => {
     const respList = await simpleProcessRouterDestFunc(groupedInputs[id], process, reqMetadata, {
       userStore,
       identifyCallsArray,
+      failedLookupIdentifiers,
     });
     return respList;
   });
diff --git a/src/v0/destinations/braze/util.js b/src/v0/destinations/braze/util.js
@@ -257,13 +257,29 @@ const BrazeDedupUtility = {
             metadata,
           },
         );
-        stats.counter('braze_lookup_failure_count', 1, {
-          http_status: lookUpResponse.status,
-          destination_id: destination.ID,
-        });
-        const { users } = lookUpResponse.response;
 
-        return users;
+        // Track failed lookups and collect failed identifiers for non-2xx responses
+        if (!isHttpStatusSuccess(lookUpResponse.status)) {
+          // Collect failed identifiers (external_ids and alias_names)
+          const failedIdentifiers = [
+            ...externalIdentifiers.map((id) => id.external_id),
+            ...aliasIdentifiers.map((id) => id.alias_name),
+          ];
+          stats.histogram('braze_lookup_failure_identifiers', failedIdentifiers.length, {
+            http_status: lookUpResponse.status,
+            destination_id: destination.ID,
+          });
+          return { users: [], failedIdentifiers };
+        }
+        stats.histogram(
+          'braze_lookup_success_identifiers',
+          externalIdentifiers.length + aliasIdentifiers.length,
+          {
+            destination_id: destination.ID,
+          },
+        );
+        const { users } = lookUpResponse.response;
+        return { users: users || [], failedIdentifiers: [] };
       }),
     );
   },
@@ -273,24 +289,37 @@ const BrazeDedupUtility = {
    * uses the external_id field and the alias_name field to lookup users
    *
    * @param {*} inputs router transform input events array
-   * @returns {Promise<Array>} array of braze user objects
+   * @returns {Promise<{users: Array, failedIdentifiers: Set}>} object containing user objects and failed identifiers
    */
   async doLookup(inputs) {
     const lookupStartTime = new Date();
     const { destination, metadata } = inputs[0];
     const { externalIdsToQuery, aliasIdsToQuery } = this.prepareInputForDedup(inputs);
     const identfierChunks = this.prepareChunksForDedup(externalIdsToQuery, aliasIdsToQuery);
-    const chunkedUserData = await this.doApiLookup(identfierChunks, { destination, metadata });
+    const chunkedResults = await this.doApiLookup(identfierChunks, { destination, metadata });
+
+    // Collect all users and failed identifiers from all chunks
+    const allUsers = [];
+    const failedIdentifiers = new Set();
+    chunkedResults.forEach((result) => {
+      if (result.users) {
+        allUsers.push(...result.users);
+      }
+      if (result.failedIdentifiers) {
+        result.failedIdentifiers.forEach((id) => failedIdentifiers.add(id));
+      }
+    });
+
     stats.timing('braze_lookup_time', lookupStartTime, {
       destination_id: destination.ID,
     });
-    stats.histogram('braze_lookup_count', chunkedUserData.length, {
+    stats.histogram('braze_lookup_count', chunkedResults.length, {
       destination_id: destination.ID,
     });
     stats.histogram('braze_lookup_user_count', externalIdsToQuery.length + aliasIdsToQuery.length, {
       destination_id: destination.ID,
     });
-    return _.flatMap(chunkedUserData);
+    return { users: allUsers, failedIdentifiers };
   },
 
   /**
@@ -424,9 +453,18 @@ const BrazeDedupUtility = {
  * @param {*} userStore
  * @param {*} payload
  * @param {*} destinationId
+ * @param {Set} failedLookupIdentifiers - Set of identifiers that failed to lookup due to API failure
  * @returns
  */
-const processDeduplication = (userStore, payload, destinationId) => {
+const processDeduplication = (userStore, payload, destinationId, failedLookupIdentifiers) => {
+  // Check if this event's identifier failed to lookup due to API failure
+  const identifier = payload.external_id || payload.user_alias?.alias_name;
+  if (failedLookupIdentifiers && failedLookupIdentifiers.has(identifier)) {
+    stats.increment('braze_dedup_skipped_due_to_lookup_failure_count', {
+      destination_id: destinationId,
+    });
+  }
+
   const dedupedAttributePayload = BrazeDedupUtility.deduplicate(payload, userStore);
   if (
     isDefinedAndNotNullAndNotEmpty(dedupedAttributePayload) &&
