[WIP] Removes delete concurrency

README.md

@@ -2265,8 +2265,8 @@ FLAGS
      [--directory]         Path to the directory of CSV files to load preferences from
      [--transcendUrl]      URL of the Transcend backend. Use https://api.us.transcend.io for US hosting                                                       [default = https://api.transcend.io]
      [--maxItemsInChunk]   When chunking, how many items to delete in a single chunk (higher = faster, but more load).                                        [default = 10]
-     [--maxConcurrency]    Number of concurrent requests to make when deleting preference records. (Higher = faster, but more load and rate limiting errors). [default = 10]
-     [--fileConcurrency]   Number of files to process concurrently when deleting preference records from multiple files.                                      [default = 5]
+     [--maxConcurrency]    Number of concurrent requests to make when deleting preference records. (Higher = faster, but more load and rate limiting errors). [default = 1]
+     [--fileConcurrency]   Number of files to process concurrently when deleting preference records from multiple files.                                      [default = 1]
      [--receiptDirectory]  Directory to write receipts of failed deletions to.                                                                                [default = ./receipts]
   -h  --help               Print help information and exit
 ```

package.json

@@ -2,7 +2,7 @@
   "author": "Transcend Inc.",
   "name": "@transcend-io/cli",
   "description": "A command line interface for programmatic operations across Transcend.",
-  "version": "8.32.1",
+  "version": "8.32.2",
   "homepage": "https://github.com/transcend-io/cli",
   "repository": {
     "type": "git",

src/commands/consent/delete-preference-records/command.ts

@@ -55,14 +55,14 @@ export const deletePreferenceRecordsCommand = buildCommand({
         parse: numberParser,
         brief:
           'Number of concurrent requests to make when deleting preference records. (Higher = faster, but more load and rate limiting errors).',
-        default: '10',
+        default: '1',
       },
       fileConcurrency: {
         kind: 'parsed',
         parse: numberParser,
         brief:
           'Number of files to process concurrently when deleting preference records from multiple files.',
-        default: '5',
+        default: '1',
       },
       receiptDirectory: {
         kind: 'parsed',

src/lib/graphql/createSombraGotInstance.ts

@@ -46,7 +46,7 @@ export async function createSombraGotInstance(
   }
   // Create got instance with default values
   return got.extend({
-    prefixUrl: customerUrl,
+    prefixUrl: process.env.SOMBRA_URL || customerUrl,
     headers: {
       Authorization: `Bearer ${transcendApiKey}`,
       ...(sombraApiKey

src/lib/preference-management/bulkDeletePreferenceRecords.ts

@@ -1,4 +1,5 @@
 import { decodeCodec } from '@transcend-io/type-utils';
+import { uniq, chunk, keyBy } from 'lodash-es';
 import colors from 'colors';
 import type { Got } from 'got';
 import { logger } from '../../logger';
@@ -7,9 +8,10 @@ import {
   DeletePreferenceRecordsResponse,
 } from './codecs';
 import { readCsv } from '../requests';
-import { chunk } from 'lodash-es';
 import { map } from '../bluebird';
 import { withPreferenceRetry } from './withPreferenceRetry';
+import { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers';
+import type { PreferenceQueryResponseItem } from '@transcend-io/privacy-types';
 
 interface FailedResult extends DeletePreferenceRecordCliCsvRow {
   /** Error message describing the failure */
@@ -74,9 +76,9 @@ async function deletePreferenceRecordsRepository(
           })
           .json(),
       {
-        maxAttempts: 3,
+        maxAttempts: 5,
         onRetry: (attempt, err, msg) => {
-          logger.debug(
+          logger.warn(
             colors.yellow(
               `Attempt ${attempt} to delete preference records failed: ${msg}`,
             ),
@@ -120,8 +122,49 @@ export async function bulkDeletePreferenceRecords(
     maxConcurrency,
   }: DeletePreferenceRecordsOptions,
 ): Promise<FailedResult[]> {
+  // Determine identifiers to delete
   const anchorIdentifiers = readCsv(filePath, DeletePreferenceRecordCliCsvRow);
-  const chunks = chunk(anchorIdentifiers, maxItemsInChunk);
+
+  // Fetch existing records
+  // FIXME progress bar in this conflicts with progress bar one level higher
+  const existingRecords = await getPreferencesForIdentifiers(sombra, {
+    identifiers: anchorIdentifiers,
+    partitionKey: partition,
+  });
+  const anchorNames = uniq(anchorIdentifiers.map((id) => id.name));
+
+  logger.info(
+    colors.magenta(
+      `Found ${existingRecords.length} existing preference records to delete ` +
+        `out of ${
+          anchorIdentifiers.length
+        } identifiers provided. Using anchors: ${anchorNames.join(', ')}`,
+    ),
+  );
+
+  // Create a lookup of records in db
+  const recordExists = anchorNames.reduce<
+    Record<string, Record<string, PreferenceQueryResponseItem>>
+  >((acc, anchorName) => {
+    acc[anchorName] = keyBy(
+      existingRecords,
+      (record) =>
+        record.identifiers?.find((id) => id.name === anchorName)?.value || '',
+    );
+    return acc;
+  }, {} as Record<string, Record<string, PreferenceQueryResponseItem>>);
+
+  // Filter identifiers to only those that exist
+  const identifiersToDelete = anchorIdentifiers.filter(
+    (identifier) => recordExists[identifier.name]?.[identifier.value],
+  );
+  if (identifiersToDelete.length !== existingRecords.length) {
+    throw new Error(
+      `Mismatch in existing records found (${existingRecords.length}) ` +
+        `and identifiers to delete (${identifiersToDelete.length})`,
+    );
+  }
+  const chunks = chunk(identifiersToDelete, maxItemsInChunk);
 
   const failedResults = await map(
     chunks,

src/lib/preference-management/withPreferenceRetry.ts

@@ -12,6 +12,7 @@ export const RETRY_PREFERENCE_MSGS: string[] = [
   'ETIMEDOUT',
   '502 Bad Gateway',
   '504 Gateway Time-out',
+  'Rate limit exceeded',
   'Task timed out after',
   'unknown request error',
 ].map((s) => s.toLowerCase());