diff --git a/README.md b/README.md index 4e9b56ea..9009e1c3 100644 --- a/README.md +++ b/README.md @@ -2265,8 +2265,8 @@ FLAGS [--directory] Path to the directory of CSV files to load preferences from [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] [--maxItemsInChunk] When chunking, how many items to delete in a single chunk (higher = faster, but more load). [default = 10] - [--maxConcurrency] Number of concurrent requests to make when deleting preference records. (Higher = faster, but more load and rate limiting errors). [default = 10] - [--fileConcurrency] Number of files to process concurrently when deleting preference records from multiple files. [default = 5] + [--maxConcurrency] Number of concurrent requests to make when deleting preference records. (Higher = faster, but more load and rate limiting errors). [default = 1] + [--fileConcurrency] Number of files to process concurrently when deleting preference records from multiple files. [default = 1] [--receiptDirectory] Directory to write receipts of failed deletions to. [default = ./receipts] -h --help Print help information and exit ``` diff --git a/package.json b/package.json index b8cbf37d..a525300b 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "author": "Transcend Inc.", "name": "@transcend-io/cli", "description": "A command line interface for programmatic operations across Transcend.", - "version": "8.32.1", + "version": "8.32.2", "homepage": "https://github.com/transcend-io/cli", "repository": { "type": "git", diff --git a/src/commands/consent/delete-preference-records/command.ts b/src/commands/consent/delete-preference-records/command.ts index f8652fb0..0708daba 100644 --- a/src/commands/consent/delete-preference-records/command.ts +++ b/src/commands/consent/delete-preference-records/command.ts @@ -55,14 +55,14 @@ export const deletePreferenceRecordsCommand = buildCommand({ parse: numberParser, brief: 'Number of concurrent requests to make when deleting preference records. (Higher = faster, but more load and rate limiting errors).', - default: '10', + default: '1', }, fileConcurrency: { kind: 'parsed', parse: numberParser, brief: 'Number of files to process concurrently when deleting preference records from multiple files.', - default: '5', + default: '1', }, receiptDirectory: { kind: 'parsed', diff --git a/src/lib/graphql/createSombraGotInstance.ts b/src/lib/graphql/createSombraGotInstance.ts index 90de20d8..4f667ea4 100644 --- a/src/lib/graphql/createSombraGotInstance.ts +++ b/src/lib/graphql/createSombraGotInstance.ts @@ -46,7 +46,7 @@ export async function createSombraGotInstance( } // Create got instance with default values return got.extend({ - prefixUrl: customerUrl, + prefixUrl: process.env.SOMBRA_URL || customerUrl, headers: { Authorization: `Bearer ${transcendApiKey}`, ...(sombraApiKey diff --git a/src/lib/preference-management/bulkDeletePreferenceRecords.ts b/src/lib/preference-management/bulkDeletePreferenceRecords.ts index bd3bc574..08a575fa 100644 --- a/src/lib/preference-management/bulkDeletePreferenceRecords.ts +++ b/src/lib/preference-management/bulkDeletePreferenceRecords.ts @@ -1,4 +1,5 @@ import { decodeCodec } from '@transcend-io/type-utils'; +import { uniq, chunk, keyBy } from 'lodash-es'; import colors from 'colors'; import type { Got } from 'got'; import { logger } from '../../logger'; @@ -7,9 +8,10 @@ import { DeletePreferenceRecordsResponse, } from './codecs'; import { readCsv } from '../requests'; -import { chunk } from 'lodash-es'; import { map } from '../bluebird'; import { withPreferenceRetry } from './withPreferenceRetry'; +import { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers'; +import type { PreferenceQueryResponseItem } from '@transcend-io/privacy-types'; interface FailedResult extends DeletePreferenceRecordCliCsvRow { /** Error message describing the failure */ @@ -74,9 +76,9 @@ async function deletePreferenceRecordsRepository( }) .json(), { - maxAttempts: 3, + maxAttempts: 5, onRetry: (attempt, err, msg) => { - logger.debug( + logger.warn( colors.yellow( `Attempt ${attempt} to delete preference records failed: ${msg}`, ), @@ -120,8 +122,49 @@ export async function bulkDeletePreferenceRecords( maxConcurrency, }: DeletePreferenceRecordsOptions, ): Promise { + // Determine identifiers to delete const anchorIdentifiers = readCsv(filePath, DeletePreferenceRecordCliCsvRow); - const chunks = chunk(anchorIdentifiers, maxItemsInChunk); + + // Fetch existing records + // FIXME progress bar in this conflicts with progress bar one level higher + const existingRecords = await getPreferencesForIdentifiers(sombra, { + identifiers: anchorIdentifiers, + partitionKey: partition, + }); + const anchorNames = uniq(anchorIdentifiers.map((id) => id.name)); + + logger.info( + colors.magenta( + `Found ${existingRecords.length} existing preference records to delete ` + + `out of ${ + anchorIdentifiers.length + } identifiers provided. Using anchors: ${anchorNames.join(', ')}`, + ), + ); + + // Create a lookup of records in db + const recordExists = anchorNames.reduce< + Record> + >((acc, anchorName) => { + acc[anchorName] = keyBy( + existingRecords, + (record) => + record.identifiers?.find((id) => id.name === anchorName)?.value || '', + ); + return acc; + }, {} as Record>); + + // Filter identifiers to only those that exist + const identifiersToDelete = anchorIdentifiers.filter( + (identifier) => recordExists[identifier.name]?.[identifier.value], + ); + if (identifiersToDelete.length !== existingRecords.length) { + throw new Error( + `Mismatch in existing records found (${existingRecords.length}) ` + + `and identifiers to delete (${identifiersToDelete.length})`, + ); + } + const chunks = chunk(identifiersToDelete, maxItemsInChunk); const failedResults = await map( chunks, diff --git a/src/lib/preference-management/withPreferenceRetry.ts b/src/lib/preference-management/withPreferenceRetry.ts index 85f9021f..510a92d8 100644 --- a/src/lib/preference-management/withPreferenceRetry.ts +++ b/src/lib/preference-management/withPreferenceRetry.ts @@ -12,6 +12,7 @@ export const RETRY_PREFERENCE_MSGS: string[] = [ 'ETIMEDOUT', '502 Bad Gateway', '504 Gateway Time-out', + 'Rate limit exceeded', 'Task timed out after', 'unknown request error', ].map((s) => s.toLowerCase());