From dd24f45fa881162b7bfc872b5de47f25ff7fa2a1 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 20:49:03 -0400 Subject: [PATCH 1/4] formatting --- .devcontainer/docker-compose.yml | 6 +- CHANGELOG.md | 115 ++-- DEVELOPERS.md | 14 +- README.md | 555 +++++++++--------- scripts/buildReadmeDocs.ts | 13 +- src/codecs.ts | 100 ++-- src/commands/admin/generate-api-keys/impl.ts | 10 +- .../consent/build-xdi-sync-endpoint/impl.ts | 6 +- .../consent/pull-consent-metrics/impl.ts | 16 +- .../consent/pull-consent-preferences/impl.ts | 1 - .../consent/update-consent-manager/impl.ts | 9 +- .../upload-consent-preferences/impl.ts | 3 +- .../consent/upload-cookies-from-csv/impl.ts | 2 +- .../upload-data-flows-from-csv/impl.ts | 2 +- .../consent/upload-preferences/impl.ts | 11 +- .../command.ts | 2 +- .../impl.ts | 18 +- .../impl.ts | 7 +- .../impl.ts | 18 +- .../derive-data-silos-from-data-flows/impl.ts | 16 +- src/commands/inventory/discover-silos/impl.ts | 12 +- .../inventory/pull-datapoints/impl.ts | 24 +- .../pull-unstructured-discovery-files/impl.ts | 2 +- src/commands/inventory/pull/command.ts | 2 +- src/commands/inventory/pull/impl.ts | 22 +- src/commands/inventory/push/impl.ts | 18 +- src/commands/inventory/routes.ts | 4 +- src/commands/inventory/scan-packages/impl.ts | 6 +- src/commands/migration/sync-ot/command.ts | 4 +- src/commands/migration/sync-ot/impl.ts | 6 +- src/commands/request/approve/impl.ts | 3 +- src/commands/request/cancel/impl.ts | 2 +- .../request/cron/pull-identifiers/command.ts | 2 +- .../request/cron/pull-identifiers/impl.ts | 7 +- .../request/cron/pull-profiles/command.ts | 2 +- .../request/cron/pull-profiles/impl.ts | 12 +- src/commands/request/download-files/impl.ts | 2 +- src/commands/request/enricher-restart/impl.ts | 4 +- src/commands/request/export/impl.ts | 9 +- src/commands/request/mark-silent/impl.ts | 2 +- .../request/notify-additional-time/impl.ts | 2 +- .../preflight/pull-identifiers/impl.ts | 2 +- .../reject-unverified-identifiers/impl.ts | 2 +- src/commands/request/restart/impl.ts | 2 +- .../mark-request-data-silos-completed/impl.ts | 5 +- .../system/retry-request-data-silos/impl.ts | 2 +- .../system/skip-request-data-silos/impl.ts | 2 +- src/constants.ts | 2 +- src/lib/ai/TranscendPromptManager.ts | 54 +- src/lib/ai/getGitFilesThatChanged.ts | 4 +- .../api-keys/generateCrossAccountApiKeys.ts | 16 +- src/lib/api-keys/validateTranscendAuth.ts | 4 +- src/lib/cli/common-parameters.ts | 4 +- src/lib/code-scanning/constants.ts | 10 +- .../code-scanning/findCodePackagesInFolder.ts | 6 +- .../code-scanning/integrations/cocoaPods.ts | 6 +- .../integrations/composerJson.ts | 4 +- src/lib/code-scanning/integrations/gemfile.ts | 6 +- src/lib/code-scanning/integrations/gradle.ts | 4 +- .../integrations/javascriptPackageJson.ts | 4 +- src/lib/code-scanning/integrations/pubspec.ts | 12 +- .../integrations/pythonRequirementsTxt.ts | 6 +- src/lib/code-scanning/integrations/swift.ts | 4 +- .../consent-manager/buildXdiSyncEndpoint.ts | 5 +- .../consent-manager/dataFlowsToDataSilos.ts | 2 +- .../fetchConsentPreferences.ts | 2 +- .../updateConsentManagerVersionToLatest.ts | 11 +- src/lib/consent-manager/uploadConsents.ts | 14 +- .../consent-manager/uploadCookiesFromCsv.ts | 8 +- .../consent-manager/uploadDataFlowsFromCsv.ts | 8 +- .../cron/markRequestDataSiloIdsCompleted.ts | 10 +- ...ChunkedCustomSiloOutstandingIdentifiers.ts | 15 +- src/lib/cron/pullCronPageOfIdentifiers.ts | 4 +- .../pullCustomSiloOutstandingIdentifiers.ts | 14 +- src/lib/cron/pushCronIdentifiersFromCsv.ts | 14 +- src/lib/cron/writeCsv.ts | 5 +- src/lib/data-inventory/pullAllDatapoints.ts | 18 +- ...UnstructuredSubDataPointRecommendations.ts | 2 +- src/lib/graphql/addMessagesToPromptRun.ts | 2 +- src/lib/graphql/createSombraGotInstance.ts | 2 +- src/lib/graphql/deployConsentManager.ts | 2 +- .../graphql/fetchAllActionItemCollections.ts | 2 +- src/lib/graphql/fetchAllActionItems.ts | 6 +- src/lib/graphql/fetchAllActions.ts | 2 +- src/lib/graphql/fetchAllAgentFiles.ts | 2 +- src/lib/graphql/fetchAllAgentFunctions.ts | 2 +- src/lib/graphql/fetchAllAgents.ts | 2 +- .../graphql/fetchAllAssessmentTemplates.ts | 12 +- src/lib/graphql/fetchAllAssessments.ts | 6 +- src/lib/graphql/fetchAllAttributes.ts | 9 +- src/lib/graphql/fetchAllBusinessEntities.ts | 6 +- src/lib/graphql/fetchAllCodePackages.ts | 2 +- src/lib/graphql/fetchAllCookies.ts | 8 +- src/lib/graphql/fetchAllDataCategories.ts | 2 +- src/lib/graphql/fetchAllDataFlows.ts | 10 +- src/lib/graphql/fetchAllMessages.ts | 2 +- src/lib/graphql/fetchAllPolicies.ts | 4 +- src/lib/graphql/fetchAllPreferenceTopics.ts | 2 +- src/lib/graphql/fetchAllPrivacyCenters.ts | 6 +- src/lib/graphql/fetchAllProcessingPurposes.ts | 2 +- src/lib/graphql/fetchAllPurposes.ts | 8 +- .../graphql/fetchAllPurposesAndPreferences.ts | 4 +- src/lib/graphql/fetchAllRequestEnrichers.ts | 2 +- src/lib/graphql/fetchAllRequestIdentifiers.ts | 1 - src/lib/graphql/fetchAllRequests.ts | 20 +- .../fetchAllSoftwareDevelopmentKits.ts | 2 +- src/lib/graphql/fetchAllTeams.ts | 2 +- src/lib/graphql/fetchAllVendors.ts | 6 +- src/lib/graphql/fetchApiKeys.ts | 8 +- src/lib/graphql/fetchConsentManagerId.ts | 30 +- src/lib/graphql/fetchDataSubjects.ts | 8 +- src/lib/graphql/fetchIdentifiers.ts | 8 +- src/lib/graphql/fetchLargeLanguageModels.ts | 2 +- src/lib/graphql/fetchPrompts.ts | 8 +- src/lib/graphql/fetchRequestDataSilo.ts | 10 +- src/lib/graphql/gqls/actionItemCollection.ts | 1 + src/lib/graphql/loginUser.ts | 2 +- src/lib/graphql/makeGraphQLRequest.ts | 2 +- src/lib/graphql/manageApiKeys.ts | 2 +- src/lib/graphql/pullTranscendConfiguration.ts | 140 ++--- src/lib/graphql/reportPromptRun.ts | 10 +- src/lib/graphql/setResourceAttributes.ts | 2 +- src/lib/graphql/syncAction.ts | 10 +- src/lib/graphql/syncActionItemCollections.ts | 23 +- src/lib/graphql/syncActionItems.ts | 14 +- src/lib/graphql/syncAgentFiles.ts | 12 +- src/lib/graphql/syncAgentFunctions.ts | 14 +- src/lib/graphql/syncAgents.ts | 12 +- src/lib/graphql/syncAttribute.ts | 10 +- src/lib/graphql/syncBusinessEntities.ts | 14 +- src/lib/graphql/syncCodePackages.ts | 14 +- .../graphql/syncConfigurationToTranscend.ts | 58 +- src/lib/graphql/syncConsentManager.ts | 42 +- src/lib/graphql/syncCookies.ts | 10 +- src/lib/graphql/syncDataCategories.ts | 14 +- src/lib/graphql/syncDataFlows.ts | 12 +- src/lib/graphql/syncDataSilos.ts | 46 +- src/lib/graphql/syncDataSubject.ts | 4 +- src/lib/graphql/syncEnrichers.ts | 8 +- src/lib/graphql/syncIdentifier.ts | 4 +- src/lib/graphql/syncIntlMessages.ts | 8 +- src/lib/graphql/syncPartitions.ts | 8 +- src/lib/graphql/syncPolicies.ts | 12 +- src/lib/graphql/syncPrivacyCenter.ts | 6 +- src/lib/graphql/syncProcessingPurposes.ts | 18 +- src/lib/graphql/syncPromptGroups.ts | 10 +- src/lib/graphql/syncPromptPartials.ts | 10 +- src/lib/graphql/syncPrompts.ts | 10 +- src/lib/graphql/syncRepositories.ts | 10 +- .../graphql/syncSoftwareDevelopmentKits.ts | 18 +- src/lib/graphql/syncTeams.ts | 12 +- src/lib/graphql/syncTemplates.ts | 4 +- src/lib/graphql/syncVendors.ts | 12 +- src/lib/graphql/uploadSiloDiscoveryResults.ts | 4 +- src/lib/helpers/buildEnabledRouteType.ts | 2 +- src/lib/helpers/inquirer.ts | 2 +- .../manual-enrichment/enrichPrivacyRequest.ts | 4 +- .../pullManualEnrichmentIdentifiersToCsv.ts | 10 +- .../pushManualEnrichmentIdentifiersFromCsv.ts | 8 +- src/lib/mergeTranscendInputs.ts | 2 +- .../endpoints/getListOfOneTrustAssessments.ts | 6 +- .../endpoints/getOneTrustAssessment.ts | 4 +- src/lib/oneTrust/endpoints/getOneTrustRisk.ts | 4 +- src/lib/oneTrust/endpoints/getOneTrustUser.ts | 4 +- .../helpers/parseCliSyncOtArguments.ts | 2 +- .../helpers/syncOneTrustAssessmentToDisk.ts | 6 +- .../syncOneTrustAssessmentToTranscend.ts | 6 +- .../syncOneTrustAssessmentsFromFile.ts | 9 +- .../syncOneTrustAssessmentsFromOneTrust.ts | 22 +- .../tests/convertToEmptyStrings.test.ts | 3 +- .../getPreferenceUpdatesFromRow.ts | 2 +- .../getPreferencesForIdentifiers.ts | 10 +- .../parsePreferenceAndPurposeValuesFromCsv.ts | 6 +- .../parsePreferenceIdentifiersFromCsv.ts | 4 +- .../parsePreferenceManagementCsv.ts | 16 +- .../parsePreferenceTimestampsFromCsv.ts | 4 +- ...kIfPendingPreferenceUpdatesAreNoOp.test.ts | 5 +- ...dingPreferenceUpdatesCauseConflict.test.ts | 5 +- .../tests/getPreferenceUpdatesFromRow.test.ts | 5 +- ...ferenceManagementPreferencesInteractive.ts | 24 +- src/lib/readTranscendYaml.ts | 2 +- src/lib/requests/approvePrivacyRequests.ts | 16 +- src/lib/requests/bulkRestartRequests.ts | 6 +- src/lib/requests/bulkRetryEnrichers.ts | 4 +- src/lib/requests/cancelPrivacyRequests.ts | 14 +- src/lib/requests/constants.ts | 4 +- .../requests/downloadPrivacyRequestFiles.ts | 12 +- src/lib/requests/filterRows.ts | 2 +- src/lib/requests/fuzzyMatchColumns.ts | 5 +- .../getFileMetadataForPrivacyRequests.ts | 13 +- src/lib/requests/mapColumnsToAttributes.ts | 2 +- src/lib/requests/mapColumnsToIdentifiers.ts | 4 +- src/lib/requests/mapCsvColumnsToApi.ts | 6 +- src/lib/requests/mapCsvRowsToRequestInputs.ts | 24 +- src/lib/requests/mapEnumValues.ts | 2 +- src/lib/requests/mapRequestEnumValues.ts | 16 +- src/lib/requests/markSilentPrivacyRequests.ts | 12 +- .../notifyPrivacyRequestsAdditionalTime.ts | 14 +- src/lib/requests/pullPrivacyRequests.ts | 9 +- src/lib/requests/readCsv.ts | 5 +- .../removeUnverifiedRequestIdentifiers.ts | 14 +- src/lib/requests/restartPrivacyRequest.ts | 6 +- src/lib/requests/retryRequestDataSilos.ts | 14 +- src/lib/requests/skipPreflightJobs.ts | 16 +- src/lib/requests/skipRequestDataSilos.ts | 12 +- src/lib/requests/streamPrivacyRequestFiles.ts | 4 +- src/lib/requests/submitPrivacyRequest.ts | 10 +- .../requests/tests/fuzzyMatchColumns.test.ts | 3 +- .../tests/getUniqueValuesForColumn.test.ts | 3 +- .../tests/parseAttributesFromString.test.ts | 3 +- src/lib/requests/tests/readCsv.test.ts | 3 +- src/lib/requests/tests/splitCsvToList.test.ts | 3 +- .../requests/uploadPrivacyRequestsFromCsv.ts | 26 +- src/lib/tests/TranscendPromptManager.test.ts | 2 +- .../tests/filterNullValuesFromObject.test.ts | 3 +- .../tests/findCodePackagesInFolder.test.ts | 5 +- src/lib/tests/getGitFilesThatChanged.test.ts | 5 +- src/lib/tests/mergeTranscendInputs.test.ts | 3 +- src/lib/tests/readTranscendYaml.test.ts | 3 +- src/lib/tests/removeLinks.test.ts | 3 +- vitest.config.ts | 2 +- 221 files changed, 1323 insertions(+), 1365 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index dae4a85e..a6f024ff 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.8" +version: '3.8' services: app: @@ -17,11 +17,11 @@ services: # Overrides default command so things don't shut down after the process ends. command: /bin/bash -c "sleep infinity" - network_mode: "host" + network_mode: 'host' environment: # This is set to `true` locally when using devcontainers, so we add this here for compatibility - REMOTE_CONTAINERS: "true" + REMOTE_CONTAINERS: 'true' volumes: pre-commit-cache: diff --git a/CHANGELOG.md b/CHANGELOG.md index a3ef8ee9..ba816468 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,13 @@ ## Table of Contents - [Changelog](#changelog) - - [[7.0.0] - 2025-07-10](#700---2025-07-10) - - [Improvements](#improvements) - - [Breaking Changes](#breaking-changes) - - [[6.0.0] - 2024-09-03](#600---2024-09-03) - - [Changed](#changed) - - [[5.0.0] - 2024-04-23](#500---2024-04-23) - - [Changed](#changed-1) + - [[7.0.0] - 2025-07-10](#700---2025-07-10) + - [Improvements](#improvements) + - [Breaking Changes](#breaking-changes) + - [[6.0.0] - 2024-09-03](#600---2024-09-03) + - [Changed](#changed) + - [[5.0.0] - 2024-04-23](#500---2024-04-23) + - [Changed](#changed-1) @@ -29,22 +29,22 @@ The CLI has been overhauled to be easier to use as a full-featured command line - All commands have `--help` flag to print help information. For example: - ```console - $ transcend consent update-consent-manager --help + ```console + $ transcend consent update-consent-manager --help - USAGE - transcend consent update-consent-manager (--auth value) (--bundleTypes PRODUCTION|TEST) [--deploy] [--transcendUrl value] - transcend consent update-consent-manager --help + USAGE + transcend consent update-consent-manager (--auth value) (--bundleTypes PRODUCTION|TEST) [--deploy] [--transcendUrl value] + transcend consent update-consent-manager --help - This command allows for updating Consent Manager to latest version. The consent manager bundle can also be deployed using this command. + This command allows for updating Consent Manager to latest version. The consent manager bundle can also be deployed using this command. - FLAGS - --auth The Transcend API key. Requires scopes: "Manage Consent Manager Developer Settings" - --bundleTypes The bundle types to deploy. Defaults to PRODUCTION,TEST. [PRODUCTION|TEST, separator = ,] - [--deploy] When true, deploy the Consent Manager after updating the version [default = false] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit - ``` + FLAGS + --auth The Transcend API key. Requires scopes: "Manage Consent Manager Developer Settings" + --bundleTypes The bundle types to deploy. Defaults to PRODUCTION,TEST. [PRODUCTION|TEST, separator = ,] + [--deploy] When true, deploy the Consent Manager after updating the version [default = false] + [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] + -h --help Print help information and exit + ``` - Boolean arguments no longer need to have `=true` or `=false` strings explicitly passed to them. For example, rather than pass `--deploy=true`, you can now pass `--deploy` alone. Passing `--deploy=true` or `--deploy=false` is still supported, as well as other boolean values described [here](https://github.com/bloomberg/stricli/blob/58a10349b427d9e5e7d75bf1767898d095e8544c/packages/core/src/parameter/parser/boolean.ts#L21-L26). For booleans which default to true, you can also prefix `no` to the argument name. For example, `--noDeploy` is equivalent to `--deploy=false`. - List arguments can either be passed as a comma-separated string or as several arguments. For example, `--bundleTypes=PRODUCTION,TEST` is equivalent to `--bundleTypes PRODUCTION --bundleTypes TEST`. @@ -154,50 +154,47 @@ partitions: ... ### Changed - Added support for encrypted identifiers to `tr-manual-enricher-pull-identifiers` command. + - Now that this command is using Sombra to decrypt request identifiers, you may need to provide the `--sombraAuth` argument. It's required when using self-hosted Sombra, but not for multi-tenant. - - Now that this command is using Sombra to decrypt request identifiers, you may need to provide the `--sombraAuth` argument. It's required when using self-hosted Sombra, but not for multi-tenant. + ``` + Before: + yarn tr-manual-enricher-pull-identifiers --auth=$TRANSCEND_API_KEY \ + --actions=ERASURE \ + --file=/Users/michaelfarrell/Desktop/test.csv - ``` - Before: - yarn tr-manual-enricher-pull-identifiers --auth=$TRANSCEND_API_KEY \ - --actions=ERASURE \ - --file=/Users/michaelfarrell/Desktop/test.csv - - Now: - yarn tr-manual-enricher-pull-identifiers --auth=$TRANSCEND_API_KEY \ - --sombraAuth=$SOMBRA_INTERNAL_KEY \ - --actions=ERASURE \ - --file=/Users/michaelfarrell/Desktop/test.csv - ``` + Now: + yarn tr-manual-enricher-pull-identifiers --auth=$TRANSCEND_API_KEY \ + --sombraAuth=$SOMBRA_INTERNAL_KEY \ + --actions=ERASURE \ + --file=/Users/michaelfarrell/Desktop/test.csv + ``` - Added support for encrypted identifiers to `tr-request-export` command. + - Now that this command is using Sombra to decrypt request identifiers, you may need to provide the `--sombraAuth` argument. It's required when using self-hosted Sombra, but not for multi-tenant. - - Now that this command is using Sombra to decrypt request identifiers, you may need to provide the `--sombraAuth` argument. It's required when using self-hosted Sombra, but not for multi-tenant. - - ``` - Before: - yarn tr-request-export --auth=$TRANSCEND_API_KEY \ - --actions=ERASURE \ - --file=/Users/michaelfarrell/Desktop/test.csv + ``` + Before: + yarn tr-request-export --auth=$TRANSCEND_API_KEY \ + --actions=ERASURE \ + --file=/Users/michaelfarrell/Desktop/test.csv - Now: - yarn tr-request-export --auth=$TRANSCEND_API_KEY \ - --sombraAuth=$SOMBRA_INTERNAL_KEY \ - --actions=ERASURE \ - --file=/Users/michaelfarrell/Desktop/test.csv - ``` + Now: + yarn tr-request-export --auth=$TRANSCEND_API_KEY \ + --sombraAuth=$SOMBRA_INTERNAL_KEY \ + --actions=ERASURE \ + --file=/Users/michaelfarrell/Desktop/test.csv + ``` - Added support for encrypted identifiers to `tr-request-restart` command, used only when `--copyIdentifiers` argument is specified. - - - Now that this command is using Sombra to decrypt request identifiers, you may need to provide the `--sombraAuth` argument. It's required only when using `--copyIdentifiers` AND self-hosted Sombra, but is otherwise not required. - - ``` - Before: - yarn tr-request-restart --auth=$TRANSCEND_API_KEY \ - --statuses=COMPILING,APPROVING --actions=ERASURE --copyIdentifiers=true - - Now: - yarn tr-request-restart --auth=$TRANSCEND_API_KEY \ - --sombraAuth=$SOMBRA_INTERNAL_KEY \ - --statuses=COMPILING,APPROVING --actions=ERASURE --copyIdentifiers=true - ``` + - Now that this command is using Sombra to decrypt request identifiers, you may need to provide the `--sombraAuth` argument. It's required only when using `--copyIdentifiers` AND self-hosted Sombra, but is otherwise not required. + + ``` + Before: + yarn tr-request-restart --auth=$TRANSCEND_API_KEY \ + --statuses=COMPILING,APPROVING --actions=ERASURE --copyIdentifiers=true + + Now: + yarn tr-request-restart --auth=$TRANSCEND_API_KEY \ + --sombraAuth=$SOMBRA_INTERNAL_KEY \ + --statuses=COMPILING,APPROVING --actions=ERASURE --copyIdentifiers=true + ``` diff --git a/DEVELOPERS.md b/DEVELOPERS.md index fc498c5b..f7cf15c1 100644 --- a/DEVELOPERS.md +++ b/DEVELOPERS.md @@ -4,13 +4,13 @@ ## Table of Contents - [Developers](#developers) - - [Getting started](#getting-started) - - [Repo Structure](#repo-structure) - - [Generated files](#generated-files) - - [README.md](#readmemd) - - [transcend.yml and pathfinder.yml JSON schemas](#transcendyml-and-pathfinderyml-json-schemas) - - [Testing](#testing) - - [Publishing](#publishing) + - [Getting started](#getting-started) + - [Repo Structure](#repo-structure) + - [Generated files](#generated-files) + - [README.md](#readmemd) + - [transcend.yml and pathfinder.yml JSON schemas](#transcendyml-and-pathfinderyml-json-schemas) + - [Testing](#testing) + - [Publishing](#publishing) diff --git a/README.md b/README.md index 98e7cd4e..30de3733 100644 --- a/README.md +++ b/README.md @@ -10,52 +10,52 @@ - [Installation](#installation) - [transcend.yml](#transcendyml) - [Usage](#usage) - - [`transcend request approve`](#transcend-request-approve) - - [`transcend request upload`](#transcend-request-upload) - - [`transcend request download-files`](#transcend-request-download-files) - - [`transcend request cancel`](#transcend-request-cancel) - - [`transcend request restart`](#transcend-request-restart) - - [`transcend request notify-additional-time`](#transcend-request-notify-additional-time) - - [`transcend request mark-silent`](#transcend-request-mark-silent) - - [`transcend request enricher-restart`](#transcend-request-enricher-restart) - - [`transcend request reject-unverified-identifiers`](#transcend-request-reject-unverified-identifiers) - - [`transcend request export`](#transcend-request-export) - - [`transcend request skip-preflight-jobs`](#transcend-request-skip-preflight-jobs) - - [`transcend request system mark-request-data-silos-completed`](#transcend-request-system-mark-request-data-silos-completed) - - [`transcend request system retry-request-data-silos`](#transcend-request-system-retry-request-data-silos) - - [`transcend request system skip-request-data-silos`](#transcend-request-system-skip-request-data-silos) - - [`transcend request preflight pull-identifiers`](#transcend-request-preflight-pull-identifiers) - - [`transcend request preflight push-identifiers`](#transcend-request-preflight-push-identifiers) - - [`transcend request cron pull-identifiers`](#transcend-request-cron-pull-identifiers) - - [`transcend request cron mark-identifiers-completed`](#transcend-request-cron-mark-identifiers-completed) - - [`transcend consent build-xdi-sync-endpoint`](#transcend-consent-build-xdi-sync-endpoint) - - [`transcend consent pull-consent-metrics`](#transcend-consent-pull-consent-metrics) - - [`transcend consent pull-consent-preferences`](#transcend-consent-pull-consent-preferences) - - [`transcend consent update-consent-manager`](#transcend-consent-update-consent-manager) - - [`transcend consent upload-consent-preferences`](#transcend-consent-upload-consent-preferences) - - [`transcend consent upload-cookies-from-csv`](#transcend-consent-upload-cookies-from-csv) - - [`transcend consent upload-data-flows-from-csv`](#transcend-consent-upload-data-flows-from-csv) - - [`transcend consent upload-preferences`](#transcend-consent-upload-preferences) - - [`transcend inventory pull`](#transcend-inventory-pull) - - [Scopes](#scopes) - - [Usage](#usage-1) - - [`transcend inventory push`](#transcend-inventory-push) - - [Scopes](#scopes-1) - - [Usage](#usage-2) - - [CI Integration](#ci-integration) - - [Dynamic Variables](#dynamic-variables) - - [`transcend inventory scan-packages`](#transcend-inventory-scan-packages) - - [`transcend inventory discover-silos`](#transcend-inventory-discover-silos) - - [Usage](#usage-3) - - [`transcend inventory pull-datapoints`](#transcend-inventory-pull-datapoints) - - [`transcend inventory pull-unstructured-discovery-files`](#transcend-inventory-pull-unstructured-discovery-files) - - [`transcend inventory derive-data-silos-from-data-flows`](#transcend-inventory-derive-data-silos-from-data-flows) - - [`transcend inventory derive-data-silos-from-data-flows-cross-instance`](#transcend-inventory-derive-data-silos-from-data-flows-cross-instance) - - [`transcend inventory consent-manager-service-json-to-yml`](#transcend-inventory-consent-manager-service-json-to-yml) - - [`transcend inventory consent-managers-to-business-entities`](#transcend-inventory-consent-managers-to-business-entities) - - [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys) - - [Usage](#usage-4) - - [`transcend migration sync-ot`](#transcend-migration-sync-ot) + - [`transcend request approve`](#transcend-request-approve) + - [`transcend request upload`](#transcend-request-upload) + - [`transcend request download-files`](#transcend-request-download-files) + - [`transcend request cancel`](#transcend-request-cancel) + - [`transcend request restart`](#transcend-request-restart) + - [`transcend request notify-additional-time`](#transcend-request-notify-additional-time) + - [`transcend request mark-silent`](#transcend-request-mark-silent) + - [`transcend request enricher-restart`](#transcend-request-enricher-restart) + - [`transcend request reject-unverified-identifiers`](#transcend-request-reject-unverified-identifiers) + - [`transcend request export`](#transcend-request-export) + - [`transcend request skip-preflight-jobs`](#transcend-request-skip-preflight-jobs) + - [`transcend request system mark-request-data-silos-completed`](#transcend-request-system-mark-request-data-silos-completed) + - [`transcend request system retry-request-data-silos`](#transcend-request-system-retry-request-data-silos) + - [`transcend request system skip-request-data-silos`](#transcend-request-system-skip-request-data-silos) + - [`transcend request preflight pull-identifiers`](#transcend-request-preflight-pull-identifiers) + - [`transcend request preflight push-identifiers`](#transcend-request-preflight-push-identifiers) + - [`transcend request cron pull-identifiers`](#transcend-request-cron-pull-identifiers) + - [`transcend request cron mark-identifiers-completed`](#transcend-request-cron-mark-identifiers-completed) + - [`transcend consent build-xdi-sync-endpoint`](#transcend-consent-build-xdi-sync-endpoint) + - [`transcend consent pull-consent-metrics`](#transcend-consent-pull-consent-metrics) + - [`transcend consent pull-consent-preferences`](#transcend-consent-pull-consent-preferences) + - [`transcend consent update-consent-manager`](#transcend-consent-update-consent-manager) + - [`transcend consent upload-consent-preferences`](#transcend-consent-upload-consent-preferences) + - [`transcend consent upload-cookies-from-csv`](#transcend-consent-upload-cookies-from-csv) + - [`transcend consent upload-data-flows-from-csv`](#transcend-consent-upload-data-flows-from-csv) + - [`transcend consent upload-preferences`](#transcend-consent-upload-preferences) + - [`transcend inventory pull`](#transcend-inventory-pull) + - [Scopes](#scopes) + - [Usage](#usage-1) + - [`transcend inventory push`](#transcend-inventory-push) + - [Scopes](#scopes-1) + - [Usage](#usage-2) + - [CI Integration](#ci-integration) + - [Dynamic Variables](#dynamic-variables) + - [`transcend inventory scan-packages`](#transcend-inventory-scan-packages) + - [`transcend inventory discover-silos`](#transcend-inventory-discover-silos) + - [Usage](#usage-3) + - [`transcend inventory pull-datapoints`](#transcend-inventory-pull-datapoints) + - [`transcend inventory pull-unstructured-discovery-files`](#transcend-inventory-pull-unstructured-discovery-files) + - [`transcend inventory derive-data-silos-from-data-flows`](#transcend-inventory-derive-data-silos-from-data-flows) + - [`transcend inventory derive-data-silos-from-data-flows-cross-instance`](#transcend-inventory-derive-data-silos-from-data-flows-cross-instance) + - [`transcend inventory consent-manager-service-json-to-yml`](#transcend-inventory-consent-manager-service-json-to-yml) + - [`transcend inventory consent-managers-to-business-entities`](#transcend-inventory-consent-managers-to-business-entities) + - [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys) + - [Usage](#usage-4) + - [`transcend migration sync-ot`](#transcend-migration-sync-ot) - [Prompt Manager](#prompt-manager) - [Proxy usage](#proxy-usage) @@ -114,8 +114,8 @@ The structure of `transcend.yml` looks something like the following: # CLI, it is possible to specify which API key should be associated # with the newly created data silo. api-keys: - - title: Webhook Key - - title: Analytics Key + - title: Webhook Key + - title: Analytics Key # Manage at: https://app.transcend.io/privacy-requests/identifiers # See https://docs.transcend.io/docs/identity-enrichment @@ -126,65 +126,65 @@ api-keys: # - fraud check: auto-cancel requests if the user is flagged for fraudulent behavior # - customer check: auto-cancel request for some custom business criteria enrichers: - - title: Basic Identity Enrichment - description: Enrich an email address to the userId and phone number - url: https://example.acme.com/transcend-enrichment-webhook - input-identifier: email - output-identifiers: - - userId - - phone - - myUniqueIdentifier - - title: Fraud Check - description: Ensure the email address is not marked as fraudulent - url: https://example.acme.com/transcend-fraud-check - input-identifier: email - output-identifiers: - - email - privacy-actions: - - ERASURE + - title: Basic Identity Enrichment + description: Enrich an email address to the userId and phone number + url: https://example.acme.com/transcend-enrichment-webhook + input-identifier: email + output-identifiers: + - userId + - phone + - myUniqueIdentifier + - title: Fraud Check + description: Ensure the email address is not marked as fraudulent + url: https://example.acme.com/transcend-fraud-check + input-identifier: email + output-identifiers: + - email + privacy-actions: + - ERASURE # Manage at: https://app.transcend.io/privacy-requests/connected-services # See https://docs.transcend.io/docs/the-data-map#data-silos # Define the data silos in your data map. A data silo can be a database, # or a web service that may use a collection of different data stores under the hood. data-silos: - # Note: title is the only required top-level field for a data silo - - title: Redshift Data Warehouse - description: The mega-warehouse that contains a copy over all SQL backed databases - integrationName: server - url: https://example.acme.com/transcend-webhook - api-key-title: Webhook Key - data-subjects: - - customer - - employee - - newsletter-subscriber - - b2b-contact - identity-keys: - - email - - userId - deletion-dependencies: - - Identity Service - owners: - - alice@transcend.io - datapoints: - - title: Webhook Notification - key: _global - privacy-actions: - - ACCESS - - ERASURE - - SALE_OPT_OUT - - title: User Model - description: The centralized user model user - key: users - privacy-actions: - - ACCESS - fields: - - key: firstName - title: First Name - description: The first name of the user, inputted during onboarding - - key: email - title: Email - description: The email address of the user + # Note: title is the only required top-level field for a data silo + - title: Redshift Data Warehouse + description: The mega-warehouse that contains a copy over all SQL backed databases + integrationName: server + url: https://example.acme.com/transcend-webhook + api-key-title: Webhook Key + data-subjects: + - customer + - employee + - newsletter-subscriber + - b2b-contact + identity-keys: + - email + - userId + deletion-dependencies: + - Identity Service + owners: + - alice@transcend.io + datapoints: + - title: Webhook Notification + key: _global + privacy-actions: + - ACCESS + - ERASURE + - SALE_OPT_OUT + - title: User Model + description: The centralized user model user + key: users + privacy-actions: + - ACCESS + fields: + - key: firstName + title: First Name + description: The first name of the user, inputted during onboarding + - key: email + title: Email + description: The email address of the user ``` ## Usage @@ -1133,32 +1133,32 @@ name: Transcend Data Map Syncing # See https://app.transcend.io/privacy-requests/connected-services on: - push: - branches: - - 'main' + push: + branches: + - 'main' jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 - - name: Setup Node.js - uses: actions/setup-node@v2 - with: - node-version: '16' + - name: Setup Node.js + uses: actions/setup-node@v2 + with: + node-version: '16' - - name: Install Transcend CLI - run: npm install --global @transcend-io/cli + - name: Install Transcend CLI + run: npm install --global @transcend-io/cli - # If you have a script that generates your transcend.yml file from - # an ORM or infrastructure configuration, add that step here - # Leave this step commented out if you want to manage your transcend.yml manually - # - name: Generate transcend.yml - # run: ./scripts/generate_transcend_yml.py + # If you have a script that generates your transcend.yml file from + # an ORM or infrastructure configuration, add that step here + # Leave this step commented out if you want to manage your transcend.yml manually + # - name: Generate transcend.yml + # run: ./scripts/generate_transcend_yml.py - - name: Push Transcend config - run: transcend inventory push --auth=${{ secrets.TRANSCEND_API_KEY }} + - name: Push Transcend config + run: transcend inventory push --auth=${{ secrets.TRANSCEND_API_KEY }} ``` #### Dynamic Variables @@ -1175,32 +1175,32 @@ This command could fill out multiple parameters in a YAML file like [./examples/ ```yml api-keys: - - title: Webhook Key + - title: Webhook Key enrichers: - - title: Basic Identity Enrichment - description: Enrich an email address to the userId and phone number - # The data silo webhook URL is the same in each environment, - # except for the base domain in the webhook URL. - url: https://example.<>/transcend-enrichment-webhook - input-identifier: email - output-identifiers: - - userId - - phone - - myUniqueIdentifier - - title: Fraud Check - description: Ensure the email address is not marked as fraudulent - url: https://example.<>/transcend-fraud-check - input-identifier: email - output-identifiers: - - email - privacy-actions: - - ERASURE + - title: Basic Identity Enrichment + description: Enrich an email address to the userId and phone number + # The data silo webhook URL is the same in each environment, + # except for the base domain in the webhook URL. + url: https://example.<>/transcend-enrichment-webhook + input-identifier: email + output-identifiers: + - userId + - phone + - myUniqueIdentifier + - title: Fraud Check + description: Ensure the email address is not marked as fraudulent + url: https://example.<>/transcend-fraud-check + input-identifier: email + output-identifiers: + - email + privacy-actions: + - ERASURE data-silos: - - title: Redshift Data Warehouse - integrationName: server - description: The mega-warehouse that contains a copy over all SQL backed databases - <> - url: https://example.<>/transcend-webhook - api-key-title: Webhook Key + - title: Redshift Data Warehouse + integrationName: server + description: The mega-warehouse that contains a copy over all SQL backed databases - <> + url: https://example.<>/transcend-webhook + api-key-title: Webhook Key ``` ### `transcend inventory scan-packages` @@ -1432,12 +1432,12 @@ Filter for only a specific organization by ID, returning all child accounts asso ```gql query { - user { - organization { - id - parentOrganizationId - } - } + user { + organization { + id + parentOrganizationId + } + } } ``` @@ -1499,142 +1499,145 @@ FLAGS If you are integrating Transcend's Prompt Manager into your code, it may look like: ```ts -import * as t from 'io-ts'; import { TranscendPromptManager } from '@transcend-io/cli'; import { - ChatCompletionMessage, - PromptRunProductArea, + ChatCompletionMessage, + PromptRunProductArea, } from '@transcend-io/privacy-types'; +import * as t from 'io-ts'; /** * Example prompt integration */ export async function main(): Promise { - // Instantiate the Transcend Prompt Manager instance - const promptManager = new TranscendPromptManager({ - // API key - transcendApiKey: process.env.TRANSCEND_API_KEY, - // Define the prompts that are stored in Transcend - prompts: { - test: { - // identify by ID - id: '30bcaa79-889a-4af3-842d-2e8ba443d36d', - // no runtime variables - paramCodec: t.type({}), - // response is list of strings - outputCodec: t.array(t.string), + // Instantiate the Transcend Prompt Manager instance + const promptManager = new TranscendPromptManager({ + // API key + transcendApiKey: process.env.TRANSCEND_API_KEY, + // Define the prompts that are stored in Transcend + prompts: { + test: { + // identify by ID + id: '30bcaa79-889a-4af3-842d-2e8ba443d36d', + // no runtime variables + paramCodec: t.type({}), + // response is list of strings + outputCodec: t.array(t.string), + }, + json: { + // identify by title + title: 'test', + // one runtime variable "test" + paramCodec: t.type({ test: t.string }), + // runtime is json object + outputCodec: t.record(t.string, t.string), + // response is stored in atg + extractFromTag: 'json', + }, + predictProductLine: { + // identify by title + title: 'Predict Product Line', + // runtime parameter for slack channel name + paramCodec: t.type({ + slackChannelName: t.string, + }), + // response is specific JSON shape + outputCodec: t.type({ + product: t.union([t.string, t.null]), + clarification: t.union([t.string, t.null]), + }), + // response is stored in atg + extractFromTag: 'json', + }, + }, + // Optional arguments + // transcendUrl: 'https://api.us.transcend.io', // defaults to 'https://api.transcend.io' + // requireApproval: false, // defaults to true + // cacheDuration: 1000 * 60 * 60, // defaults to undefined, no cache + // defaultVariables: { myVariable: 'this is custom', other: [{ name: 'custom' }] }, // defaults to {} + // handlebarsOptions: { helpers, templates }, // defaults to {} + }); + + // Fetch the prompt from Transcend and template any variables + // in this case, we template the slack channel name in the LLM prompt + const systemPrompt = await promptManager.compilePrompt( + 'predictProductLine', + { + slackChannelName: channelName, + }, + ); + + // Parameters to pass to the LLM + const input: ChatCompletionMessage[] = [ + { + role: 'system', + content: systemPrompt, }, - json: { - // identify by title - title: 'test', - // one runtime variable "test" - paramCodec: t.type({ test: t.string }), - // runtime is json object - outputCodec: t.record(t.string, t.string), - // response is stored in atg - extractFromTag: 'json', + { + role: 'user', + content: input, }, - predictProductLine: { - // identify by title - title: 'Predict Product Line', - // runtime parameter for slack channel name - paramCodec: t.type({ - slackChannelName: t.string, - }), - // response is specific JSON shape - outputCodec: t.type({ - product: t.union([t.string, t.null]), - clarification: t.union([t.string, t.null]), - }), - // response is stored in atg - extractFromTag: 'json', + ]; + const largeLanguageModel = { + name: 'gpt-4', + client: 'openai' as const, + }; + const temperature = 1; + const topP = 1; + const maxTokensToSample = 1000; + + // Run prompt against LLM + let response: string; + const t0 = new Date().getTime(); + try { + response = await openai.createCompletion(input, { + temperature, + top_p: topP, + max_tokens: maxTokensToSample, + }); + } catch (err) { + // report error upon failure + await promptManager.reportPromptRunError('predictProductLine', { + promptRunMessages: input, + duration: new Date().getTime() - t0, + temperature, + topP, + error: err.message, + maxTokensToSample, + largeLanguageModel, + }); + } + const t1 = new Date().getTime(); + + // Parsed response as JSON and do not report to Transcend + // const parsedResponse = promptManager.parseAiResponse( + // 'predictProductLine', + // response, + // ); + + // Parsed response as JSON and report output to Transcend + const parsedResponse = await promptManager.reportAndParsePromptRun( + 'predictProductLine', + { + promptRunMessages: [ + ...input, + { + role: 'assistant', + content: response, + }, + ], + duration: t1 - t0, + temperature, + topP, + maxTokensToSample, + largeLanguageModel, + // Optional parameters + // name, // unique identifier for this run + // productArea, // Transcend product area that the prompt relates to + // runByEmployeeEmail, // Employee email that is executing the request + // promptGroupId, // The prompt group being reported }, - }, - // Optional arguments - // transcendUrl: 'https://api.us.transcend.io', // defaults to 'https://api.transcend.io' - // requireApproval: false, // defaults to true - // cacheDuration: 1000 * 60 * 60, // defaults to undefined, no cache - // defaultVariables: { myVariable: 'this is custom', other: [{ name: 'custom' }] }, // defaults to {} - // handlebarsOptions: { helpers, templates }, // defaults to {} - }); - - // Fetch the prompt from Transcend and template any variables - // in this case, we template the slack channel name in the LLM prompt - const systemPrompt = await promptManager.compilePrompt('predictProductLine', { - slackChannelName: channelName, - }); - - // Parameters to pass to the LLM - const input: ChatCompletionMessage[] = [ - { - role: 'system', - content: systemPrompt, - }, - { - role: 'user', - content: input, - }, - ]; - const largeLanguageModel = { - name: 'gpt-4', - client: 'openai' as const, - }; - const temperature = 1; - const topP = 1; - const maxTokensToSample = 1000; - - // Run prompt against LLM - let response: string; - const t0 = new Date().getTime(); - try { - response = await openai.createCompletion(input, { - temperature, - top_p: topP, - max_tokens: maxTokensToSample, - }); - } catch (err) { - // report error upon failure - await promptManager.reportPromptRunError('predictProductLine', { - promptRunMessages: input, - duration: new Date().getTime() - t0, - temperature, - topP, - error: err.message, - maxTokensToSample, - largeLanguageModel, - }); - } - const t1 = new Date().getTime(); - - // Parsed response as JSON and do not report to Transcend - // const parsedResponse = promptManager.parseAiResponse( - // 'predictProductLine', - // response, - // ); - - // Parsed response as JSON and report output to Transcend - const parsedResponse = await promptManager.reportAndParsePromptRun( - 'predictProductLine', - { - promptRunMessages: [ - ...input, - { - role: 'assistant', - content: response, - }, - ], - duration: t1 - t0, - temperature, - topP, - maxTokensToSample, - largeLanguageModel, - // Optional parameters - // name, // unique identifier for this run - // productArea, // Transcend product area that the prompt relates to - // runByEmployeeEmail, // Employee email that is executing the request - // promptGroupId, // The prompt group being reported - }, - ); + ); } ``` diff --git a/scripts/buildReadmeDocs.ts b/scripts/buildReadmeDocs.ts index 20d8c66a..c02b9cb9 100644 --- a/scripts/buildReadmeDocs.ts +++ b/scripts/buildReadmeDocs.ts @@ -1,15 +1,14 @@ +import { execSync } from 'node:child_process'; +import fs from 'node:fs'; import { generateHelpTextForAllCommands, type Application, type CommandContext, } from '@stricli/core'; -import { app } from '../src/app'; -import fs from 'node:fs'; -import { execSync } from 'node:child_process'; import { fdir } from 'fdir'; +import { app } from '../src/app'; -// eslint-disable-next-line new-cap -const docFiles = new fdir() +const documentFiles = new fdir() .withRelativePaths() .glob('**/readme.ts') .crawl('./src/commands') @@ -18,7 +17,7 @@ const docFiles = new fdir() // For each src/commands/**/readme.ts file, create a key-value pair of the command and the exported Markdown documentation const additionalDocumentation: Record = Object.fromEntries( await Promise.all( - docFiles.map(async (file) => { + documentFiles.map(async (file) => { const command = `transcend ${file.split('/').slice(0, -1).join(' ')}`; const readme = (await import(`../src/commands/${file}`)).default; return [command, readme]; @@ -42,7 +41,7 @@ const formattedMarkdown: string = helpTextForAllCommands const readme = fs.readFileSync('README.md', 'utf8'); -const newReadme = readme.replace( +const newReadme = readme.replaceAll( /[\s\S]*?/g, `\n${formattedMarkdown}\n`, ); diff --git a/src/codecs.ts b/src/codecs.ts index b9f3b1f6..3cab16cf 100644 --- a/src/codecs.ts +++ b/src/codecs.ts @@ -1,69 +1,69 @@ // eslint-disable-next-line eslint-comments/disable-enable-pair /* eslint-disable max-lines */ -import * as t from 'io-ts'; -import { applyEnum, valuesOf } from '@transcend-io/type-utils'; import { - DataCategoryType, - ConsentBundleType, - EnricherType, - ProcessingPurpose, - RequestAction, - ComparisonOperator, - RequestActionObjectResolver, - AssessmentSyncColumn, - RetentionScheduleType, + BrowserLanguage, + InitialViewState, + OnConsentExpiry, + UserPrivacySignalEnum, +} from '@transcend-io/airgap.js-types'; +import { LanguageKey } from '@transcend-io/internationalization'; +import { + ActionItemCode, + ActionItemPriorityOverride, + AssessmentFormStatus, + AssessmentFormTemplateSource, + AssessmentFormTemplateStatus, + AssessmentQuestionSubType, AssessmentQuestionType, - UspapiOption, - DataFlowScope, - PromptAVendorEmailSendType, - RetentionScheduleOperation, AssessmentsDisplayLogicAction, - DefaultConsentOption, - LogicOperator, - ConsentPrecedenceOption, + AssessmentSyncColumn, AssessmentSyncModel, - AssessmentQuestionSubType, - IsoCountryCode, + AttributeKeyType, + AttributeSupportedResourceType, BrowserTimeZone, - IsoCountrySubdivisionCode, + CodePackageType, + ComparisonOperator, + ConfigurableColorPaletteColor, + ConsentBundleType, + ConsentPrecedenceOption, ConsentTrackerStatus, - AttributeKeyType, + DataCategoryType, + DataFlowScope, + DefaultConsentOption, + EnricherType, + IsoCountryCode, + IsoCountrySubdivisionCode, + LargeLanguageModelClient, + LogicOperator, + PreferenceStoreAuthLevel, + PreferenceTopicType, + PreflightRequestStatus, + PrivacyCenterComponentStyles, + PrivacyCenterTextStyles, + ProcessingPurpose, PromptAVendorEmailCompletionLinkType, + PromptAVendorEmailSendType, + PromptFilePurpose, + RegionDetectionMethod, RegionsOperator, - UnknownRequestPolicy, - TelemetryPartitionStrategy, + RequestAction, + RequestActionObjectResolver, + RetentionScheduleOperation, + RetentionScheduleType, + ScopeName, SignedIabAgreementOption, - RegionDetectionMethod, - PreflightRequestStatus, - AttributeSupportedResourceType, SubDataPointDataSubCategoryGuessStatus, - LargeLanguageModelClient, - PromptFilePurpose, - CodePackageType, - ActionItemPriorityOverride, - ActionItemCode, - ScopeName, + TelemetryPartitionStrategy, TranscendProduct, - PrivacyCenterComponentStyles, - PrivacyCenterTextStyles, - PreferenceStoreAuthLevel, - ConfigurableColorPaletteColor, - AssessmentFormTemplateStatus, - AssessmentFormStatus, - AssessmentFormTemplateSource, + UnknownRequestPolicy, UnstructuredSubDataPointRecommendationStatus, - PreferenceTopicType, + UspapiOption, } from '@transcend-io/privacy-types'; -import { - InitialViewState, - BrowserLanguage, - UserPrivacySignalEnum, - OnConsentExpiry, -} from '@transcend-io/airgap.js-types'; -import { buildEnabledRouteType } from './lib/helpers/buildEnabledRouteType'; -import { buildAIIntegrationType } from './lib/helpers/buildAIIntegrationType'; +import { applyEnum, valuesOf } from '@transcend-io/type-utils'; +import * as t from 'io-ts'; import { OpenAIRouteName, PathfinderPolicyName } from './enums'; -import { LanguageKey } from '@transcend-io/internationalization'; +import { buildAIIntegrationType } from './lib/helpers/buildAIIntegrationType'; +import { buildEnabledRouteType } from './lib/helpers/buildEnabledRouteType'; /** * Input to define email templates that can be used to communicate to end-users diff --git a/src/commands/admin/generate-api-keys/impl.ts b/src/commands/admin/generate-api-keys/impl.ts index 1e15be84..8f37de0a 100644 --- a/src/commands/admin/generate-api-keys/impl.ts +++ b/src/commands/admin/generate-api-keys/impl.ts @@ -1,12 +1,10 @@ -import type { LocalContext } from '../../../context'; -import colors from 'colors'; import { writeFileSync } from 'fs'; - import { ScopeName, TRANSCEND_SCOPES } from '@transcend-io/privacy-types'; - -import { logger } from '../../../logger'; -import { generateCrossAccountApiKeys } from '../../../lib/api-keys'; +import colors from 'colors'; import { keyBy } from 'lodash-es'; +import type { LocalContext } from '../../../context'; +import { generateCrossAccountApiKeys } from '../../../lib/api-keys'; +import { logger } from '../../../logger'; const SCOPES_BY_TITLE = keyBy( Object.entries(TRANSCEND_SCOPES).map(([name, value]) => ({ diff --git a/src/commands/consent/build-xdi-sync-endpoint/impl.ts b/src/commands/consent/build-xdi-sync-endpoint/impl.ts index d930bf53..a4735488 100644 --- a/src/commands/consent/build-xdi-sync-endpoint/impl.ts +++ b/src/commands/consent/build-xdi-sync-endpoint/impl.ts @@ -1,9 +1,9 @@ -import type { LocalContext } from '../../../context'; -import { logger } from '../../../logger'; -import colors from 'colors'; import { writeFileSync } from 'fs'; +import colors from 'colors'; +import type { LocalContext } from '../../../context'; import { validateTranscendAuth } from '../../../lib/api-keys'; import { buildXdiSyncEndpoint as buildXdiSyncEndpointHelper } from '../../../lib/consent-manager'; +import { logger } from '../../../logger'; interface BuildXdiSyncEndpointCommandFlags { auth: string; diff --git a/src/commands/consent/pull-consent-metrics/impl.ts b/src/commands/consent/pull-consent-metrics/impl.ts index 5efea655..ffba9d54 100644 --- a/src/commands/consent/pull-consent-metrics/impl.ts +++ b/src/commands/consent/pull-consent-metrics/impl.ts @@ -1,17 +1,17 @@ -import type { LocalContext } from '../../../context'; -import { logger } from '../../../logger'; +import fs, { existsSync, mkdirSync } from 'fs'; +import { join } from 'path'; import colors from 'colors'; +import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { validateTranscendAuth } from '../../../lib/api-keys'; import { mapSeries } from '../../../lib/bluebird-replace'; -import { join } from 'path'; -import fs, { existsSync, mkdirSync } from 'fs'; +import { pullConsentManagerMetrics } from '../../../lib/consent-manager'; +import { writeCsv } from '../../../lib/cron'; import { buildTranscendGraphQLClient, ConsentManagerMetricBin, } from '../../../lib/graphql'; -import { validateTranscendAuth } from '../../../lib/api-keys'; -import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; -import { pullConsentManagerMetrics } from '../../../lib/consent-manager'; -import { writeCsv } from '../../../lib/cron'; +import { logger } from '../../../logger'; interface PullConsentMetricsCommandFlags { auth: string; diff --git a/src/commands/consent/pull-consent-preferences/impl.ts b/src/commands/consent/pull-consent-preferences/impl.ts index aaf4b6fe..afb94a29 100644 --- a/src/commands/consent/pull-consent-preferences/impl.ts +++ b/src/commands/consent/pull-consent-preferences/impl.ts @@ -1,5 +1,4 @@ import type { LocalContext } from '../../../context'; - import { fetchConsentPreferences } from '../../../lib/consent-manager'; import { writeCsv } from '../../../lib/cron'; import { createSombraGotInstance } from '../../../lib/graphql'; diff --git a/src/commands/consent/update-consent-manager/impl.ts b/src/commands/consent/update-consent-manager/impl.ts index 7b549293..25ae2763 100644 --- a/src/commands/consent/update-consent-manager/impl.ts +++ b/src/commands/consent/update-consent-manager/impl.ts @@ -1,11 +1,10 @@ -import type { LocalContext } from '../../../context'; -import colors from 'colors'; import { ConsentBundleType } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import type { LocalContext } from '../../../context'; +import { validateTranscendAuth } from '../../../lib/api-keys'; import { mapSeries } from '../../../lib/bluebird-replace'; - -import { logger } from '../../../logger'; import { updateConsentManagerVersionToLatest } from '../../../lib/consent-manager'; -import { validateTranscendAuth } from '../../../lib/api-keys'; +import { logger } from '../../../logger'; interface UpdateConsentManagerCommandFlags { auth: string; diff --git a/src/commands/consent/upload-consent-preferences/impl.ts b/src/commands/consent/upload-consent-preferences/impl.ts index 365a7f72..267f2ee9 100644 --- a/src/commands/consent/upload-consent-preferences/impl.ts +++ b/src/commands/consent/upload-consent-preferences/impl.ts @@ -1,7 +1,6 @@ import type { LocalContext } from '../../../context'; - -import { uploadConsents } from '../../../lib/consent-manager/uploadConsents'; import { ConsentPreferenceUpload } from '../../../lib/consent-manager/types'; +import { uploadConsents } from '../../../lib/consent-manager/uploadConsents'; import { readCsv } from '../../../lib/requests'; interface UploadConsentPreferencesCommandFlags { diff --git a/src/commands/consent/upload-cookies-from-csv/impl.ts b/src/commands/consent/upload-cookies-from-csv/impl.ts index 117b44b0..3e716dea 100644 --- a/src/commands/consent/upload-cookies-from-csv/impl.ts +++ b/src/commands/consent/upload-cookies-from-csv/impl.ts @@ -1,6 +1,6 @@ +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; import type { LocalContext } from '../../../context'; import { uploadCookiesFromCsv as uploadCookiesFromCsvHelper } from '../../../lib/consent-manager'; -import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; interface UploadCookiesFromCsvCommandFlags { auth: string; diff --git a/src/commands/consent/upload-data-flows-from-csv/impl.ts b/src/commands/consent/upload-data-flows-from-csv/impl.ts index c65edaa9..e1cfd34e 100644 --- a/src/commands/consent/upload-data-flows-from-csv/impl.ts +++ b/src/commands/consent/upload-data-flows-from-csv/impl.ts @@ -1,6 +1,6 @@ +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; import type { LocalContext } from '../../../context'; import { uploadDataFlowsFromCsv as uploadDataFlowsFromCsvHelper } from '../../../lib/consent-manager'; -import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; interface UploadDataFlowsFromCsvCommandFlags { auth: string; diff --git a/src/commands/consent/upload-preferences/impl.ts b/src/commands/consent/upload-preferences/impl.ts index eace39ac..9fdf8f55 100644 --- a/src/commands/consent/upload-preferences/impl.ts +++ b/src/commands/consent/upload-preferences/impl.ts @@ -1,12 +1,11 @@ -import type { LocalContext } from '../../../context'; +import { readdirSync } from 'fs'; +import { basename, join } from 'path'; import colors from 'colors'; - -import { logger } from '../../../logger'; +import type { LocalContext } from '../../../context'; +import { map } from '../../../lib/bluebird-replace'; import { uploadPreferenceManagementPreferencesInteractive } from '../../../lib/preference-management'; import { splitCsvToList } from '../../../lib/requests'; -import { readdirSync } from 'fs'; -import { map } from '../../../lib/bluebird-replace'; -import { basename, join } from 'path'; +import { logger } from '../../../logger'; interface UploadPreferencesCommandFlags { auth: string; diff --git a/src/commands/inventory/consent-manager-service-json-to-yml/command.ts b/src/commands/inventory/consent-manager-service-json-to-yml/command.ts index 248474e8..faa662b1 100644 --- a/src/commands/inventory/consent-manager-service-json-to-yml/command.ts +++ b/src/commands/inventory/consent-manager-service-json-to-yml/command.ts @@ -1,5 +1,5 @@ -import { name } from '../../../constants'; import { buildCommand } from '@stricli/core'; +import { name } from '../../../constants'; export const consentManagerServiceJsonToYmlCommand = buildCommand({ loader: async () => { diff --git a/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts b/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts index 39697da0..bd02f44c 100644 --- a/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts +++ b/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts @@ -1,19 +1,19 @@ -import type { LocalContext } from '../../../context'; -import * as t from 'io-ts'; -import { writeTranscendYaml } from '../../../lib/readTranscendYaml'; -import colors from 'colors'; -import { logger } from '../../../logger'; import { existsSync, readFileSync } from 'fs'; +import { + ConsentTrackerStatus, + DataFlowScope, +} from '@transcend-io/privacy-types'; import { decodeCodec } from '@transcend-io/type-utils'; +import colors from 'colors'; +import * as t from 'io-ts'; import { ConsentManagerServiceMetadata, CookieInput, DataFlowInput, } from '../../../codecs'; -import { - ConsentTrackerStatus, - DataFlowScope, -} from '@transcend-io/privacy-types'; +import type { LocalContext } from '../../../context'; +import { writeTranscendYaml } from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; interface ConsentManagerServiceJsonToYmlCommandFlags { file: string; diff --git a/src/commands/inventory/consent-managers-to-business-entities/impl.ts b/src/commands/inventory/consent-managers-to-business-entities/impl.ts index 9d47ffce..090b203b 100644 --- a/src/commands/inventory/consent-managers-to-business-entities/impl.ts +++ b/src/commands/inventory/consent-managers-to-business-entities/impl.ts @@ -1,3 +1,6 @@ +import { existsSync, lstatSync } from 'fs'; +import { join } from 'path'; +import colors from 'colors'; import type { LocalContext } from '../../../context'; import { listFiles } from '../../../lib/api-keys'; import { consentManagersToBusinessEntities as consentManagersToBusinessEntitiesHelper } from '../../../lib/consent-manager'; @@ -5,11 +8,7 @@ import { readTranscendYaml, writeTranscendYaml, } from '../../../lib/readTranscendYaml'; -import { join } from 'path'; - -import colors from 'colors'; import { logger } from '../../../logger'; -import { existsSync, lstatSync } from 'fs'; interface ConsentManagersToBusinessEntitiesCommandFlags { consentManagerYmlFolder: string; diff --git a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts index 6425c764..9ed62e77 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts @@ -1,20 +1,20 @@ -import type { LocalContext } from '../../../context'; -import { - fetchAndIndexCatalogs, - buildTranscendGraphQLClient, -} from '../../../lib/graphql'; +import { existsSync, lstatSync } from 'fs'; import { join } from 'path'; -import { difference } from 'lodash-es'; import colors from 'colors'; -import { logger } from '../../../logger'; -import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; +import { difference } from 'lodash-es'; import { DataFlowInput } from '../../../codecs'; -import { existsSync, lstatSync } from 'fs'; +import type { LocalContext } from '../../../context'; import { listFiles } from '../../../lib/api-keys'; +import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; +import { + buildTranscendGraphQLClient, + fetchAndIndexCatalogs, +} from '../../../lib/graphql'; import { readTranscendYaml, writeTranscendYaml, } from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; interface DeriveDataSilosFromDataFlowsCrossInstanceCommandFlags { auth: string; diff --git a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts index 71541d35..7f7112fa 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts @@ -1,19 +1,19 @@ -import type { LocalContext } from '../../../context'; -import { - fetchAndIndexCatalogs, - buildTranscendGraphQLClient, -} from '../../../lib/graphql'; +import { existsSync, lstatSync } from 'fs'; import { join } from 'path'; import colors from 'colors'; -import { logger } from '../../../logger'; -import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; import { DataFlowInput } from '../../../codecs'; -import { existsSync, lstatSync } from 'fs'; +import type { LocalContext } from '../../../context'; import { listFiles } from '../../../lib/api-keys'; +import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; +import { + buildTranscendGraphQLClient, + fetchAndIndexCatalogs, +} from '../../../lib/graphql'; import { readTranscendYaml, writeTranscendYaml, } from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; interface DeriveDataSilosFromDataFlowsCommandFlags { auth: string; diff --git a/src/commands/inventory/discover-silos/impl.ts b/src/commands/inventory/discover-silos/impl.ts index 89ad545e..57341841 100644 --- a/src/commands/inventory/discover-silos/impl.ts +++ b/src/commands/inventory/discover-silos/impl.ts @@ -1,15 +1,15 @@ -import type { LocalContext } from '../../../context'; -import { stringify } from 'query-string'; -import { logger } from '../../../logger'; import colors from 'colors'; +import { stringify } from 'query-string'; import { ADMIN_DASH } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { SILO_DISCOVERY_CONFIGS } from '../../../lib/code-scanning'; +import { findFilesToScan } from '../../../lib/code-scanning/findFilesToScan'; import { - fetchActiveSiloDiscoPlugin, buildTranscendGraphQLClient, + fetchActiveSiloDiscoPlugin, uploadSiloDiscoveryResults, } from '../../../lib/graphql'; -import { findFilesToScan } from '../../../lib/code-scanning/findFilesToScan'; -import { SILO_DISCOVERY_CONFIGS } from '../../../lib/code-scanning'; +import { logger } from '../../../logger'; interface DiscoverSilosCommandFlags { scanPath: string; diff --git a/src/commands/inventory/pull-datapoints/impl.ts b/src/commands/inventory/pull-datapoints/impl.ts index 682d979c..0bab667d 100644 --- a/src/commands/inventory/pull-datapoints/impl.ts +++ b/src/commands/inventory/pull-datapoints/impl.ts @@ -1,13 +1,12 @@ -import type { LocalContext } from '../../../context'; -import { uniq, groupBy } from 'lodash-es'; - -import { logger } from '../../../logger'; +import { DataCategoryType } from '@transcend-io/privacy-types'; import colors from 'colors'; -import { buildTranscendGraphQLClient } from '../../../lib/graphql'; +import { groupBy, uniq } from 'lodash-es'; import { ADMIN_DASH_DATAPOINTS } from '../../../constants'; -import { pullAllDatapoints } from '../../../lib/data-inventory'; +import type { LocalContext } from '../../../context'; import { writeCsv } from '../../../lib/cron'; -import { DataCategoryType } from '@transcend-io/privacy-types'; +import { pullAllDatapoints } from '../../../lib/data-inventory'; +import { buildTranscendGraphQLClient } from '../../../lib/graphql'; +import { logger } from '../../../logger'; interface PullDatapointsCommandFlags { auth: string; @@ -71,10 +70,13 @@ export async function pullDatapoints( point.attributeValues || [], ({ attributeKey }) => attributeKey.name, ), - ).reduce((acc, [key, values]) => { - acc[key] = values.map((value) => value.name).join(','); - return acc; - }, {} as Record), + ).reduce( + (acc, [key, values]) => { + acc[key] = values.map((value) => value.name).join(','); + return acc; + }, + {} as Record, + ), }; headers = uniq([...headers, ...Object.keys(result)]); return result; diff --git a/src/commands/inventory/pull-unstructured-discovery-files/impl.ts b/src/commands/inventory/pull-unstructured-discovery-files/impl.ts index 6bc06c37..a66d0358 100644 --- a/src/commands/inventory/pull-unstructured-discovery-files/impl.ts +++ b/src/commands/inventory/pull-unstructured-discovery-files/impl.ts @@ -1,7 +1,7 @@ -import type { LocalContext } from '../../../context'; import type { UnstructuredSubDataPointRecommendationStatus } from '@transcend-io/privacy-types'; import colors from 'colors'; import { uniq } from 'lodash-es'; +import type { LocalContext } from '../../../context'; import { writeCsv } from '../../../lib/cron'; import { pullUnstructuredSubDataPointRecommendations } from '../../../lib/data-inventory'; import { buildTranscendGraphQLClient } from '../../../lib/graphql'; diff --git a/src/commands/inventory/pull/command.ts b/src/commands/inventory/pull/command.ts index a63f349a..bff74c8a 100644 --- a/src/commands/inventory/pull/command.ts +++ b/src/commands/inventory/pull/command.ts @@ -1,10 +1,10 @@ import { buildCommand, numberParser } from '@stricli/core'; import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; +import { TranscendPullResource } from '../../../enums'; import { createAuthParameter, createTranscendUrlParameter, } from '../../../lib/cli/common-parameters'; -import { TranscendPullResource } from '../../../enums'; export const DEFAULT_TRANSCEND_PULL_RESOURCES = [ TranscendPullResource.DataSilos, diff --git a/src/commands/inventory/pull/impl.ts b/src/commands/inventory/pull/impl.ts index d1796c9c..1bfa37ab 100644 --- a/src/commands/inventory/pull/impl.ts +++ b/src/commands/inventory/pull/impl.ts @@ -1,24 +1,22 @@ +import fs from 'fs'; +import { join } from 'path'; import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; import type { LocalContext } from '../../../context'; import { TranscendPullResource } from '../../../enums'; -import { - DEFAULT_CONSENT_TRACKER_STATUSES, - DEFAULT_TRANSCEND_PULL_RESOURCES, -} from './command'; - -import { logger } from '../../../logger'; -import colors from 'colors'; +import { validateTranscendAuth } from '../../../lib/api-keys'; import { mapSeries } from '../../../lib/bluebird-replace'; -import { join } from 'path'; -import fs from 'fs'; import { buildTranscendGraphQLClient, pullTranscendConfiguration, } from '../../../lib/graphql'; - import { writeTranscendYaml } from '../../../lib/readTranscendYaml'; -import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; -import { validateTranscendAuth } from '../../../lib/api-keys'; +import { logger } from '../../../logger'; +import { + DEFAULT_CONSENT_TRACKER_STATUSES, + DEFAULT_TRANSCEND_PULL_RESOURCES, +} from './command'; interface PullCommandFlags { auth: string; diff --git a/src/commands/inventory/push/impl.ts b/src/commands/inventory/push/impl.ts index 1904d6da..7bac297a 100644 --- a/src/commands/inventory/push/impl.ts +++ b/src/commands/inventory/push/impl.ts @@ -1,21 +1,19 @@ -import type { LocalContext } from '../../../context'; - -import { logger } from '../../../logger'; -import { mapSeries } from '../../../lib/bluebird-replace'; import { existsSync, lstatSync } from 'fs'; import { join } from 'path'; -import { readTranscendYaml } from '../../../lib/readTranscendYaml'; import colors from 'colors'; +import { TranscendInput } from '../../../codecs'; +import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { listFiles, validateTranscendAuth } from '../../../lib/api-keys'; +import { mapSeries } from '../../../lib/bluebird-replace'; import { buildTranscendGraphQLClient, syncConfigurationToTranscend, } from '../../../lib/graphql'; - -import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; -import { TranscendInput } from '../../../codecs'; -import { validateTranscendAuth, listFiles } from '../../../lib/api-keys'; -import { mergeTranscendInputs } from '../../../lib/mergeTranscendInputs'; import { parseVariablesFromString } from '../../../lib/helpers/parseVariablesFromString'; +import { mergeTranscendInputs } from '../../../lib/mergeTranscendInputs'; +import { readTranscendYaml } from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; /** * Sync configuration to Transcend diff --git a/src/commands/inventory/routes.ts b/src/commands/inventory/routes.ts index 39613782..93f500f1 100644 --- a/src/commands/inventory/routes.ts +++ b/src/commands/inventory/routes.ts @@ -1,4 +1,6 @@ import { buildRouteMap } from '@stricli/core'; +import { consentManagerServiceJsonToYmlCommand } from './consent-manager-service-json-to-yml/command'; +import { consentManagersToBusinessEntitiesCommand } from './consent-managers-to-business-entities/command'; import { deriveDataSilosFromDataFlowsCrossInstanceCommand } from './derive-data-silos-from-data-flows-cross-instance/command'; import { deriveDataSilosFromDataFlowsCommand } from './derive-data-silos-from-data-flows/command'; import { discoverSilosCommand } from './discover-silos/command'; @@ -7,8 +9,6 @@ import { pullUnstructuredDiscoveryFilesCommand } from './pull-unstructured-disco import { pullCommand } from './pull/command'; import { pushCommand } from './push/command'; import { scanPackagesCommand } from './scan-packages/command'; -import { consentManagerServiceJsonToYmlCommand } from './consent-manager-service-json-to-yml/command'; -import { consentManagersToBusinessEntitiesCommand } from './consent-managers-to-business-entities/command'; export const inventoryRoutes = buildRouteMap({ routes: { diff --git a/src/commands/inventory/scan-packages/impl.ts b/src/commands/inventory/scan-packages/impl.ts index a8c6c49e..e06466f1 100644 --- a/src/commands/inventory/scan-packages/impl.ts +++ b/src/commands/inventory/scan-packages/impl.ts @@ -1,13 +1,13 @@ -import type { LocalContext } from '../../../context'; -import { logger } from '../../../logger'; +import { execSync } from 'child_process'; import colors from 'colors'; import { ADMIN_DASH } from '../../../constants'; +import type { LocalContext } from '../../../context'; import { findCodePackagesInFolder } from '../../../lib/code-scanning'; import { buildTranscendGraphQLClient, syncCodePackages, } from '../../../lib/graphql'; -import { execSync } from 'child_process'; +import { logger } from '../../../logger'; const REPO_ERROR = 'A repository name must be provided. ' + diff --git a/src/commands/migration/sync-ot/command.ts b/src/commands/migration/sync-ot/command.ts index df3fa1eb..0d81fab6 100644 --- a/src/commands/migration/sync-ot/command.ts +++ b/src/commands/migration/sync-ot/command.ts @@ -1,11 +1,11 @@ import { buildCommand, type TypedFlagParameter } from '@stricli/core'; import { ScopeName } from '@transcend-io/privacy-types'; +import type { LocalContext } from '../../../context'; +import { OneTrustPullResource, OneTrustPullSource } from '../../../enums'; import { createAuthParameter, createTranscendUrlParameter, } from '../../../lib/cli/common-parameters'; -import { OneTrustPullResource, OneTrustPullSource } from '../../../enums'; -import type { LocalContext } from '../../../context'; export const syncOtCommand = buildCommand({ loader: async () => { diff --git a/src/commands/migration/sync-ot/impl.ts b/src/commands/migration/sync-ot/impl.ts index d9ef4db3..a2ed2d4e 100644 --- a/src/commands/migration/sync-ot/impl.ts +++ b/src/commands/migration/sync-ot/impl.ts @@ -1,17 +1,17 @@ -import type { LocalContext } from '../../../context'; -import { logger } from '../../../logger'; import colors from 'colors'; -import { createOneTrustGotInstance } from '../../../lib/oneTrust'; +import type { LocalContext } from '../../../context'; import { OneTrustFileFormat, OneTrustPullResource, OneTrustPullSource, } from '../../../enums'; import { buildTranscendGraphQLClient } from '../../../lib/graphql'; +import { createOneTrustGotInstance } from '../../../lib/oneTrust'; import { syncOneTrustAssessmentsFromFile, syncOneTrustAssessmentsFromOneTrust, } from '../../../lib/oneTrust/helpers'; +import { logger } from '../../../logger'; // Command flag interface interface SyncOtCommandFlags { diff --git a/src/commands/request/approve/impl.ts b/src/commands/request/approve/impl.ts index 5c36465c..803cc779 100644 --- a/src/commands/request/approve/impl.ts +++ b/src/commands/request/approve/impl.ts @@ -1,6 +1,5 @@ -import type { LocalContext } from '../../../context'; - import { RequestAction, RequestOrigin } from '@transcend-io/privacy-types'; +import type { LocalContext } from '../../../context'; import { approvePrivacyRequests } from '../../../lib/requests'; interface ApproveCommandFlags { diff --git a/src/commands/request/cancel/impl.ts b/src/commands/request/cancel/impl.ts index c7e9993e..80504a4c 100644 --- a/src/commands/request/cancel/impl.ts +++ b/src/commands/request/cancel/impl.ts @@ -1,5 +1,5 @@ -import type { LocalContext } from '../../../context'; import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import type { LocalContext } from '../../../context'; import { cancelPrivacyRequests } from '../../../lib/requests'; interface CancelCommandFlags { diff --git a/src/commands/request/cron/pull-identifiers/command.ts b/src/commands/request/cron/pull-identifiers/command.ts index ad8a628a..97a6d4f4 100644 --- a/src/commands/request/cron/pull-identifiers/command.ts +++ b/src/commands/request/cron/pull-identifiers/command.ts @@ -1,11 +1,11 @@ import { buildCommand, numberParser } from '@stricli/core'; +import { RequestAction } from '@transcend-io/privacy-types'; import { createAuthParameter, createSombraAuthParameter, createTranscendUrlParameter, } from '../../../../lib/cli/common-parameters'; import { uuidParser } from '../../../../lib/cli/parsers'; -import { RequestAction } from '@transcend-io/privacy-types'; export const pullIdentifiersCommand = buildCommand({ loader: async () => { diff --git a/src/commands/request/cron/pull-identifiers/impl.ts b/src/commands/request/cron/pull-identifiers/impl.ts index 2dc503ae..49ea95c2 100644 --- a/src/commands/request/cron/pull-identifiers/impl.ts +++ b/src/commands/request/cron/pull-identifiers/impl.ts @@ -1,15 +1,14 @@ -import type { LocalContext } from '../../../../context'; +import { RequestAction } from '@transcend-io/privacy-types'; import colors from 'colors'; - -import { logger } from '../../../../logger'; import { uniq } from 'lodash-es'; +import type { LocalContext } from '../../../../context'; import { CsvFormattedIdentifier, parseFilePath, pullChunkedCustomSiloOutstandingIdentifiers, writeCsv, } from '../../../../lib/cron'; -import { RequestAction } from '@transcend-io/privacy-types'; +import { logger } from '../../../../logger'; interface PullIdentifiersCommandFlags { file: string; diff --git a/src/commands/request/cron/pull-profiles/command.ts b/src/commands/request/cron/pull-profiles/command.ts index e20ecfd4..9294f319 100644 --- a/src/commands/request/cron/pull-profiles/command.ts +++ b/src/commands/request/cron/pull-profiles/command.ts @@ -1,11 +1,11 @@ import { buildCommand, numberParser } from '@stricli/core'; +import { RequestAction } from '@transcend-io/privacy-types'; import { createAuthParameter, createSombraAuthParameter, createTranscendUrlParameter, } from '../../../../lib/cli/common-parameters'; import { uuidParser } from '../../../../lib/cli/parsers'; -import { RequestAction } from '@transcend-io/privacy-types'; export const pullProfilesCommand = buildCommand({ loader: async () => { diff --git a/src/commands/request/cron/pull-profiles/impl.ts b/src/commands/request/cron/pull-profiles/impl.ts index 8c7ec0ec..1d791cb6 100644 --- a/src/commands/request/cron/pull-profiles/impl.ts +++ b/src/commands/request/cron/pull-profiles/impl.ts @@ -1,19 +1,19 @@ import type { RequestAction } from '@transcend-io/privacy-types'; -import { logger } from '../../../../logger'; import colors from 'colors'; import { uniq } from 'lodash-es'; -import { map } from '../../../../lib/bluebird-replace'; -import { - buildTranscendGraphQLClient, - fetchRequestFilesForRequest, -} from '../../../../lib/graphql'; import type { LocalContext } from '../../../../context'; +import { map } from '../../../../lib/bluebird-replace'; import { parseFilePath, pullChunkedCustomSiloOutstandingIdentifiers, writeCsv, type CsvFormattedIdentifier, } from '../../../../lib/cron'; +import { + buildTranscendGraphQLClient, + fetchRequestFilesForRequest, +} from '../../../../lib/graphql'; +import { logger } from '../../../../logger'; interface PullProfilesCommandFlags { file: string; diff --git a/src/commands/request/download-files/impl.ts b/src/commands/request/download-files/impl.ts index 06eceaa2..9828837b 100644 --- a/src/commands/request/download-files/impl.ts +++ b/src/commands/request/download-files/impl.ts @@ -1,6 +1,6 @@ +import { RequestStatus } from '@transcend-io/privacy-types'; import type { LocalContext } from '../../../context'; import { downloadPrivacyRequestFiles } from '../../../lib/requests'; -import { RequestStatus } from '@transcend-io/privacy-types'; interface DownloadFilesCommandFlags { auth: string; diff --git a/src/commands/request/enricher-restart/impl.ts b/src/commands/request/enricher-restart/impl.ts index bef41059..6a28894e 100644 --- a/src/commands/request/enricher-restart/impl.ts +++ b/src/commands/request/enricher-restart/impl.ts @@ -1,9 +1,9 @@ -import type { LocalContext } from '../../../context'; -import { bulkRetryEnrichers } from '../../../lib/requests'; import type { RequestAction, RequestEnricherStatus, } from '@transcend-io/privacy-types'; +import type { LocalContext } from '../../../context'; +import { bulkRetryEnrichers } from '../../../lib/requests'; interface EnricherRestartCommandFlags { auth: string; diff --git a/src/commands/request/export/impl.ts b/src/commands/request/export/impl.ts index 7df4258e..0b1a939d 100644 --- a/src/commands/request/export/impl.ts +++ b/src/commands/request/export/impl.ts @@ -1,11 +1,10 @@ -import type { LocalContext } from '../../../context'; +import type { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; import colors from 'colors'; - -import { logger } from '../../../logger'; import { uniq } from 'lodash-es'; -import { pullPrivacyRequests } from '../../../lib/requests'; +import type { LocalContext } from '../../../context'; import { writeCsv } from '../../../lib/cron'; -import type { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import { pullPrivacyRequests } from '../../../lib/requests'; +import { logger } from '../../../logger'; interface ExportCommandFlags { auth: string; diff --git a/src/commands/request/mark-silent/impl.ts b/src/commands/request/mark-silent/impl.ts index 21ba1bc7..5c4e0840 100644 --- a/src/commands/request/mark-silent/impl.ts +++ b/src/commands/request/mark-silent/impl.ts @@ -1,6 +1,6 @@ +import type { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; import type { LocalContext } from '../../../context'; import { markSilentPrivacyRequests } from '../../../lib/requests'; -import type { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; interface MarkSilentCommandFlags { auth: string; diff --git a/src/commands/request/notify-additional-time/impl.ts b/src/commands/request/notify-additional-time/impl.ts index 1c9e061e..df0ad3f9 100644 --- a/src/commands/request/notify-additional-time/impl.ts +++ b/src/commands/request/notify-additional-time/impl.ts @@ -1,6 +1,6 @@ +import type { RequestAction } from '@transcend-io/privacy-types'; import type { LocalContext } from '../../../context'; import { notifyPrivacyRequestsAdditionalTime } from '../../../lib/requests'; -import type { RequestAction } from '@transcend-io/privacy-types'; interface NotifyAdditionalTimeCommandFlags { auth: string; diff --git a/src/commands/request/preflight/pull-identifiers/impl.ts b/src/commands/request/preflight/pull-identifiers/impl.ts index 62638177..f6a1666f 100644 --- a/src/commands/request/preflight/pull-identifiers/impl.ts +++ b/src/commands/request/preflight/pull-identifiers/impl.ts @@ -1,6 +1,6 @@ +import type { RequestAction } from '@transcend-io/privacy-types'; import type { LocalContext } from '../../../../context'; import { pullManualEnrichmentIdentifiersToCsv } from '../../../../lib/manual-enrichment'; -import type { RequestAction } from '@transcend-io/privacy-types'; interface PullIdentifiersCommandFlags { auth: string; diff --git a/src/commands/request/reject-unverified-identifiers/impl.ts b/src/commands/request/reject-unverified-identifiers/impl.ts index 5b381f87..a49015c8 100644 --- a/src/commands/request/reject-unverified-identifiers/impl.ts +++ b/src/commands/request/reject-unverified-identifiers/impl.ts @@ -1,6 +1,6 @@ +import type { RequestAction } from '@transcend-io/privacy-types'; import type { LocalContext } from '../../../context'; import { removeUnverifiedRequestIdentifiers } from '../../../lib/requests'; -import type { RequestAction } from '@transcend-io/privacy-types'; interface RejectUnverifiedIdentifiersCommandFlags { auth: string; diff --git a/src/commands/request/restart/impl.ts b/src/commands/request/restart/impl.ts index e2e6a665..73a84202 100644 --- a/src/commands/request/restart/impl.ts +++ b/src/commands/request/restart/impl.ts @@ -1,6 +1,6 @@ +import type { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; import type { LocalContext } from '../../../context'; import { bulkRestartRequests } from '../../../lib/requests'; -import type { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; interface RestartCommandFlags { auth: string; diff --git a/src/commands/request/system/mark-request-data-silos-completed/impl.ts b/src/commands/request/system/mark-request-data-silos-completed/impl.ts index b30bcb27..0def94b3 100644 --- a/src/commands/request/system/mark-request-data-silos-completed/impl.ts +++ b/src/commands/request/system/mark-request-data-silos-completed/impl.ts @@ -1,10 +1,9 @@ -import type { LocalContext } from '../../../../context'; import colors from 'colors'; import * as t from 'io-ts'; - -import { logger } from '../../../../logger'; +import type { LocalContext } from '../../../../context'; import { markRequestDataSiloIdsCompleted } from '../../../../lib/cron'; import { readCsv } from '../../../../lib/requests'; +import { logger } from '../../../../logger'; const RequestIdRow = t.type({ 'Request Id': t.string, diff --git a/src/commands/request/system/retry-request-data-silos/impl.ts b/src/commands/request/system/retry-request-data-silos/impl.ts index 80f331a1..ce739e85 100644 --- a/src/commands/request/system/retry-request-data-silos/impl.ts +++ b/src/commands/request/system/retry-request-data-silos/impl.ts @@ -1,5 +1,5 @@ -import type { LocalContext } from '../../../../context'; import type { RequestAction } from '@transcend-io/privacy-types'; +import type { LocalContext } from '../../../../context'; import { retryRequestDataSilos as retryRequestDataSilosHelper } from '../../../../lib/requests'; interface RetryRequestDataSilosCommandFlags { diff --git a/src/commands/request/system/skip-request-data-silos/impl.ts b/src/commands/request/system/skip-request-data-silos/impl.ts index 38a1e0a8..65af084d 100644 --- a/src/commands/request/system/skip-request-data-silos/impl.ts +++ b/src/commands/request/system/skip-request-data-silos/impl.ts @@ -1,5 +1,5 @@ -import type { LocalContext } from '../../../../context'; import type { RequestStatus } from '@transcend-io/privacy-types'; +import type { LocalContext } from '../../../../context'; import { skipRequestDataSilos as skipRequestDataSilosHelper } from '../../../../lib/requests'; interface SkipRequestDataSilosCommandFlags { diff --git a/src/constants.ts b/src/constants.ts index 0c2484b4..ba8b68ea 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,6 +1,6 @@ import { ScopeName } from '@transcend-io/privacy-types'; -import { TranscendPullResource } from './enums'; import { TranscendInput } from './codecs'; +import { TranscendPullResource } from './enums'; export { description, version } from '../package.json'; export const name = 'transcend'; diff --git a/src/lib/ai/TranscendPromptManager.ts b/src/lib/ai/TranscendPromptManager.ts index 1299199e..5b56750c 100644 --- a/src/lib/ai/TranscendPromptManager.ts +++ b/src/lib/ai/TranscendPromptManager.ts @@ -1,51 +1,51 @@ /* eslint-disable max-lines */ +import type { Handlebars } from '@transcend-io/handlebars-utils'; +import { + createHandlebarsWithHelpers, + HandlebarsInput, +} from '@transcend-io/handlebars-utils'; +import { + ChatCompletionRole, + LargeLanguageModelClient, + PromptRunProductArea, + PromptStatus, + QueueStatus, +} from '@transcend-io/privacy-types'; +import { Secret } from '@transcend-io/secret-value'; import { - Optionalize, - Requirize, apply, decodeCodec, getValues, + Optionalize, + Requirize, } from '@transcend-io/type-utils'; -import type { Handlebars } from '@transcend-io/handlebars-utils'; -import { Secret } from '@transcend-io/secret-value'; +import { GraphQLClient } from 'graphql-request'; import * as t from 'io-ts'; +import { chunk, groupBy, keyBy, uniq } from 'lodash-es'; import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { mapSeries } from '../bluebird-replace'; import { Agent, AgentFile, AgentFileFilterBy, - ReportPromptRunInput, buildTranscendGraphQLClient, fetchAllAgentFiles, fetchAllAgents, reportPromptRun, + ReportPromptRunInput, } from '../graphql'; import { - HandlebarsInput, - createHandlebarsWithHelpers, -} from '@transcend-io/handlebars-utils'; + fetchAllLargeLanguageModels, + LargeLanguageModel, +} from '../graphql/fetchLargeLanguageModels'; import { - TranscendPromptTemplated, - TranscendPromptsAndVariables, fetchPromptsWithVariables, + TranscendPromptsAndVariables, + TranscendPromptTemplated, } from '../graphql/fetchPrompts'; -import { GraphQLClient } from 'graphql-request'; -import { - PromptStatus, - ChatCompletionRole, - PromptRunProductArea, - QueueStatus, - LargeLanguageModelClient, -} from '@transcend-io/privacy-types'; -import { - LargeLanguageModel, - fetchAllLargeLanguageModels, -} from '../graphql/fetchLargeLanguageModels'; -import { groupBy, keyBy, uniq, chunk } from 'lodash-es'; -import { mapSeries } from '../bluebird-replace'; import { - PromptThread, fetchAllPromptThreads, + PromptThread, } from '../graphql/fetchPromptThreads'; /** @@ -340,8 +340,8 @@ export class TranscendPromptManager< const result = id ? promptById[id] : title - ? promptByTitle[title] - : undefined; + ? promptByTitle[title] + : undefined; if (!result) { throw new Error( `Failed to find prompt with title: "${title}" and id: "${id}"`, diff --git a/src/lib/ai/getGitFilesThatChanged.ts b/src/lib/ai/getGitFilesThatChanged.ts index 1f9da166..717b3933 100644 --- a/src/lib/ai/getGitFilesThatChanged.ts +++ b/src/lib/ai/getGitFilesThatChanged.ts @@ -1,6 +1,6 @@ -import { difference } from 'lodash-es'; -import fastGlob from 'fast-glob'; import { execSync } from 'child_process'; +import fastGlob from 'fast-glob'; +import { difference } from 'lodash-es'; /** * Function thats gets the git files that have changed diff --git a/src/lib/api-keys/generateCrossAccountApiKeys.ts b/src/lib/api-keys/generateCrossAccountApiKeys.ts index 9f1ce407..420465ba 100644 --- a/src/lib/api-keys/generateCrossAccountApiKeys.ts +++ b/src/lib/api-keys/generateCrossAccountApiKeys.ts @@ -1,17 +1,17 @@ +import { ScopeName } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { StoredApiKey } from '../../codecs'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; import { mapSeries } from '../bluebird-replace'; import { + assumeRole, buildTranscendGraphQLClientGeneric, - loginUser, createApiKey, - fetchAllApiKeys, deleteApiKey, - assumeRole, + fetchAllApiKeys, + loginUser, } from '../graphql'; -import { ScopeName } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { StoredApiKey } from '../../codecs'; -import { logger } from '../../logger'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; export interface ApiKeyGenerateError { /** Name of instance */ diff --git a/src/lib/api-keys/validateTranscendAuth.ts b/src/lib/api-keys/validateTranscendAuth.ts index e907022e..0db9b392 100644 --- a/src/lib/api-keys/validateTranscendAuth.ts +++ b/src/lib/api-keys/validateTranscendAuth.ts @@ -1,9 +1,9 @@ +import { existsSync, readFileSync } from 'fs'; import { decodeCodec } from '@transcend-io/type-utils'; import colors from 'colors'; import * as t from 'io-ts'; -import { logger } from '../../logger'; -import { existsSync, readFileSync } from 'fs'; import { StoredApiKey } from '../../codecs'; +import { logger } from '../../logger'; /** * Determine if the `--auth` parameter is an API key or a path to a JSON diff --git a/src/lib/cli/common-parameters.ts b/src/lib/cli/common-parameters.ts index 45ba8ae5..acafbca8 100644 --- a/src/lib/cli/common-parameters.ts +++ b/src/lib/cli/common-parameters.ts @@ -1,11 +1,11 @@ +import type { TypedFlagParameter } from '@stricli/core'; import { ScopeName, TRANSCEND_SCOPES } from '@transcend-io/privacy-types'; -import { urlParser } from './parsers'; import { DEFAULT_TRANSCEND_API, DEFAULT_TRANSCEND_CONSENT_API, } from '../../constants'; import type { LocalContext } from '../../context'; -import type { TypedFlagParameter } from '@stricli/core'; +import { urlParser } from './parsers'; /** * Common parameter builders for CLI commands diff --git a/src/lib/code-scanning/constants.ts b/src/lib/code-scanning/constants.ts index 5af068a0..a7225a3a 100644 --- a/src/lib/code-scanning/constants.ts +++ b/src/lib/code-scanning/constants.ts @@ -1,15 +1,15 @@ -import { CodeScanningConfig } from './types'; +import { CodePackageType } from '@transcend-io/privacy-types'; import { cocoaPods, + composerJson, + gemfile, gradle, javascriptPackageJson, - gemfile, - composerJson, pubspec, - swift, pythonRequirementsTxt, + swift, } from './integrations'; -import { CodePackageType } from '@transcend-io/privacy-types'; +import { CodeScanningConfig } from './types'; /** * @deprecated TODO: https://transcend.height.app/T-32325 - use code scanning instead diff --git a/src/lib/code-scanning/findCodePackagesInFolder.ts b/src/lib/code-scanning/findCodePackagesInFolder.ts index 903a879b..77c0c58c 100644 --- a/src/lib/code-scanning/findCodePackagesInFolder.ts +++ b/src/lib/code-scanning/findCodePackagesInFolder.ts @@ -1,9 +1,9 @@ -import fastGlob from 'fast-glob'; +import { getEntries } from '@transcend-io/type-utils'; import colors from 'colors'; +import fastGlob from 'fast-glob'; import { CodePackageInput } from '../../codecs'; -import { getEntries } from '@transcend-io/type-utils'; -import { CODE_SCANNING_CONFIGS } from './constants'; import { logger } from '../../logger'; +import { CODE_SCANNING_CONFIGS } from './constants'; /** * Helper to scan and discovery all of the code packages within a folder diff --git a/src/lib/code-scanning/integrations/cocoaPods.ts b/src/lib/code-scanning/integrations/cocoaPods.ts index c0b774bd..b0fcaa38 100644 --- a/src/lib/code-scanning/integrations/cocoaPods.ts +++ b/src/lib/code-scanning/integrations/cocoaPods.ts @@ -1,8 +1,8 @@ import { readFileSync } from 'fs'; -import { CodeScanningConfig } from '../types'; -import { CodePackageSdk } from '../../../codecs'; -import { findAllWithRegex } from '@transcend-io/type-utils'; import { CodePackageType } from '@transcend-io/privacy-types'; +import { findAllWithRegex } from '@transcend-io/type-utils'; +import { CodePackageSdk } from '../../../codecs'; +import { CodeScanningConfig } from '../types'; const POD_TARGET_REGEX = /target ('|")(.*?)('|")/; const POD_PACKAGE_REGEX = /pod ('|")(.*?)('|")(, ('|")~> (.+?)('|")|)/; diff --git a/src/lib/code-scanning/integrations/composerJson.ts b/src/lib/code-scanning/integrations/composerJson.ts index 48ab2385..82f7f54c 100644 --- a/src/lib/code-scanning/integrations/composerJson.ts +++ b/src/lib/code-scanning/integrations/composerJson.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs'; -import { CodeScanningConfig } from '../types'; -import { CodePackageSdk } from '../../../codecs'; import { dirname } from 'path'; +import { CodePackageSdk } from '../../../codecs'; +import { CodeScanningConfig } from '../types'; export const composerJson: CodeScanningConfig = { supportedFiles: ['composer.json'], diff --git a/src/lib/code-scanning/integrations/gemfile.ts b/src/lib/code-scanning/integrations/gemfile.ts index af7a7be2..e9b6ca3a 100644 --- a/src/lib/code-scanning/integrations/gemfile.ts +++ b/src/lib/code-scanning/integrations/gemfile.ts @@ -1,9 +1,9 @@ import { readFileSync } from 'fs'; -import { CodeScanningConfig } from '../types'; -import { findAllWithRegex } from '@transcend-io/type-utils'; -import { listFiles } from '../../api-keys'; import { dirname } from 'path'; import { CodePackageType } from '@transcend-io/privacy-types'; +import { findAllWithRegex } from '@transcend-io/type-utils'; +import { listFiles } from '../../api-keys'; +import { CodeScanningConfig } from '../types'; const GEM_PACKAGE_REGEX = /gem *('|")(.+?)('|")(, *('|")(.+?)('|")|)/; const GEMFILE_PACKAGE_NAME_REGEX = /spec\.name *= *('|")(.+?)('|")/; diff --git a/src/lib/code-scanning/integrations/gradle.ts b/src/lib/code-scanning/integrations/gradle.ts index 1f36ff4b..a29431ef 100644 --- a/src/lib/code-scanning/integrations/gradle.ts +++ b/src/lib/code-scanning/integrations/gradle.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs'; -import { CodeScanningConfig } from '../types'; -import { findAllWithRegex } from '@transcend-io/type-utils'; import { dirname } from 'path'; +import { findAllWithRegex } from '@transcend-io/type-utils'; +import { CodeScanningConfig } from '../types'; const GRADLE_IMPLEMENTATION_REGEX = /implementation( *)('|")(.+?):(.+?):(.+?|)('|")/; diff --git a/src/lib/code-scanning/integrations/javascriptPackageJson.ts b/src/lib/code-scanning/integrations/javascriptPackageJson.ts index 68739429..99ae0164 100644 --- a/src/lib/code-scanning/integrations/javascriptPackageJson.ts +++ b/src/lib/code-scanning/integrations/javascriptPackageJson.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs'; -import { CodeScanningConfig } from '../types'; -import { CodePackageSdk } from '../../../codecs'; import { dirname } from 'path'; +import { CodePackageSdk } from '../../../codecs'; +import { CodeScanningConfig } from '../types'; export const javascriptPackageJson: CodeScanningConfig = { supportedFiles: ['package.json'], diff --git a/src/lib/code-scanning/integrations/pubspec.ts b/src/lib/code-scanning/integrations/pubspec.ts index c427601b..0ef1ccd5 100644 --- a/src/lib/code-scanning/integrations/pubspec.ts +++ b/src/lib/code-scanning/integrations/pubspec.ts @@ -1,8 +1,8 @@ import { readFileSync } from 'fs'; -import { CodeScanningConfig } from '../types'; +import { dirname } from 'path'; import { CodePackageType } from '@transcend-io/privacy-types'; import yaml from 'js-yaml'; -import { dirname } from 'path'; +import { CodeScanningConfig } from '../types'; /** * Remove YAML comments from a string @@ -64,8 +64,8 @@ export const pubspec: CodeScanningConfig = { typeof version === 'string' ? version : typeof version === 'number' - ? version.toString() - : version?.sdk, + ? version.toString() + : version?.sdk, })), ...Object.entries(dev_dependencies).map(([name, version]) => ({ name, @@ -73,8 +73,8 @@ export const pubspec: CodeScanningConfig = { typeof version === 'string' ? version : typeof version === 'number' - ? version.toString() - : version?.sdk, + ? version.toString() + : version?.sdk, isDevDependency: true, })), ], diff --git a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts index 219e5443..237a46b8 100644 --- a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts +++ b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts @@ -1,9 +1,9 @@ import { readFileSync } from 'fs'; -import { CodeScanningConfig } from '../types'; -import { findAllWithRegex } from '@transcend-io/type-utils'; -import { listFiles } from '../../api-keys'; import { dirname, join } from 'path'; import { CodePackageType } from '@transcend-io/privacy-types'; +import { findAllWithRegex } from '@transcend-io/type-utils'; +import { listFiles } from '../../api-keys'; +import { CodeScanningConfig } from '../types'; const REQUIREMENTS_PACKAGE_MATCH = /(.+?)(=+)(.+)/; const PACKAGE_NAME = /name *= *('|")(.+?)('|")/; diff --git a/src/lib/code-scanning/integrations/swift.ts b/src/lib/code-scanning/integrations/swift.ts index 5b864a04..c4a71501 100644 --- a/src/lib/code-scanning/integrations/swift.ts +++ b/src/lib/code-scanning/integrations/swift.ts @@ -1,9 +1,9 @@ import { readFileSync } from 'fs'; -import { CodeScanningConfig } from '../types'; +import { dirname } from 'path'; import { CodePackageType } from '@transcend-io/privacy-types'; import { decodeCodec } from '@transcend-io/type-utils'; import * as t from 'io-ts'; -import { dirname } from 'path'; +import { CodeScanningConfig } from '../types'; const SwiftPackage = t.type({ pins: t.array( diff --git a/src/lib/consent-manager/buildXdiSyncEndpoint.ts b/src/lib/consent-manager/buildXdiSyncEndpoint.ts index 38711fcf..3a11ecb7 100644 --- a/src/lib/consent-manager/buildXdiSyncEndpoint.ts +++ b/src/lib/consent-manager/buildXdiSyncEndpoint.ts @@ -1,11 +1,10 @@ import colors from 'colors'; - -import { buildTranscendGraphQLClient, fetchConsentManager } from '../graphql'; import { difference } from 'lodash-es'; -import { map } from '../bluebird-replace'; import { StoredApiKey } from '../../codecs'; import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { buildTranscendGraphQLClient, fetchConsentManager } from '../graphql'; import { domainToHost } from './domainToHost'; /** diff --git a/src/lib/consent-manager/dataFlowsToDataSilos.ts b/src/lib/consent-manager/dataFlowsToDataSilos.ts index 03da7f98..fb83e176 100644 --- a/src/lib/consent-manager/dataFlowsToDataSilos.ts +++ b/src/lib/consent-manager/dataFlowsToDataSilos.ts @@ -1,5 +1,5 @@ -import { DataFlowInput, DataSiloInput } from '../../codecs'; import { union } from 'lodash-es'; +import { DataFlowInput, DataSiloInput } from '../../codecs'; import { IndexedCatalogs } from '../graphql'; /** diff --git a/src/lib/consent-manager/fetchConsentPreferences.ts b/src/lib/consent-manager/fetchConsentPreferences.ts index 5d78a43b..dbecc85f 100644 --- a/src/lib/consent-manager/fetchConsentPreferences.ts +++ b/src/lib/consent-manager/fetchConsentPreferences.ts @@ -1,6 +1,6 @@ -import * as t from 'io-ts'; import { decodeCodec } from '@transcend-io/type-utils'; import type { Got } from 'got'; +import * as t from 'io-ts'; import { ConsentPreferenceFetch } from './types'; export const ConsentPreferenceResponse = t.intersection([ diff --git a/src/lib/consent-manager/updateConsentManagerVersionToLatest.ts b/src/lib/consent-manager/updateConsentManagerVersionToLatest.ts index edeafda0..f846ff00 100644 --- a/src/lib/consent-manager/updateConsentManagerVersionToLatest.ts +++ b/src/lib/consent-manager/updateConsentManagerVersionToLatest.ts @@ -1,15 +1,14 @@ import { ConsentBundleType } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; import { mapSeries } from '../bluebird-replace'; import { - updateConsentManagerToLatest, buildTranscendGraphQLClient, - fetchConsentManagerId, deployConsentManager, + fetchConsentManagerId, + updateConsentManagerToLatest, } from '../graphql'; -import colors from 'colors'; - -import { logger } from '../../logger'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; /** * Update the consent manager to latest version diff --git a/src/lib/consent-manager/uploadConsents.ts b/src/lib/consent-manager/uploadConsents.ts index 628b2ac9..b80063e2 100644 --- a/src/lib/consent-manager/uploadConsents.ts +++ b/src/lib/consent-manager/uploadConsents.ts @@ -1,14 +1,14 @@ -import { createTranscendConsentGotInstance } from '../graphql'; +import { ConsentPreferencesBody } from '@transcend-io/airgap.js-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; import colors from 'colors'; import * as t from 'io-ts'; import { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants'; +import { logger } from '../../logger'; import { map } from '../bluebird-replace'; +import { createTranscendConsentGotInstance } from '../graphql'; import { createConsentToken } from './createConsentToken'; -import { logger } from '../../logger'; -import cliProgress from 'cli-progress'; -import { decodeCodec } from '@transcend-io/type-utils'; import type { ConsentPreferenceUpload } from './types'; -import { ConsentPreferencesBody } from '@transcend-io/airgap.js-types'; export const USP_STRING_REGEX = /^[0-9][Y|N]([Y|N])[Y|N]$/; @@ -144,8 +144,8 @@ export async function uploadConsents({ purposes: purposes ? decodeCodec(PurposeMap, purposes) : consent.usp - ? { SaleOfInfo: saleStatus === 'Y' } - : {}, + ? { SaleOfInfo: saleStatus === 'Y' } + : {}, ...(updated ? { updated: updated === 'true' } : {}), ...(prompted ? { prompted: prompted === 'true' } : {}), ...consent, diff --git a/src/lib/consent-manager/uploadCookiesFromCsv.ts b/src/lib/consent-manager/uploadCookiesFromCsv.ts index 1b0ef50b..02d40635 100644 --- a/src/lib/consent-manager/uploadCookiesFromCsv.ts +++ b/src/lib/consent-manager/uploadCookiesFromCsv.ts @@ -1,11 +1,11 @@ +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; import colors from 'colors'; +import { CookieCsvInput, CookieInput } from '../../codecs'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; -import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; import { buildTranscendGraphQLClient, syncCookies } from '../graphql'; -import { readCsv } from '../requests/readCsv'; -import { CookieInput, CookieCsvInput } from '../../codecs'; import { splitCsvToList } from '../requests'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { readCsv } from '../requests/readCsv'; const OMIT_COLUMNS = [ 'ID', diff --git a/src/lib/consent-manager/uploadDataFlowsFromCsv.ts b/src/lib/consent-manager/uploadDataFlowsFromCsv.ts index fcb18d35..c9b1ed87 100644 --- a/src/lib/consent-manager/uploadDataFlowsFromCsv.ts +++ b/src/lib/consent-manager/uploadDataFlowsFromCsv.ts @@ -1,11 +1,11 @@ +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; import colors from 'colors'; +import { DataFlowCsvInput, DataFlowInput } from '../../codecs'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; -import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; import { buildTranscendGraphQLClient, syncDataFlows } from '../graphql'; -import { readCsv } from '../requests/readCsv'; -import { DataFlowInput, DataFlowCsvInput } from '../../codecs'; import { splitCsvToList } from '../requests'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { readCsv } from '../requests/readCsv'; const OMIT_COLUMNS = [ 'ID', diff --git a/src/lib/cron/markRequestDataSiloIdsCompleted.ts b/src/lib/cron/markRequestDataSiloIdsCompleted.ts index d6b5983c..a6b6566a 100644 --- a/src/lib/cron/markRequestDataSiloIdsCompleted.ts +++ b/src/lib/cron/markRequestDataSiloIdsCompleted.ts @@ -1,15 +1,15 @@ -import { map } from '../bluebird-replace'; +import { RequestDataSiloStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { + buildTranscendGraphQLClient, CHANGE_REQUEST_DATA_SILO_STATUS, fetchRequestDataSilo, makeGraphQLRequest, - buildTranscendGraphQLClient, } from '../graphql'; -import cliProgress from 'cli-progress'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { RequestDataSiloStatus } from '@transcend-io/privacy-types'; /** * Given a CSV of Request IDs, mark associated RequestDataSilos as completed diff --git a/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts b/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts index 07d9c6b1..a35a0755 100644 --- a/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts +++ b/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts @@ -1,19 +1,18 @@ +import { RequestAction } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchRequestDataSiloActiveCount, } from '../graphql'; -import colors from 'colors'; -import cliProgress from 'cli-progress'; import { - pullCronPageOfIdentifiers, CronIdentifier, + pullCronPageOfIdentifiers, } from './pullCronPageOfIdentifiers'; -import { RequestAction } from '@transcend-io/privacy-types'; - -import { logger } from '../../logger'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { mapSeries } from '../bluebird-replace'; /** * A CSV formatted identifier diff --git a/src/lib/cron/pullCronPageOfIdentifiers.ts b/src/lib/cron/pullCronPageOfIdentifiers.ts index b12aff31..ade1f39c 100644 --- a/src/lib/cron/pullCronPageOfIdentifiers.ts +++ b/src/lib/cron/pullCronPageOfIdentifiers.ts @@ -1,7 +1,7 @@ -import * as t from 'io-ts'; -import { decodeCodec } from '@transcend-io/type-utils'; import { RequestAction } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; import type { Got } from 'got'; +import * as t from 'io-ts'; export const CronIdentifier = t.type({ /** The identifier value */ diff --git a/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts b/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts index 10e7cbf4..a35a0755 100644 --- a/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts +++ b/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts @@ -1,18 +1,18 @@ +import { RequestAction } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchRequestDataSiloActiveCount, } from '../graphql'; -import colors from 'colors'; -import cliProgress from 'cli-progress'; import { - pullCronPageOfIdentifiers, CronIdentifier, + pullCronPageOfIdentifiers, } from './pullCronPageOfIdentifiers'; -import { RequestAction } from '@transcend-io/privacy-types'; -import { logger } from '../../logger'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { mapSeries } from '../bluebird-replace'; /** * A CSV formatted identifier diff --git a/src/lib/cron/pushCronIdentifiersFromCsv.ts b/src/lib/cron/pushCronIdentifiersFromCsv.ts index f00a1668..790e653f 100644 --- a/src/lib/cron/pushCronIdentifiersFromCsv.ts +++ b/src/lib/cron/pushCronIdentifiersFromCsv.ts @@ -1,15 +1,15 @@ -import { map, mapSeries } from '../bluebird-replace'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; import { chunk } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; import { createSombraGotInstance } from '../graphql'; -import colors from 'colors'; +import { readCsv } from '../requests'; import { - markCronIdentifierCompleted, CronIdentifierPush, + markCronIdentifierCompleted, } from './markCronIdentifierCompleted'; -import cliProgress from 'cli-progress'; -import { logger } from '../../logger'; -import { readCsv } from '../requests'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; /** * Given a CSV of cron job outputs, mark all requests as completed in Transcend diff --git a/src/lib/cron/writeCsv.ts b/src/lib/cron/writeCsv.ts index d3c81fe2..677c67f1 100644 --- a/src/lib/cron/writeCsv.ts +++ b/src/lib/cron/writeCsv.ts @@ -1,7 +1,6 @@ -import * as fastcsv from 'fast-csv'; -import { createWriteStream, writeFileSync, appendFileSync } from 'fs'; - +import { appendFileSync, createWriteStream, writeFileSync } from 'fs'; import { ObjByString } from '@transcend-io/type-utils'; +import * as fastcsv from 'fast-csv'; /** * Escape a CSV value diff --git a/src/lib/data-inventory/pullAllDatapoints.ts b/src/lib/data-inventory/pullAllDatapoints.ts index a1f3f184..b955a988 100644 --- a/src/lib/data-inventory/pullAllDatapoints.ts +++ b/src/lib/data-inventory/pullAllDatapoints.ts @@ -1,23 +1,23 @@ /* eslint-disable max-lines */ -import { keyBy, uniq, chunk, sortBy } from 'lodash-es'; import { - type DataCategoryType, SubDataPointDataSubCategoryGuessStatus, + type DataCategoryType, } from '@transcend-io/privacy-types'; import cliProgress from 'cli-progress'; -import { gql } from 'graphql-request'; import colors from 'colors'; +import { gql } from 'graphql-request'; import type { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy, sortBy, uniq } from 'lodash-es'; +import type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { - DATAPOINT_EXPORT, DATA_SILO_EXPORT, - type DataSiloAttributeValue, - SUB_DATA_POINTS_COUNT, + DATAPOINT_EXPORT, makeGraphQLRequest, + SUB_DATA_POINTS_COUNT, + type DataSiloAttributeValue, } from '../graphql'; -import { logger } from '../../logger'; -import type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs'; -import { mapSeries } from '../bluebird-replace'; export interface DataSiloCsvPreview { /** ID of dataSilo */ diff --git a/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts b/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts index 170cf73d..5c08cf25 100644 --- a/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts +++ b/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts @@ -4,8 +4,8 @@ import colors from 'colors'; import { gql, type GraphQLClient } from 'graphql-request'; import { sortBy } from 'lodash-es'; import type { DataCategoryInput } from '../../codecs'; -import { ENTRY_COUNT, makeGraphQLRequest } from '../graphql'; import { logger } from '../../logger'; +import { ENTRY_COUNT, makeGraphQLRequest } from '../graphql'; interface UnstructuredSubDataPointRecommendationCsvPreview { /** ID of subDatapoint */ diff --git a/src/lib/graphql/addMessagesToPromptRun.ts b/src/lib/graphql/addMessagesToPromptRun.ts index 81949f1b..bafa002d 100644 --- a/src/lib/graphql/addMessagesToPromptRun.ts +++ b/src/lib/graphql/addMessagesToPromptRun.ts @@ -1,7 +1,7 @@ +import { ChatCompletionRole, QueueStatus } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { ADD_MESSAGES_TO_PROMPT_RUN } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { QueueStatus, ChatCompletionRole } from '@transcend-io/privacy-types'; export interface AddMessagesToPromptRunInput { /** ID of run */ diff --git a/src/lib/graphql/createSombraGotInstance.ts b/src/lib/graphql/createSombraGotInstance.ts index aca717ce..9bd392f0 100644 --- a/src/lib/graphql/createSombraGotInstance.ts +++ b/src/lib/graphql/createSombraGotInstance.ts @@ -1,7 +1,7 @@ import got, { Got } from 'got'; +import { buildTranscendGraphQLClient } from './buildTranscendGraphQLClient'; import { ORGANIZATION } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { buildTranscendGraphQLClient } from './buildTranscendGraphQLClient'; /** * Instantiate an instance of got that is capable of making requests diff --git a/src/lib/graphql/deployConsentManager.ts b/src/lib/graphql/deployConsentManager.ts index 89b96840..008adc2d 100644 --- a/src/lib/graphql/deployConsentManager.ts +++ b/src/lib/graphql/deployConsentManager.ts @@ -1,8 +1,8 @@ import { ConsentBundleType } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { - UPDATE_CONSENT_MANAGER_TO_LATEST, DEPLOY_CONSENT_MANAGER, + UPDATE_CONSENT_MANAGER_TO_LATEST, } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; diff --git a/src/lib/graphql/fetchAllActionItemCollections.ts b/src/lib/graphql/fetchAllActionItemCollections.ts index 66e51783..362c0d33 100644 --- a/src/lib/graphql/fetchAllActionItemCollections.ts +++ b/src/lib/graphql/fetchAllActionItemCollections.ts @@ -1,7 +1,7 @@ +import { TranscendProduct } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { GLOBAL_ACTION_ITEM_COLLECTIONS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { TranscendProduct } from '@transcend-io/privacy-types'; export interface ActionItemCollection { /** ID of collection */ diff --git a/src/lib/graphql/fetchAllActionItems.ts b/src/lib/graphql/fetchAllActionItems.ts index 69a05771..cd6c409d 100644 --- a/src/lib/graphql/fetchAllActionItems.ts +++ b/src/lib/graphql/fetchAllActionItems.ts @@ -1,10 +1,10 @@ -import { GraphQLClient } from 'graphql-request'; -import { GLOBAL_ACTION_ITEMS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; import { ActionItemCode, ActionItemPriorityOverride, } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { GLOBAL_ACTION_ITEMS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface ActionItemRaw { /** ID of action item */ diff --git a/src/lib/graphql/fetchAllActions.ts b/src/lib/graphql/fetchAllActions.ts index 22520264..fb4fb13c 100644 --- a/src/lib/graphql/fetchAllActions.ts +++ b/src/lib/graphql/fetchAllActions.ts @@ -1,10 +1,10 @@ -import { GraphQLClient } from 'graphql-request'; import { IsoCountryCode, IsoCountrySubdivisionCode, RegionDetectionMethod, RequestAction, } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; import { ACTIONS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; diff --git a/src/lib/graphql/fetchAllAgentFiles.ts b/src/lib/graphql/fetchAllAgentFiles.ts index 3ed47256..27813b64 100644 --- a/src/lib/graphql/fetchAllAgentFiles.ts +++ b/src/lib/graphql/fetchAllAgentFiles.ts @@ -1,7 +1,7 @@ +import { PromptFilePurpose } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { AGENT_FILES } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { PromptFilePurpose } from '@transcend-io/privacy-types'; export interface AgentFile { /** ID of agentFile */ diff --git a/src/lib/graphql/fetchAllAgentFunctions.ts b/src/lib/graphql/fetchAllAgentFunctions.ts index fedecc62..2f0ba406 100644 --- a/src/lib/graphql/fetchAllAgentFunctions.ts +++ b/src/lib/graphql/fetchAllAgentFunctions.ts @@ -1,7 +1,7 @@ import { GraphQLClient } from 'graphql-request'; +import type { JSONSchema7 } from 'json-schema'; import { AGENT_FUNCTIONS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import type { JSONSchema7 } from 'json-schema'; export interface AgentFunction { /** ID of agentFunction */ diff --git a/src/lib/graphql/fetchAllAgents.ts b/src/lib/graphql/fetchAllAgents.ts index 939cac34..8f59af03 100644 --- a/src/lib/graphql/fetchAllAgents.ts +++ b/src/lib/graphql/fetchAllAgents.ts @@ -1,7 +1,7 @@ +import { LargeLanguageModelClient } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { AGENTS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { LargeLanguageModelClient } from '@transcend-io/privacy-types'; export interface Agent { /** ID of agent */ diff --git a/src/lib/graphql/fetchAllAssessmentTemplates.ts b/src/lib/graphql/fetchAllAssessmentTemplates.ts index 28d8c3c7..3af4853e 100644 --- a/src/lib/graphql/fetchAllAssessmentTemplates.ts +++ b/src/lib/graphql/fetchAllAssessmentTemplates.ts @@ -1,15 +1,15 @@ +import { + AssessmentFormTemplateSource, + AssessmentFormTemplateStatus, +} from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; -import { ASSESSMENT_TEMPLATES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; import type { AssessmentSection, RetentionSchedule, UserPreview, } from './fetchAllAssessments'; -import { - AssessmentFormTemplateSource, - AssessmentFormTemplateStatus, -} from '@transcend-io/privacy-types'; +import { ASSESSMENT_TEMPLATES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Represents an assessment template with various properties and metadata. diff --git a/src/lib/graphql/fetchAllAssessments.ts b/src/lib/graphql/fetchAllAssessments.ts index 910df076..f5b9af12 100644 --- a/src/lib/graphql/fetchAllAssessments.ts +++ b/src/lib/graphql/fetchAllAssessments.ts @@ -1,7 +1,4 @@ /* eslint-disable max-lines */ -import { GraphQLClient } from 'graphql-request'; -import { ASSESSMENTS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; import { AssessmentFormStatus, AssessmentQuestionSubType, @@ -14,6 +11,9 @@ import { RetentionScheduleOperation, RetentionScheduleType, } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { ASSESSMENTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Represents an assessment with various properties and metadata. diff --git a/src/lib/graphql/fetchAllAttributes.ts b/src/lib/graphql/fetchAllAttributes.ts index dd1f4ef8..9d8c49f6 100644 --- a/src/lib/graphql/fetchAllAttributes.ts +++ b/src/lib/graphql/fetchAllAttributes.ts @@ -1,12 +1,11 @@ -import { GraphQLClient } from 'graphql-request'; -import { ATTRIBUTES, ATTRIBUTE_VALUES } from './gqls'; - -import { logger } from '../../logger'; -import colors from 'colors'; import { AttributeKeyType, AttributeSupportedResourceType, } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { logger } from '../../logger'; +import { ATTRIBUTE_VALUES, ATTRIBUTES } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface AttributeValue { diff --git a/src/lib/graphql/fetchAllBusinessEntities.ts b/src/lib/graphql/fetchAllBusinessEntities.ts index 8605212b..1a87f99a 100644 --- a/src/lib/graphql/fetchAllBusinessEntities.ts +++ b/src/lib/graphql/fetchAllBusinessEntities.ts @@ -1,10 +1,10 @@ -import { GraphQLClient } from 'graphql-request'; -import { BUSINESS_ENTITIES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; import { IsoCountryCode, IsoCountrySubdivisionCode, } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { BUSINESS_ENTITIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface BusinessEntity { /** ID of business entity */ diff --git a/src/lib/graphql/fetchAllCodePackages.ts b/src/lib/graphql/fetchAllCodePackages.ts index 49fd3d3e..5360c50a 100644 --- a/src/lib/graphql/fetchAllCodePackages.ts +++ b/src/lib/graphql/fetchAllCodePackages.ts @@ -1,7 +1,7 @@ +import { CodePackageType } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { CODE_PACKAGES } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { CodePackageType } from '@transcend-io/privacy-types'; export interface CodePackage { /** ID of code package */ diff --git a/src/lib/graphql/fetchAllCookies.ts b/src/lib/graphql/fetchAllCookies.ts index f5b9d0ed..ced750ec 100644 --- a/src/lib/graphql/fetchAllCookies.ts +++ b/src/lib/graphql/fetchAllCookies.ts @@ -1,11 +1,11 @@ -import { GraphQLClient } from 'graphql-request'; -import { COOKIES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { fetchConsentManagerId } from './fetchConsentManagerId'; import { ConsentTrackerSource, ConsentTrackerStatus, } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { fetchConsentManagerId } from './fetchConsentManagerId'; +import { COOKIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Cookie { /** ID of the cookie */ diff --git a/src/lib/graphql/fetchAllDataCategories.ts b/src/lib/graphql/fetchAllDataCategories.ts index 027a6cdb..f01c386d 100644 --- a/src/lib/graphql/fetchAllDataCategories.ts +++ b/src/lib/graphql/fetchAllDataCategories.ts @@ -1,7 +1,7 @@ +import { DataCategoryType } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { DATA_SUB_CATEGORIES } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { DataCategoryType } from '@transcend-io/privacy-types'; export interface DataSubCategory { /** ID of data category */ diff --git a/src/lib/graphql/fetchAllDataFlows.ts b/src/lib/graphql/fetchAllDataFlows.ts index f78176b1..04dfd505 100644 --- a/src/lib/graphql/fetchAllDataFlows.ts +++ b/src/lib/graphql/fetchAllDataFlows.ts @@ -1,12 +1,12 @@ -import { GraphQLClient } from 'graphql-request'; -import { DATA_FLOWS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { fetchConsentManagerId } from './fetchConsentManagerId'; import { - DataFlowScope, ConsentTrackerSource, ConsentTrackerStatus, + DataFlowScope, } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { fetchConsentManagerId } from './fetchConsentManagerId'; +import { DATA_FLOWS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface DataFlow { /** ID of data flow */ diff --git a/src/lib/graphql/fetchAllMessages.ts b/src/lib/graphql/fetchAllMessages.ts index 2451d55f..da94a991 100644 --- a/src/lib/graphql/fetchAllMessages.ts +++ b/src/lib/graphql/fetchAllMessages.ts @@ -1,7 +1,7 @@ +import { LanguageKey } from '@transcend-io/internationalization'; import { GraphQLClient } from 'graphql-request'; import { MESSAGES } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { LanguageKey } from '@transcend-io/internationalization'; export interface Message { /** ID of message */ diff --git a/src/lib/graphql/fetchAllPolicies.ts b/src/lib/graphql/fetchAllPolicies.ts index 831d4fa5..88095351 100644 --- a/src/lib/graphql/fetchAllPolicies.ts +++ b/src/lib/graphql/fetchAllPolicies.ts @@ -1,8 +1,8 @@ +import { LanguageKey } from '@transcend-io/internationalization'; import { GraphQLClient } from 'graphql-request'; +import { fetchPrivacyCenterUrl } from './fetchPrivacyCenterId'; import { POLICIES } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { LanguageKey } from '@transcend-io/internationalization'; -import { fetchPrivacyCenterUrl } from './fetchPrivacyCenterId'; export interface Policy { /** ID of policy */ diff --git a/src/lib/graphql/fetchAllPreferenceTopics.ts b/src/lib/graphql/fetchAllPreferenceTopics.ts index 25ab1380..710a81eb 100644 --- a/src/lib/graphql/fetchAllPreferenceTopics.ts +++ b/src/lib/graphql/fetchAllPreferenceTopics.ts @@ -1,7 +1,7 @@ +import { PreferenceTopicType } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { PREFERENCE_TOPICS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { PreferenceTopicType } from '@transcend-io/privacy-types'; export interface PreferenceTopic { /** ID of preference topic */ diff --git a/src/lib/graphql/fetchAllPrivacyCenters.ts b/src/lib/graphql/fetchAllPrivacyCenters.ts index a1837d70..e36d7696 100644 --- a/src/lib/graphql/fetchAllPrivacyCenters.ts +++ b/src/lib/graphql/fetchAllPrivacyCenters.ts @@ -1,9 +1,9 @@ -import { GraphQLClient } from 'graphql-request'; -import { PRIVACY_CENTER } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; import { LanguageKey } from '@transcend-io/internationalization'; import { PrivacyCenterThemePartial } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; import { fetchPrivacyCenterUrl } from './fetchPrivacyCenterId'; +import { PRIVACY_CENTER } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface PrivacyCenter { /** ID of the privacy center */ diff --git a/src/lib/graphql/fetchAllProcessingPurposes.ts b/src/lib/graphql/fetchAllProcessingPurposes.ts index e6387ecc..4f1c8cd4 100644 --- a/src/lib/graphql/fetchAllProcessingPurposes.ts +++ b/src/lib/graphql/fetchAllProcessingPurposes.ts @@ -1,7 +1,7 @@ +import { ProcessingPurpose } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { PROCESSING_PURPOSE_SUB_CATEGORIES } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { ProcessingPurpose } from '@transcend-io/privacy-types'; export interface ProcessingPurposeSubCategory { /** ID of processing purpose */ diff --git a/src/lib/graphql/fetchAllPurposes.ts b/src/lib/graphql/fetchAllPurposes.ts index 43e8b13d..de3e548b 100644 --- a/src/lib/graphql/fetchAllPurposes.ts +++ b/src/lib/graphql/fetchAllPurposes.ts @@ -1,11 +1,11 @@ -import { GraphQLClient } from 'graphql-request'; -import { PURPOSES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { UserPrivacySignalEnum } from '@transcend-io/airgap.js-types'; import { DefaultConsentOption, PreferenceStoreAuthLevel, } from '@transcend-io/privacy-types'; -import { UserPrivacySignalEnum } from '@transcend-io/airgap.js-types'; +import { GraphQLClient } from 'graphql-request'; +import { PURPOSES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Purpose { /** ID of purpose */ diff --git a/src/lib/graphql/fetchAllPurposesAndPreferences.ts b/src/lib/graphql/fetchAllPurposesAndPreferences.ts index 757a8d8b..a0e9ad39 100644 --- a/src/lib/graphql/fetchAllPurposesAndPreferences.ts +++ b/src/lib/graphql/fetchAllPurposesAndPreferences.ts @@ -1,9 +1,9 @@ import { GraphQLClient } from 'graphql-request'; -import { Purpose, fetchAllPurposes } from './fetchAllPurposes'; import { - PreferenceTopic, fetchAllPreferenceTopics, + PreferenceTopic, } from './fetchAllPreferenceTopics'; +import { fetchAllPurposes, Purpose } from './fetchAllPurposes'; export interface PurposeWithPreferences extends Purpose { /** Topics */ diff --git a/src/lib/graphql/fetchAllRequestEnrichers.ts b/src/lib/graphql/fetchAllRequestEnrichers.ts index 7b64d124..98970f78 100644 --- a/src/lib/graphql/fetchAllRequestEnrichers.ts +++ b/src/lib/graphql/fetchAllRequestEnrichers.ts @@ -1,5 +1,5 @@ -import { GraphQLClient } from 'graphql-request'; import { RequestEnricherStatus } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; import { REQUEST_ENRICHERS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; diff --git a/src/lib/graphql/fetchAllRequestIdentifiers.ts b/src/lib/graphql/fetchAllRequestIdentifiers.ts index 6dab2d67..a1b07547 100644 --- a/src/lib/graphql/fetchAllRequestIdentifiers.ts +++ b/src/lib/graphql/fetchAllRequestIdentifiers.ts @@ -4,7 +4,6 @@ import type { Got } from 'got'; import { GraphQLClient } from 'graphql-request'; import * as t from 'io-ts'; import semver from 'semver'; - import { SOMBRA_VERSION } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; diff --git a/src/lib/graphql/fetchAllRequests.ts b/src/lib/graphql/fetchAllRequests.ts index 81b8615b..4a5284e0 100644 --- a/src/lib/graphql/fetchAllRequests.ts +++ b/src/lib/graphql/fetchAllRequests.ts @@ -1,19 +1,19 @@ -import { GraphQLClient } from 'graphql-request'; -import colors from 'colors'; -import { REQUESTS } from './gqls'; -import * as t from 'io-ts'; -import cliProgress from 'cli-progress'; -import { valuesOf } from '@transcend-io/type-utils'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { LanguageKey } from '@transcend-io/internationalization'; import { + IsoCountryCode, + IsoCountrySubdivisionCode, RequestAction, RequestOrigin, RequestStatus, - IsoCountryCode, - IsoCountrySubdivisionCode, } from '@transcend-io/privacy-types'; +import { valuesOf } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import * as t from 'io-ts'; import { logger } from '../../logger'; -import { LanguageKey } from '@transcend-io/internationalization'; +import { REQUESTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export const PrivacyRequest = t.intersection([ t.type({ diff --git a/src/lib/graphql/fetchAllSoftwareDevelopmentKits.ts b/src/lib/graphql/fetchAllSoftwareDevelopmentKits.ts index 4dbbf32b..eb73dc90 100644 --- a/src/lib/graphql/fetchAllSoftwareDevelopmentKits.ts +++ b/src/lib/graphql/fetchAllSoftwareDevelopmentKits.ts @@ -1,7 +1,7 @@ +import { CodePackageType } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { SOFTWARE_DEVELOPMENT_KITS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { CodePackageType } from '@transcend-io/privacy-types'; export interface SoftwareDevelopmentKit { /** ID of software development kit */ diff --git a/src/lib/graphql/fetchAllTeams.ts b/src/lib/graphql/fetchAllTeams.ts index 527c94d6..2acaafcb 100644 --- a/src/lib/graphql/fetchAllTeams.ts +++ b/src/lib/graphql/fetchAllTeams.ts @@ -1,7 +1,7 @@ +import { ScopeName } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { TEAMS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { ScopeName } from '@transcend-io/privacy-types'; export interface Team { /** ID of team */ diff --git a/src/lib/graphql/fetchAllVendors.ts b/src/lib/graphql/fetchAllVendors.ts index 798daed4..08fe77b7 100644 --- a/src/lib/graphql/fetchAllVendors.ts +++ b/src/lib/graphql/fetchAllVendors.ts @@ -1,10 +1,10 @@ -import { GraphQLClient } from 'graphql-request'; -import { VENDORS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; import { IsoCountryCode, IsoCountrySubdivisionCode, } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { VENDORS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Vendor { /** ID of vendor */ diff --git a/src/lib/graphql/fetchApiKeys.ts b/src/lib/graphql/fetchApiKeys.ts index fb8ffefa..d11aeedd 100644 --- a/src/lib/graphql/fetchApiKeys.ts +++ b/src/lib/graphql/fetchApiKeys.ts @@ -1,9 +1,9 @@ -import { GraphQLClient } from 'graphql-request'; -import { API_KEYS } from './gqls'; -import { keyBy, uniq, difference } from 'lodash-es'; -import { logger } from '../../logger'; import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference, keyBy, uniq } from 'lodash-es'; import { TranscendInput } from '../../codecs'; +import { logger } from '../../logger'; +import { API_KEYS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface ApiKey { diff --git a/src/lib/graphql/fetchConsentManagerId.ts b/src/lib/graphql/fetchConsentManagerId.ts index f6ea9934..e8a7d4ca 100644 --- a/src/lib/graphql/fetchConsentManagerId.ts +++ b/src/lib/graphql/fetchConsentManagerId.ts @@ -1,25 +1,25 @@ -import { GraphQLClient } from 'graphql-request'; import { + BrowserLanguage, + InitialViewState, + OnConsentExpiry, +} from '@transcend-io/airgap.js-types'; +import { + BrowserTimeZone, ConsentPrecedenceOption, - UnknownRequestPolicy, - UspapiOption, - TelemetryPartitionStrategy, - RegionsOperator, - IsoCountrySubdivisionCode, IsoCountryCode, - BrowserTimeZone, + IsoCountrySubdivisionCode, + RegionsOperator, SignedIabAgreementOption, + TelemetryPartitionStrategy, + UnknownRequestPolicy, + UspapiOption, } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; import { - InitialViewState, - BrowserLanguage, - OnConsentExpiry, -} from '@transcend-io/airgap.js-types'; -import { - FETCH_CONSENT_MANAGER_ID, - FETCH_CONSENT_MANAGER, - EXPERIENCES, CONSENT_MANAGER_ANALYTICS_DATA, + EXPERIENCES, + FETCH_CONSENT_MANAGER, + FETCH_CONSENT_MANAGER_ID, FETCH_CONSENT_MANAGER_THEME, } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; diff --git a/src/lib/graphql/fetchDataSubjects.ts b/src/lib/graphql/fetchDataSubjects.ts index 17fcfb54..b6a40b66 100644 --- a/src/lib/graphql/fetchDataSubjects.ts +++ b/src/lib/graphql/fetchDataSubjects.ts @@ -1,11 +1,11 @@ -import { GraphQLClient } from 'graphql-request'; -import { CREATE_DATA_SUBJECT, DATA_SUBJECTS } from './gqls'; -import { keyBy, flatten, uniq, difference } from 'lodash-es'; import { RequestActionObjectResolver } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference, flatten, keyBy, uniq } from 'lodash-es'; import { TranscendInput } from '../../codecs'; import { logger } from '../../logger'; -import colors from 'colors'; import { mapSeries } from '../bluebird-replace'; +import { CREATE_DATA_SUBJECT, DATA_SUBJECTS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface DataSubject { diff --git a/src/lib/graphql/fetchIdentifiers.ts b/src/lib/graphql/fetchIdentifiers.ts index 8a778fc6..4333bf74 100644 --- a/src/lib/graphql/fetchIdentifiers.ts +++ b/src/lib/graphql/fetchIdentifiers.ts @@ -1,11 +1,11 @@ -import { GraphQLClient } from 'graphql-request'; import { IdentifierType, RequestAction } from '@transcend-io/privacy-types'; -import { CREATE_IDENTIFIER, IDENTIFIERS, NEW_IDENTIFIER_TYPES } from './gqls'; -import { keyBy, uniq, flatten, difference } from 'lodash-es'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference, flatten, keyBy, uniq } from 'lodash-es'; import { TranscendInput } from '../../codecs'; import { logger } from '../../logger'; -import colors from 'colors'; import { mapSeries } from '../bluebird-replace'; +import { CREATE_IDENTIFIER, IDENTIFIERS, NEW_IDENTIFIER_TYPES } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Identifier { diff --git a/src/lib/graphql/fetchLargeLanguageModels.ts b/src/lib/graphql/fetchLargeLanguageModels.ts index 98beb69e..4b097340 100644 --- a/src/lib/graphql/fetchLargeLanguageModels.ts +++ b/src/lib/graphql/fetchLargeLanguageModels.ts @@ -1,5 +1,5 @@ -import { GraphQLClient } from 'graphql-request'; import { LargeLanguageModelClient } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; import { LARGE_LANGUAGE_MODELS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; diff --git a/src/lib/graphql/fetchPrompts.ts b/src/lib/graphql/fetchPrompts.ts index b673ea84..ceee7254 100644 --- a/src/lib/graphql/fetchPrompts.ts +++ b/src/lib/graphql/fetchPrompts.ts @@ -1,10 +1,10 @@ -import { GraphQLClient } from 'graphql-request'; -import { PROMPTS, PROMPTS_WITH_VARIABLES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; import { - PromptStatus, PromptResponseFormat, + PromptStatus, } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { PROMPTS, PROMPTS_WITH_VARIABLES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Prompt { /** ID of prompt */ diff --git a/src/lib/graphql/fetchRequestDataSilo.ts b/src/lib/graphql/fetchRequestDataSilo.ts index 670d5423..a01a87e7 100644 --- a/src/lib/graphql/fetchRequestDataSilo.ts +++ b/src/lib/graphql/fetchRequestDataSilo.ts @@ -1,13 +1,13 @@ -import { GraphQLClient } from 'graphql-request'; -import colors from 'colors'; -import cliProgress from 'cli-progress'; -import { REQUEST_DATA_SILOS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; import { RequestDataSiloStatus, RequestStatus, } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; import { logger } from '../../logger'; +import { REQUEST_DATA_SILOS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface RequestDataSilo { /** ID of RequestDataSilo */ diff --git a/src/lib/graphql/gqls/actionItemCollection.ts b/src/lib/graphql/gqls/actionItemCollection.ts index 3fa6a794..581f4c8d 100644 --- a/src/lib/graphql/gqls/actionItemCollection.ts +++ b/src/lib/graphql/gqls/actionItemCollection.ts @@ -1,4 +1,5 @@ import { gql } from 'graphql-request'; + // TODO: https://transcend.height.app/T-27909 - enable optimizations // isExportCsv: true // useMaster: false diff --git a/src/lib/graphql/loginUser.ts b/src/lib/graphql/loginUser.ts index 59851ad8..4603dba6 100644 --- a/src/lib/graphql/loginUser.ts +++ b/src/lib/graphql/loginUser.ts @@ -1,5 +1,5 @@ import { GraphQLClient } from 'graphql-request'; -import { DETERMINE_LOGIN_METHOD, ASSUME_ROLE, LOGIN } from './gqls'; +import { ASSUME_ROLE, DETERMINE_LOGIN_METHOD, LOGIN } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface OrganizationPreview { diff --git a/src/lib/graphql/makeGraphQLRequest.ts b/src/lib/graphql/makeGraphQLRequest.ts index b29bbe44..cd89f82c 100644 --- a/src/lib/graphql/makeGraphQLRequest.ts +++ b/src/lib/graphql/makeGraphQLRequest.ts @@ -1,10 +1,10 @@ +import colors from 'colors'; import type { GraphQLClient, RequestDocument, Variables, } from 'graphql-request'; import { logger } from '../../logger'; -import colors from 'colors'; const MAX_RETRIES = 4; diff --git a/src/lib/graphql/manageApiKeys.ts b/src/lib/graphql/manageApiKeys.ts index 8a5b7a75..19233b3b 100644 --- a/src/lib/graphql/manageApiKeys.ts +++ b/src/lib/graphql/manageApiKeys.ts @@ -1,7 +1,7 @@ +import { ScopeName } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { CREATE_API_KEY, DELETE_API_KEY } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { ScopeName } from '@transcend-io/privacy-types'; export interface CreatedApiKey { /** ID of API key */ diff --git a/src/lib/graphql/pullTranscendConfiguration.ts b/src/lib/graphql/pullTranscendConfiguration.ts index bf22ebc5..67a5041b 100644 --- a/src/lib/graphql/pullTranscendConfiguration.ts +++ b/src/lib/graphql/pullTranscendConfiguration.ts @@ -1,94 +1,94 @@ /* eslint-disable max-lines */ +import { LanguageKey } from '@transcend-io/internationalization'; +import { + ActionItemCode, + ConsentTrackerStatus, + RequestAction, +} from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { flatten, keyBy, mapValues } from 'lodash-es'; import { - TranscendInput, - ApiKeyInput, - DataSiloInput, - AttributeInput, ActionInput, - IdentifierInput, - BusinessEntityInput, - EnricherInput, - PromptGroupInput, - DataFlowInput, - PromptPartialInput, - DataSubjectInput, - CookieInput, - PromptInput, - DatapointInput, - FieldInput, - ProcessingPurposeInput, - DataCategoryInput, - VendorInput, + ActionItemCollectionInput, + ActionItemInput, AgentFileInput, AgentFunctionInput, AgentInput, - PolicyInput, - IntlMessageInput, - ActionItemInput, - TeamInput, - ActionItemCollectionInput, + ApiKeyInput, AssessmentInput, - AssessmentTemplateInput, AssessmentSectionInput, AssessmentSectionQuestionInput, - RiskLogicInput, + AssessmentTemplateInput, + AttributeInput, + BusinessEntityInput, ConsentPurpose, + CookieInput, + DataCategoryInput, + DataFlowInput, + DatapointInput, + DataSiloInput, + DataSubjectInput, + EnricherInput, + FieldInput, + IdentifierInput, + IntlMessageInput, + PolicyInput, + ProcessingPurposeInput, + PromptGroupInput, + PromptInput, + PromptPartialInput, + RiskLogicInput, + TeamInput, + TranscendInput, + VendorInput, } from '../../codecs'; -import { - RequestAction, - ConsentTrackerStatus, - ActionItemCode, -} from '@transcend-io/privacy-types'; -import { GraphQLClient } from 'graphql-request'; -import { flatten, keyBy, mapValues } from 'lodash-es'; -import { fetchEnrichedDataSilos } from './syncDataSilos'; -import { - convertToDataSubjectAllowlist, - fetchAllDataSubjects, -} from './fetchDataSubjects'; +import { TranscendPullResource } from '../../enums'; +import { logger } from '../../logger'; +import { fetchAllActionItemCollections } from './fetchAllActionItemCollections'; +import { fetchAllActionItems } from './fetchAllActionItems'; +import { fetchAllActions } from './fetchAllActions'; +import { fetchAllAgentFiles } from './fetchAllAgentFiles'; +import { fetchAllAgentFunctions } from './fetchAllAgentFunctions'; +import { fetchAllAgents } from './fetchAllAgents'; +import { fetchAllAssessments } from './fetchAllAssessments'; +import { fetchAllAssessmentTemplates } from './fetchAllAssessmentTemplates'; +import { fetchAllAttributes } from './fetchAllAttributes'; +import { fetchAllBusinessEntities } from './fetchAllBusinessEntities'; +import { fetchAllCookies } from './fetchAllCookies'; +import { fetchAllDataCategories } from './fetchAllDataCategories'; +import { fetchAllDataFlows } from './fetchAllDataFlows'; +import { fetchAllMessages } from './fetchAllMessages'; +import { fetchAllPolicies } from './fetchAllPolicies'; +import { fetchAllPrivacyCenters } from './fetchAllPrivacyCenters'; +import { fetchAllProcessingPurposes } from './fetchAllProcessingPurposes'; +import { fetchAllPurposesAndPreferences } from './fetchAllPurposesAndPreferences'; +import { fetchAllTeams } from './fetchAllTeams'; +import { fetchAllVendors } from './fetchAllVendors'; import { fetchApiKeys } from './fetchApiKeys'; import { fetchConsentManager, fetchConsentManagerExperiences, fetchConsentManagerTheme, } from './fetchConsentManagerId'; -import { fetchAllEnrichers } from './syncEnrichers'; -import { fetchAllDataFlows } from './fetchAllDataFlows'; -import { fetchAllBusinessEntities } from './fetchAllBusinessEntities'; -import { fetchAllActions } from './fetchAllActions'; -import { fetchAllAgents } from './fetchAllAgents'; -import { fetchAllAgentFunctions } from './fetchAllAgentFunctions'; -import { fetchAllAgentFiles } from './fetchAllAgentFiles'; -import { fetchAllVendors } from './fetchAllVendors'; -import { fetchAllDataCategories } from './fetchAllDataCategories'; -import { fetchAllProcessingPurposes } from './fetchAllProcessingPurposes'; +import { + convertToDataSubjectAllowlist, + fetchAllDataSubjects, +} from './fetchDataSubjects'; import { fetchAllIdentifiers } from './fetchIdentifiers'; -import { fetchAllPrompts } from './fetchPrompts'; -import { fetchAllPromptPartials } from './fetchPromptPartials'; -import { fetchAllPolicies } from './fetchAllPolicies'; -import { fetchAllPrivacyCenters } from './fetchAllPrivacyCenters'; -import { fetchAllMessages } from './fetchAllMessages'; import { fetchAllPromptGroups } from './fetchPromptGroups'; -import { fetchAllCookies } from './fetchAllCookies'; -import { fetchAllTemplates } from './syncTemplates'; -import { fetchAllAttributes } from './fetchAllAttributes'; +import { fetchAllPromptPartials } from './fetchPromptPartials'; +import { fetchAllPrompts } from './fetchPrompts'; import { formatAttributeValues } from './formatAttributeValues'; -import { logger } from '../../logger'; -import colors from 'colors'; -import { TranscendPullResource } from '../../enums'; -import { fetchAllActionItems } from './fetchAllActionItems'; -import { fetchAllTeams } from './fetchAllTeams'; -import { fetchAllActionItemCollections } from './fetchAllActionItemCollections'; -import { LanguageKey } from '@transcend-io/internationalization'; -import { fetchPartitions } from './syncPartitions'; -import { fetchAllAssessments } from './fetchAllAssessments'; -import { fetchAllAssessmentTemplates } from './fetchAllAssessmentTemplates'; import { AssessmentNestedRule, parseAssessmentDisplayLogic, } from './parseAssessmentDisplayLogic'; import { parseAssessmentRiskLogic } from './parseAssessmentRiskLogic'; -import { fetchAllPurposesAndPreferences } from './fetchAllPurposesAndPreferences'; +import { fetchEnrichedDataSilos } from './syncDataSilos'; +import { fetchAllEnrichers } from './syncEnrichers'; +import { fetchPartitions } from './syncPartitions'; +import { fetchAllTemplates } from './syncTemplates'; export const DEFAULT_TRANSCEND_PULL_RESOURCES = [ TranscendPullResource.DataSilos, @@ -611,8 +611,8 @@ export async function pullTranscendConfiguration( title: category ? `${category} - ${name}` : purpose - ? `${purpose} - ${name}` - : title || name || type || '', + ? `${purpose} - ${name}` + : title || name || type || '', }), ), rows: syncedRows.map( @@ -621,8 +621,8 @@ export async function pullTranscendConfiguration( title: category ? `${category} - ${name}` : purpose - ? `${purpose} - ${name}` - : title || name || type || '', + ? `${purpose} - ${name}` + : title || name || type || '', }), ), }), diff --git a/src/lib/graphql/reportPromptRun.ts b/src/lib/graphql/reportPromptRun.ts index 5dead494..e5c89e40 100644 --- a/src/lib/graphql/reportPromptRun.ts +++ b/src/lib/graphql/reportPromptRun.ts @@ -1,12 +1,12 @@ -import { GraphQLClient } from 'graphql-request'; -import { REPORT_PROMPT_RUN } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; import { - QueueStatus, ChatCompletionRole, - PromptRunProductArea, LargeLanguageModelClient, + PromptRunProductArea, + QueueStatus, } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { REPORT_PROMPT_RUN } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface ReportPromptRunInput { /** Name of run */ diff --git a/src/lib/graphql/setResourceAttributes.ts b/src/lib/graphql/setResourceAttributes.ts index 3d82bcb2..b2fb1ca4 100644 --- a/src/lib/graphql/setResourceAttributes.ts +++ b/src/lib/graphql/setResourceAttributes.ts @@ -1,7 +1,7 @@ +import { AttributeSupportedResourceType } from '@transcend-io/privacy-types'; import { GraphQLClient } from 'graphql-request'; import { SET_RESOURCE_ATTRIBUTES } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { AttributeSupportedResourceType } from '@transcend-io/privacy-types'; interface SetResourceAttributesInput { /** ID of resource */ diff --git a/src/lib/graphql/syncAction.ts b/src/lib/graphql/syncAction.ts index 294f135a..1788cbec 100644 --- a/src/lib/graphql/syncAction.ts +++ b/src/lib/graphql/syncAction.ts @@ -1,12 +1,12 @@ -import { ActionInput } from '../../codecs'; -import { GraphQLClient } from 'graphql-request'; -import { UPDATE_ACTION } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { difference } from 'lodash-es'; import { IsoCountryCode, IsoCountrySubdivisionCode, } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { difference } from 'lodash-es'; +import { ActionInput } from '../../codecs'; +import { UPDATE_ACTION } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const ALL_COUNTRIES_AND_SUBDIVISIONS = [ ...Object.values(IsoCountryCode), diff --git a/src/lib/graphql/syncActionItemCollections.ts b/src/lib/graphql/syncActionItemCollections.ts index 404f780b..4c3f257b 100644 --- a/src/lib/graphql/syncActionItemCollections.ts +++ b/src/lib/graphql/syncActionItemCollections.ts @@ -1,18 +1,18 @@ -import { ActionItemCollectionInput } from '../../codecs'; +import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { mapSeries } from '../bluebird-replace'; -import { - UPDATE_ACTION_ITEM_COLLECTION, - CREATE_ACTION_ITEM_COLLECTION, -} from './gqls'; -import { logger } from '../../logger'; import { keyBy } from 'lodash-es'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import colors from 'colors'; +import { ActionItemCollectionInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { ActionItemCollection, fetchAllActionItemCollections, } from './fetchAllActionItemCollections'; +import { + CREATE_ACTION_ITEM_COLLECTION, + UPDATE_ACTION_ITEM_COLLECTION, +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new action item collection @@ -85,9 +85,8 @@ export async function syncActionItemCollections( ); // Fetch existing - const existingActionItemCollections = await fetchAllActionItemCollections( - client, - ); + const existingActionItemCollections = + await fetchAllActionItemCollections(client); // Look up by title const collectionByTitle: { [k in string]: ActionItemCollection } = keyBy( diff --git a/src/lib/graphql/syncActionItems.ts b/src/lib/graphql/syncActionItems.ts index 095f8118..4c4060c4 100644 --- a/src/lib/graphql/syncActionItems.ts +++ b/src/lib/graphql/syncActionItems.ts @@ -1,17 +1,17 @@ -import { ActionItemInput } from '../../codecs'; -import { uniq, keyBy, chunk } from 'lodash-es'; +import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { mapSeries } from '../bluebird-replace'; -import { UPDATE_ACTION_ITEMS, CREATE_ACTION_ITEMS } from './gqls'; +import { chunk, keyBy, uniq } from 'lodash-es'; +import { ActionItemInput } from '../../codecs'; import { logger } from '../../logger'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import colors from 'colors'; -import { fetchAllActionItems, ActionItem } from './fetchAllActionItems'; +import { mapSeries } from '../bluebird-replace'; import { ActionItemCollection, fetchAllActionItemCollections, } from './fetchAllActionItemCollections'; +import { ActionItem, fetchAllActionItems } from './fetchAllActionItems'; import { Attribute, fetchAllAttributes } from './fetchAllAttributes'; +import { CREATE_ACTION_ITEMS, UPDATE_ACTION_ITEMS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new actionItem diff --git a/src/lib/graphql/syncAgentFiles.ts b/src/lib/graphql/syncAgentFiles.ts index 3207aa8e..9ed5a59d 100644 --- a/src/lib/graphql/syncAgentFiles.ts +++ b/src/lib/graphql/syncAgentFiles.ts @@ -1,12 +1,12 @@ -import { AgentFileInput } from '../../codecs'; +import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { mapSeries } from '../bluebird-replace'; -import { UPDATE_AGENT_FILES, CREATE_AGENT_FILE } from './gqls'; -import { logger } from '../../logger'; import { keyBy } from 'lodash-es'; +import { AgentFileInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { AgentFile, fetchAllAgentFiles } from './fetchAllAgentFiles'; +import { CREATE_AGENT_FILE, UPDATE_AGENT_FILES } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import colors from 'colors'; -import { fetchAllAgentFiles, AgentFile } from './fetchAllAgentFiles'; /** * Input to create a new agent file diff --git a/src/lib/graphql/syncAgentFunctions.ts b/src/lib/graphql/syncAgentFunctions.ts index 2faa7b66..2fea50fd 100644 --- a/src/lib/graphql/syncAgentFunctions.ts +++ b/src/lib/graphql/syncAgentFunctions.ts @@ -1,15 +1,15 @@ -import { AgentFunctionInput } from '../../codecs'; +import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { mapSeries } from '../bluebird-replace'; -import { UPDATE_AGENT_FUNCTIONS, CREATE_AGENT_FUNCTION } from './gqls'; -import { logger } from '../../logger'; import { keyBy } from 'lodash-es'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import colors from 'colors'; +import { AgentFunctionInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { - fetchAllAgentFunctions, AgentFunction, + fetchAllAgentFunctions, } from './fetchAllAgentFunctions'; +import { CREATE_AGENT_FUNCTION, UPDATE_AGENT_FUNCTIONS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new agent function diff --git a/src/lib/graphql/syncAgents.ts b/src/lib/graphql/syncAgents.ts index 56037b80..31e731b9 100644 --- a/src/lib/graphql/syncAgents.ts +++ b/src/lib/graphql/syncAgents.ts @@ -1,12 +1,12 @@ -import { AgentInput } from '../../codecs'; +import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { mapSeries } from '../bluebird-replace'; -import { UPDATE_AGENTS, CREATE_AGENT } from './gqls'; -import { logger } from '../../logger'; import { keyBy } from 'lodash-es'; +import { AgentInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { Agent, fetchAllAgents } from './fetchAllAgents'; +import { CREATE_AGENT, UPDATE_AGENTS } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import colors from 'colors'; -import { fetchAllAgents, Agent } from './fetchAllAgents'; /** * Input to create a new agent diff --git a/src/lib/graphql/syncAttribute.ts b/src/lib/graphql/syncAttribute.ts index e0c093f8..6ccfa73f 100644 --- a/src/lib/graphql/syncAttribute.ts +++ b/src/lib/graphql/syncAttribute.ts @@ -1,7 +1,10 @@ -import { AttributeInput } from '../../codecs'; import colors from 'colors'; -import { keyBy, difference, groupBy } from 'lodash-es'; import { GraphQLClient } from 'graphql-request'; +import { difference, groupBy, keyBy } from 'lodash-es'; +import { AttributeInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { Attribute } from './fetchAllAttributes'; import { CREATE_ATTRIBUTE, CREATE_ATTRIBUTE_VALUES, @@ -10,9 +13,6 @@ import { UPDATE_ATTRIBUTE_VALUES, } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { Attribute } from './fetchAllAttributes'; -import { map } from '../bluebird-replace'; -import { logger } from '../../logger'; /** * Sync attribute diff --git a/src/lib/graphql/syncBusinessEntities.ts b/src/lib/graphql/syncBusinessEntities.ts index 23e81cfc..8fec0a27 100644 --- a/src/lib/graphql/syncBusinessEntities.ts +++ b/src/lib/graphql/syncBusinessEntities.ts @@ -1,15 +1,15 @@ -import { BusinessEntityInput } from '../../codecs'; +import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { mapSeries } from '../bluebird-replace'; -import { UPDATE_BUSINESS_ENTITIES, CREATE_BUSINESS_ENTITY } from './gqls'; +import { chunk, keyBy } from 'lodash-es'; +import { BusinessEntityInput } from '../../codecs'; import { logger } from '../../logger'; -import { keyBy, chunk } from 'lodash-es'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { mapSeries } from '../bluebird-replace'; import { - fetchAllBusinessEntities, BusinessEntity, + fetchAllBusinessEntities, } from './fetchAllBusinessEntities'; -import colors from 'colors'; +import { CREATE_BUSINESS_ENTITY, UPDATE_BUSINESS_ENTITIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new business entity diff --git a/src/lib/graphql/syncCodePackages.ts b/src/lib/graphql/syncCodePackages.ts index 6ef4e8de..700a23ac 100644 --- a/src/lib/graphql/syncCodePackages.ts +++ b/src/lib/graphql/syncCodePackages.ts @@ -1,15 +1,15 @@ -import { chunk, uniq, keyBy, uniqBy } from 'lodash-es'; +import { CodePackageType } from '@transcend-io/privacy-types'; import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { CodePackage, fetchAllCodePackages } from './fetchAllCodePackages'; +import { chunk, keyBy, uniq, uniqBy } from 'lodash-es'; +import { CodePackageInput, RepositoryInput } from '../../codecs'; import { logger } from '../../logger'; -import { syncSoftwareDevelopmentKits } from './syncSoftwareDevelopmentKits'; import { map, mapSeries } from '../bluebird-replace'; -import { CodePackageInput, RepositoryInput } from '../../codecs'; -import { CodePackageType } from '@transcend-io/privacy-types'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { CodePackage, fetchAllCodePackages } from './fetchAllCodePackages'; import { CREATE_CODE_PACKAGE, UPDATE_CODE_PACKAGES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; import { syncRepositories } from './syncRepositories'; +import { syncSoftwareDevelopmentKits } from './syncSoftwareDevelopmentKits'; const CHUNK_SIZE = 100; @@ -166,7 +166,7 @@ export async function syncCodePackages( ({ name: repositoryName, url: `https://github.com/${repositoryName}`, - } as RepositoryInput), + }) as RepositoryInput, ), ), ]); diff --git a/src/lib/graphql/syncConfigurationToTranscend.ts b/src/lib/graphql/syncConfigurationToTranscend.ts index f59d9d8f..e08eb574 100644 --- a/src/lib/graphql/syncConfigurationToTranscend.ts +++ b/src/lib/graphql/syncConfigurationToTranscend.ts @@ -1,47 +1,47 @@ /* eslint-disable max-lines */ -import { TranscendInput } from '../../codecs'; +import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; +import { TranscendInput } from '../../codecs'; import { logger } from '../../logger'; -import colors from 'colors'; import { map } from '../bluebird-replace'; +import { fetchAllActions } from './fetchAllActions'; +import { fetchAllAttributes } from './fetchAllAttributes'; +import { fetchApiKeys } from './fetchApiKeys'; +import { + ensureAllDataSubjectsExist, + fetchAllDataSubjects, +} from './fetchDataSubjects'; import { fetchIdentifiersAndCreateMissing, Identifier, } from './fetchIdentifiers'; -import { syncIdentifier } from './syncIdentifier'; -import { syncEnricher } from './syncEnrichers'; +import { syncAction } from './syncAction'; +import { syncActionItemCollections } from './syncActionItemCollections'; +import { syncActionItems } from './syncActionItems'; +import { syncAgentFiles } from './syncAgentFiles'; +import { syncAgentFunctions } from './syncAgentFunctions'; +import { syncAgents } from './syncAgents'; import { syncAttribute } from './syncAttribute'; -import { syncDataSiloDependencies, syncDataSilos } from './syncDataSilos'; +import { syncBusinessEntities } from './syncBusinessEntities'; +import { syncConsentManager } from './syncConsentManager'; import { syncCookies } from './syncCookies'; -import { - fetchAllDataSubjects, - ensureAllDataSubjectsExist, -} from './fetchDataSubjects'; -import { syncTeams } from './syncTeams'; +import { syncDataCategories } from './syncDataCategories'; +import { syncDataFlows } from './syncDataFlows'; +import { syncDataSiloDependencies, syncDataSilos } from './syncDataSilos'; import { syncDataSubject } from './syncDataSubject'; -import { fetchApiKeys } from './fetchApiKeys'; -import { syncPrompts } from './syncPrompts'; -import { syncPolicies } from './syncPolicies'; +import { syncEnricher } from './syncEnrichers'; +import { syncIdentifier } from './syncIdentifier'; import { syncIntlMessages } from './syncIntlMessages'; +import { syncPartitions } from './syncPartitions'; +import { syncPolicies } from './syncPolicies'; import { syncPrivacyCenter } from './syncPrivacyCenter'; -import { syncConsentManager } from './syncConsentManager'; -import { fetchAllAttributes } from './fetchAllAttributes'; -import { syncBusinessEntities } from './syncBusinessEntities'; -import { syncDataFlows } from './syncDataFlows'; -import { syncAction } from './syncAction'; -import { syncTemplate } from './syncTemplates'; -import { fetchAllActions } from './fetchAllActions'; -import { syncPromptPartials } from './syncPromptPartials'; +import { syncProcessingPurposes } from './syncProcessingPurposes'; import { syncPromptGroups } from './syncPromptGroups'; -import { syncAgents } from './syncAgents'; -import { syncActionItemCollections } from './syncActionItemCollections'; -import { syncActionItems } from './syncActionItems'; -import { syncAgentFunctions } from './syncAgentFunctions'; -import { syncAgentFiles } from './syncAgentFiles'; +import { syncPromptPartials } from './syncPromptPartials'; +import { syncPrompts } from './syncPrompts'; +import { syncTeams } from './syncTeams'; +import { syncTemplate } from './syncTemplates'; import { syncVendors } from './syncVendors'; -import { syncDataCategories } from './syncDataCategories'; -import { syncProcessingPurposes } from './syncProcessingPurposes'; -import { syncPartitions } from './syncPartitions'; const CONCURRENCY = 10; diff --git a/src/lib/graphql/syncConsentManager.ts b/src/lib/graphql/syncConsentManager.ts index 7f1802f5..818451ec 100644 --- a/src/lib/graphql/syncConsentManager.ts +++ b/src/lib/graphql/syncConsentManager.ts @@ -1,38 +1,38 @@ +import { + InitialViewState, + OnConsentExpiry, +} from '@transcend-io/airgap.js-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; import { ConsentManageExperienceInput, ConsentManagerInput, } from '../../codecs'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllPurposes } from './fetchAllPurposes'; import { - UPDATE_CONSENT_MANAGER_DOMAINS, + fetchConsentManagerExperiences, + fetchConsentManagerId, +} from './fetchConsentManagerId'; +import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; +import { + CREATE_CONSENT_EXPERIENCE, CREATE_CONSENT_MANAGER, - UPDATE_TOGGLE_USP_API, - UPDATE_CONSENT_MANAGER_PARTITION, - UPDATE_CONSENT_MANAGER_VERSION, + TOGGLE_CONSENT_PRECEDENCE, TOGGLE_TELEMETRY_PARTITION_STRATEGY, TOGGLE_UNKNOWN_COOKIE_POLICY, - TOGGLE_CONSENT_PRECEDENCE, TOGGLE_UNKNOWN_REQUEST_POLICY, UPDATE_CONSENT_EXPERIENCE, - CREATE_CONSENT_EXPERIENCE, + UPDATE_CONSENT_MANAGER_DOMAINS, + UPDATE_CONSENT_MANAGER_PARTITION, UPDATE_CONSENT_MANAGER_THEME, + UPDATE_CONSENT_MANAGER_VERSION, + UPDATE_TOGGLE_USP_API, } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { - fetchConsentManagerId, - fetchConsentManagerExperiences, -} from './fetchConsentManagerId'; -import { keyBy } from 'lodash-es'; -import { map } from '../bluebird-replace'; -import { - InitialViewState, - OnConsentExpiry, -} from '@transcend-io/airgap.js-types'; -import { logger } from '../../logger'; -import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; import { fetchPartitions } from './syncPartitions'; -import { fetchAllPurposes } from './fetchAllPurposes'; const PURPOSES_LINK = 'https://app.transcend.io/consent-manager/regional-experiences/purposes'; diff --git a/src/lib/graphql/syncCookies.ts b/src/lib/graphql/syncCookies.ts index fa464bfd..0faa8ace 100644 --- a/src/lib/graphql/syncCookies.ts +++ b/src/lib/graphql/syncCookies.ts @@ -1,11 +1,11 @@ -import { GraphQLClient } from 'graphql-request'; -import { logger } from '../../logger'; -import { CookieInput } from '../../codecs'; import colors from 'colors'; -import { UPDATE_OR_CREATE_COOKIES } from './gqls'; +import { GraphQLClient } from 'graphql-request'; import { chunk } from 'lodash-es'; -import { fetchConsentManagerId } from './fetchConsentManagerId'; +import { CookieInput } from '../../codecs'; +import { logger } from '../../logger'; import { mapSeries } from '../bluebird-replace'; +import { fetchConsentManagerId } from './fetchConsentManagerId'; +import { UPDATE_OR_CREATE_COOKIES } from './gqls'; // import { keyBy } from 'lodash-es'; import { makeGraphQLRequest } from './makeGraphQLRequest'; diff --git a/src/lib/graphql/syncDataCategories.ts b/src/lib/graphql/syncDataCategories.ts index 174dfe14..c00912d2 100644 --- a/src/lib/graphql/syncDataCategories.ts +++ b/src/lib/graphql/syncDataCategories.ts @@ -1,15 +1,15 @@ -import { DataCategoryInput } from '../../codecs'; +import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { mapSeries } from '../bluebird-replace'; -import { UPDATE_DATA_SUB_CATEGORIES, CREATE_DATA_SUB_CATEGORY } from './gqls'; -import { logger } from '../../logger'; import { keyBy } from 'lodash-es'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import colors from 'colors'; +import { DataCategoryInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { - fetchAllDataCategories, DataSubCategory, + fetchAllDataCategories, } from './fetchAllDataCategories'; +import { CREATE_DATA_SUB_CATEGORY, UPDATE_DATA_SUB_CATEGORIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new data category diff --git a/src/lib/graphql/syncDataFlows.ts b/src/lib/graphql/syncDataFlows.ts index 075bce47..0c119837 100644 --- a/src/lib/graphql/syncDataFlows.ts +++ b/src/lib/graphql/syncDataFlows.ts @@ -1,14 +1,14 @@ +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { CREATE_DATA_FLOWS, UPDATE_DATA_FLOWS } from './gqls'; import { chunk } from 'lodash-es'; -import { mapSeries } from '../bluebird-replace'; import { DataFlowInput } from '../../codecs'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { fetchConsentManagerId } from './fetchConsentManagerId'; import { logger } from '../../logger'; -import colors from 'colors'; +import { mapSeries } from '../bluebird-replace'; import { fetchAllDataFlows } from './fetchAllDataFlows'; -import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; +import { fetchConsentManagerId } from './fetchConsentManagerId'; +import { CREATE_DATA_FLOWS, UPDATE_DATA_FLOWS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const MAX_PAGE_SIZE = 100; diff --git a/src/lib/graphql/syncDataSilos.ts b/src/lib/graphql/syncDataSilos.ts index 2c66f7bb..3e38c3fc 100644 --- a/src/lib/graphql/syncDataSilos.ts +++ b/src/lib/graphql/syncDataSilos.ts @@ -1,41 +1,41 @@ /* eslint-disable max-lines */ +import { + ConfidenceLabel, + IsoCountryCode, + IsoCountrySubdivisionCode, + PromptAVendorEmailCompletionLinkType, + PromptAVendorEmailSendType, + RequestActionObjectResolver, + SubDataPointDataSubCategoryGuessStatus, +} from '@transcend-io/privacy-types'; +import { apply } from '@transcend-io/type-utils'; import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy, sortBy } from 'lodash-es'; import { DataCategoryInput, DataSiloInput, ProcessingPurposeInput, } from '../../codecs'; -import { GraphQLClient } from 'graphql-request'; import { logger } from '../../logger'; -import colors from 'colors'; -import { mapSeries, map } from '../bluebird-replace'; +import { map, mapSeries } from '../bluebird-replace'; +import { ApiKey } from './fetchApiKeys'; +import { + convertToDataSubjectBlockList, + DataSubject, +} from './fetchDataSubjects'; import { - DATA_SILOS, CREATE_DATA_SILOS, - UPDATE_OR_CREATE_DATA_POINT, DATA_POINTS, - SUB_DATA_POINTS, - UPDATE_DATA_SILOS, + DATA_SILOS, DATA_SILOS_ENRICHED, + SUB_DATA_POINTS, SUB_DATA_POINTS_WITH_GUESSES, + UPDATE_DATA_SILOS, + UPDATE_OR_CREATE_DATA_POINT, } from './gqls'; -import { - convertToDataSubjectBlockList, - DataSubject, -} from './fetchDataSubjects'; -import { ApiKey } from './fetchApiKeys'; -import { - IsoCountryCode, - IsoCountrySubdivisionCode, - PromptAVendorEmailCompletionLinkType, - PromptAVendorEmailSendType, - ConfidenceLabel, - RequestActionObjectResolver, - SubDataPointDataSubCategoryGuessStatus, -} from '@transcend-io/privacy-types'; -import { sortBy, chunk, keyBy } from 'lodash-es'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { apply } from '@transcend-io/type-utils'; export interface DataSiloAttributeValue { /** Key associated to value */ diff --git a/src/lib/graphql/syncDataSubject.ts b/src/lib/graphql/syncDataSubject.ts index dcd8c896..48d868df 100644 --- a/src/lib/graphql/syncDataSubject.ts +++ b/src/lib/graphql/syncDataSubject.ts @@ -1,6 +1,6 @@ -import { DataSubjectInput } from '../../codecs'; import { GraphQLClient } from 'graphql-request'; -import { UPDATE_DATA_SUBJECT, TOGGLE_DATA_SUBJECT } from './gqls'; +import { DataSubjectInput } from '../../codecs'; +import { TOGGLE_DATA_SUBJECT, UPDATE_DATA_SUBJECT } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; /** diff --git a/src/lib/graphql/syncEnrichers.ts b/src/lib/graphql/syncEnrichers.ts index 74f3e163..00d6a0a1 100644 --- a/src/lib/graphql/syncEnrichers.ts +++ b/src/lib/graphql/syncEnrichers.ts @@ -1,6 +1,3 @@ -import { EnricherInput } from '../../codecs'; -import { GraphQLClient } from 'graphql-request'; -import { ENRICHERS, CREATE_ENRICHER, UPDATE_ENRICHER } from './gqls'; import { EnricherType, IsoCountryCode, @@ -8,9 +5,12 @@ import { PreflightRequestStatus, RequestAction, } from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { EnricherInput } from '../../codecs'; +import { DataSubject } from './fetchDataSubjects'; import { Identifier } from './fetchIdentifiers'; +import { CREATE_ENRICHER, ENRICHERS, UPDATE_ENRICHER } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { DataSubject } from './fetchDataSubjects'; export interface Enricher { /** ID of enricher */ diff --git a/src/lib/graphql/syncIdentifier.ts b/src/lib/graphql/syncIdentifier.ts index 081b4a3a..fd653c3c 100644 --- a/src/lib/graphql/syncIdentifier.ts +++ b/src/lib/graphql/syncIdentifier.ts @@ -1,8 +1,8 @@ -import { IdentifierInput } from '../../codecs'; import { GraphQLClient } from 'graphql-request'; +import { IdentifierInput } from '../../codecs'; +import type { DataSubject } from './fetchDataSubjects'; import { UPDATE_IDENTIFIER } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import type { DataSubject } from './fetchDataSubjects'; /** * Sync the consent manager diff --git a/src/lib/graphql/syncIntlMessages.ts b/src/lib/graphql/syncIntlMessages.ts index 0debecf9..e6d88fcb 100644 --- a/src/lib/graphql/syncIntlMessages.ts +++ b/src/lib/graphql/syncIntlMessages.ts @@ -1,10 +1,10 @@ -import { GraphQLClient } from 'graphql-request'; -import { logger } from '../../logger'; -import { IntlMessageInput } from '../../codecs'; import colors from 'colors'; -import { UPDATE_INTL_MESSAGES } from './gqls'; +import { GraphQLClient } from 'graphql-request'; import { chunk } from 'lodash-es'; +import { IntlMessageInput } from '../../codecs'; +import { logger } from '../../logger'; import { mapSeries } from '../bluebird-replace'; +import { UPDATE_INTL_MESSAGES } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; const MAX_PAGE_SIZE = 100; diff --git a/src/lib/graphql/syncPartitions.ts b/src/lib/graphql/syncPartitions.ts index 4bebdf4d..f0153e2b 100644 --- a/src/lib/graphql/syncPartitions.ts +++ b/src/lib/graphql/syncPartitions.ts @@ -1,12 +1,12 @@ import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { CREATE_CONSENT_PARTITION, CONSENT_PARTITIONS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { mapSeries } from '../bluebird-replace'; import { difference } from 'lodash-es'; -import { logger } from '../../logger'; import { PartitionInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { fetchConsentManagerId } from './fetchConsentManagerId'; +import { CONSENT_PARTITIONS, CREATE_CONSENT_PARTITION } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const PAGE_SIZE = 50; diff --git a/src/lib/graphql/syncPolicies.ts b/src/lib/graphql/syncPolicies.ts index 79b8dcb9..46d0c0ff 100644 --- a/src/lib/graphql/syncPolicies.ts +++ b/src/lib/graphql/syncPolicies.ts @@ -1,13 +1,13 @@ -import { GraphQLClient } from 'graphql-request'; -import { logger } from '../../logger'; -import { PolicyInput } from '../../codecs'; import colors from 'colors'; -import { UPDATE_POLICIES } from './gqls'; +import { GraphQLClient } from 'graphql-request'; import { chunk, keyBy } from 'lodash-es'; +import { PolicyInput } from '../../codecs'; +import { logger } from '../../logger'; import { mapSeries } from '../bluebird-replace'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; import { fetchAllPolicies } from './fetchAllPolicies'; +import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; +import { UPDATE_POLICIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const MAX_PAGE_SIZE = 100; diff --git a/src/lib/graphql/syncPrivacyCenter.ts b/src/lib/graphql/syncPrivacyCenter.ts index 140fd9df..b6e21db7 100644 --- a/src/lib/graphql/syncPrivacyCenter.ts +++ b/src/lib/graphql/syncPrivacyCenter.ts @@ -1,10 +1,10 @@ -import { PrivacyCenterInput } from '../../codecs'; -import { logger } from '../../logger'; import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; +import { PrivacyCenterInput } from '../../codecs'; +import { logger } from '../../logger'; +import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; import { UPDATE_PRIVACY_CENTER } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; /** * Sync the privacy center diff --git a/src/lib/graphql/syncProcessingPurposes.ts b/src/lib/graphql/syncProcessingPurposes.ts index b5c0ff41..c8a74b37 100644 --- a/src/lib/graphql/syncProcessingPurposes.ts +++ b/src/lib/graphql/syncProcessingPurposes.ts @@ -1,18 +1,18 @@ -import { ProcessingPurposeInput } from '../../codecs'; +import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { mapSeries } from '../bluebird-replace'; -import { - UPDATE_PROCESSING_PURPOSE_SUB_CATEGORIES, - CREATE_PROCESSING_PURPOSE_SUB_CATEGORY, -} from './gqls'; -import { logger } from '../../logger'; import { keyBy } from 'lodash-es'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import colors from 'colors'; +import { ProcessingPurposeInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { fetchAllProcessingPurposes, ProcessingPurposeSubCategory, } from './fetchAllProcessingPurposes'; +import { + CREATE_PROCESSING_PURPOSE_SUB_CATEGORY, + UPDATE_PROCESSING_PURPOSE_SUB_CATEGORIES, +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new processing purpose diff --git a/src/lib/graphql/syncPromptGroups.ts b/src/lib/graphql/syncPromptGroups.ts index 1fe6e2f9..82865c42 100644 --- a/src/lib/graphql/syncPromptGroups.ts +++ b/src/lib/graphql/syncPromptGroups.ts @@ -1,13 +1,13 @@ -import { PromptGroupInput } from '../../codecs'; import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { UPDATE_PROMPT_GROUPS, CREATE_PROMPT_GROUP } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { map } from '../bluebird-replace'; -import { fetchAllPromptGroups } from './fetchPromptGroups'; import { keyBy } from 'lodash-es'; +import { PromptGroupInput } from '../../codecs'; import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllPromptGroups } from './fetchPromptGroups'; import { fetchAllPrompts } from './fetchPrompts'; +import { CREATE_PROMPT_GROUP, UPDATE_PROMPT_GROUPS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface EditPromptGroupInput { /** Title of prompt group */ diff --git a/src/lib/graphql/syncPromptPartials.ts b/src/lib/graphql/syncPromptPartials.ts index c6e6c44e..424e245a 100644 --- a/src/lib/graphql/syncPromptPartials.ts +++ b/src/lib/graphql/syncPromptPartials.ts @@ -1,12 +1,12 @@ -import { PromptPartialInput } from '../../codecs'; import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { UPDATE_PROMPT_PARTIALS, CREATE_PROMPT_PARTIAL } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { map } from '../bluebird-replace'; -import { fetchAllPromptPartials } from './fetchPromptPartials'; import { keyBy } from 'lodash-es'; +import { PromptPartialInput } from '../../codecs'; import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllPromptPartials } from './fetchPromptPartials'; +import { CREATE_PROMPT_PARTIAL, UPDATE_PROMPT_PARTIALS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Create a new prompt partial diff --git a/src/lib/graphql/syncPrompts.ts b/src/lib/graphql/syncPrompts.ts index 06d1bea1..6e0819de 100644 --- a/src/lib/graphql/syncPrompts.ts +++ b/src/lib/graphql/syncPrompts.ts @@ -1,12 +1,12 @@ -import { PromptInput } from '../../codecs'; import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { UPDATE_PROMPTS, CREATE_PROMPT } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { map } from '../bluebird-replace'; -import { fetchAllPrompts } from './fetchPrompts'; import { keyBy } from 'lodash-es'; +import { PromptInput } from '../../codecs'; import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllPrompts } from './fetchPrompts'; +import { CREATE_PROMPT, UPDATE_PROMPTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Create a new prompt diff --git a/src/lib/graphql/syncRepositories.ts b/src/lib/graphql/syncRepositories.ts index 78c035af..49fdfde8 100644 --- a/src/lib/graphql/syncRepositories.ts +++ b/src/lib/graphql/syncRepositories.ts @@ -1,12 +1,12 @@ import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; import { chunk, keyBy } from 'lodash-es'; import { RepositoryInput } from '../../codecs'; -import { GraphQLClient } from 'graphql-request'; -import { UPDATE_REPOSITORIES, CREATE_REPOSITORY } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { mapSeries, map } from '../bluebird-replace'; -import { fetchAllRepositories, Repository } from './fetchAllRepositories'; import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; +import { fetchAllRepositories, Repository } from './fetchAllRepositories'; +import { CREATE_REPOSITORY, UPDATE_REPOSITORIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const CHUNK_SIZE = 100; diff --git a/src/lib/graphql/syncSoftwareDevelopmentKits.ts b/src/lib/graphql/syncSoftwareDevelopmentKits.ts index 5a9ed6fe..fff8a0f9 100644 --- a/src/lib/graphql/syncSoftwareDevelopmentKits.ts +++ b/src/lib/graphql/syncSoftwareDevelopmentKits.ts @@ -1,19 +1,19 @@ +import { CodePackageType } from '@transcend-io/privacy-types'; import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; import { chunk, keyBy } from 'lodash-es'; import { SoftwareDevelopmentKitInput } from '../../codecs'; -import { GraphQLClient } from 'graphql-request'; -import { - UPDATE_SOFTWARE_DEVELOPMENT_KITS, - CREATE_SOFTWARE_DEVELOPMENT_KIT, -} from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { mapSeries, map } from '../bluebird-replace'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; import { fetchAllSoftwareDevelopmentKits, SoftwareDevelopmentKit, } from './fetchAllSoftwareDevelopmentKits'; -import { logger } from '../../logger'; -import { CodePackageType } from '@transcend-io/privacy-types'; +import { + CREATE_SOFTWARE_DEVELOPMENT_KIT, + UPDATE_SOFTWARE_DEVELOPMENT_KITS, +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const CHUNK_SIZE = 100; diff --git a/src/lib/graphql/syncTeams.ts b/src/lib/graphql/syncTeams.ts index 48fcd5bd..998ddc58 100644 --- a/src/lib/graphql/syncTeams.ts +++ b/src/lib/graphql/syncTeams.ts @@ -1,12 +1,12 @@ -import { TeamInput } from '../../codecs'; +import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { mapSeries } from '../bluebird-replace'; -import { UPDATE_TEAM, CREATE_TEAM } from './gqls'; -import { logger } from '../../logger'; import { keyBy } from 'lodash-es'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import colors from 'colors'; +import { TeamInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { fetchAllTeams, Team } from './fetchAllTeams'; +import { CREATE_TEAM, UPDATE_TEAM } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new team diff --git a/src/lib/graphql/syncTemplates.ts b/src/lib/graphql/syncTemplates.ts index 1445af50..ac8c7dbc 100644 --- a/src/lib/graphql/syncTemplates.ts +++ b/src/lib/graphql/syncTemplates.ts @@ -1,6 +1,6 @@ -import { TemplateInput } from '../../codecs'; import { GraphQLClient } from 'graphql-request'; -import { TEMPLATES, CREATE_TEMPLATE } from './gqls'; +import { TemplateInput } from '../../codecs'; +import { CREATE_TEMPLATE, TEMPLATES } from './gqls'; import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Template { diff --git a/src/lib/graphql/syncVendors.ts b/src/lib/graphql/syncVendors.ts index 0dfc4bb3..9df26c69 100644 --- a/src/lib/graphql/syncVendors.ts +++ b/src/lib/graphql/syncVendors.ts @@ -1,12 +1,12 @@ -import { VendorInput } from '../../codecs'; +import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; -import { mapSeries } from '../bluebird-replace'; -import { UPDATE_VENDORS, CREATE_VENDOR } from './gqls'; -import { logger } from '../../logger'; import { keyBy } from 'lodash-es'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import colors from 'colors'; +import { VendorInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { fetchAllVendors, Vendor } from './fetchAllVendors'; +import { CREATE_VENDOR, UPDATE_VENDORS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new vendor diff --git a/src/lib/graphql/uploadSiloDiscoveryResults.ts b/src/lib/graphql/uploadSiloDiscoveryResults.ts index b99cfc07..5ecc80d5 100644 --- a/src/lib/graphql/uploadSiloDiscoveryResults.ts +++ b/src/lib/graphql/uploadSiloDiscoveryResults.ts @@ -1,9 +1,9 @@ +import { GraphQLClient } from 'graphql-request'; import { chunk } from 'lodash-es'; import { mapSeries } from '../bluebird-replace'; +import { SiloDiscoveryRawResults } from '../code-scanning/findFilesToScan'; import { ADD_SILO_DISCOVERY_RESULTS } from './gqls'; -import { GraphQLClient } from 'graphql-request'; import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { SiloDiscoveryRawResults } from '../code-scanning/findFilesToScan'; const CHUNK_SIZE = 1000; diff --git a/src/lib/helpers/buildEnabledRouteType.ts b/src/lib/helpers/buildEnabledRouteType.ts index adb43b26..77b488fc 100644 --- a/src/lib/helpers/buildEnabledRouteType.ts +++ b/src/lib/helpers/buildEnabledRouteType.ts @@ -1,6 +1,6 @@ +import { valuesOf } from '@transcend-io/type-utils'; import * as t from 'io-ts'; import { EnabledRouteC } from '../../codecs'; -import { valuesOf } from '@transcend-io/type-utils'; import { PathfinderPolicyName } from '../../enums'; /** diff --git a/src/lib/helpers/inquirer.ts b/src/lib/helpers/inquirer.ts index 7c82f96e..21b501c8 100644 --- a/src/lib/helpers/inquirer.ts +++ b/src/lib/helpers/inquirer.ts @@ -1,7 +1,7 @@ +import { ObjByString } from '@transcend-io/type-utils'; import inquirer from 'inquirer'; import autoCompletePrompt from 'inquirer-autocomplete-prompt'; import { fuzzySearch } from '../requests'; -import { ObjByString } from '@transcend-io/type-utils'; /** * Inquirer confirm text diff --git a/src/lib/manual-enrichment/enrichPrivacyRequest.ts b/src/lib/manual-enrichment/enrichPrivacyRequest.ts index 44c39864..1d7859e5 100644 --- a/src/lib/manual-enrichment/enrichPrivacyRequest.ts +++ b/src/lib/manual-enrichment/enrichPrivacyRequest.ts @@ -1,8 +1,8 @@ +import colors from 'colors'; import type { Got } from 'got'; import * as t from 'io-ts'; -import { logger } from '../../logger'; import { uniq } from 'lodash-es'; -import colors from 'colors'; +import { logger } from '../../logger'; import { splitCsvToList } from '../requests/splitCsvToList'; const ADMIN_URL = diff --git a/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts b/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts index c3650070..37ff3f59 100644 --- a/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts +++ b/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts @@ -1,20 +1,20 @@ import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; -import { map } from '../bluebird-replace'; import colors from 'colors'; import { groupBy, uniq } from 'lodash-es'; import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { writeCsv } from '../cron/writeCsv'; import { - PrivacyRequest, - RequestEnricher, - RequestIdentifier, buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequestEnrichers, fetchAllRequestIdentifiers, fetchAllRequests, + PrivacyRequest, + RequestEnricher, + RequestIdentifier, } from '../graphql'; -import { logger } from '../../logger'; export interface PrivacyRequestWithIdentifiers extends PrivacyRequest { /** Request Enrichers */ diff --git a/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts b/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts index 9bf33641..248be712 100644 --- a/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts +++ b/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts @@ -1,18 +1,18 @@ import colors from 'colors'; -import { map } from '../bluebird-replace'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { - UPDATE_PRIVACY_REQUEST, buildTranscendGraphQLClient, createSombraGotInstance, makeGraphQLRequest, + UPDATE_PRIVACY_REQUEST, } from '../graphql'; +import { readCsv } from '../requests'; import { enrichPrivacyRequest, EnrichPrivacyRequest, } from './enrichPrivacyRequest'; -import { readCsv } from '../requests'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; /** * Push a CSV of enriched requests back into Transcend diff --git a/src/lib/mergeTranscendInputs.ts b/src/lib/mergeTranscendInputs.ts index 12084abc..de6c04e4 100644 --- a/src/lib/mergeTranscendInputs.ts +++ b/src/lib/mergeTranscendInputs.ts @@ -1,5 +1,5 @@ -import { TranscendInput } from '../codecs'; import { getEntries } from '@transcend-io/type-utils'; +import { TranscendInput } from '../codecs'; /** * Combine a set of TranscendInput yaml files into a single yaml diff --git a/src/lib/oneTrust/endpoints/getListOfOneTrustAssessments.ts b/src/lib/oneTrust/endpoints/getListOfOneTrustAssessments.ts index b2bcbd4a..ef7a9b13 100644 --- a/src/lib/oneTrust/endpoints/getListOfOneTrustAssessments.ts +++ b/src/lib/oneTrust/endpoints/getListOfOneTrustAssessments.ts @@ -1,10 +1,10 @@ -import { Got } from 'got'; -import { logger } from '../../../logger'; -import { decodeCodec } from '@transcend-io/type-utils'; import { OneTrustAssessment, OneTrustGetListOfAssessmentsResponse, } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import { Got } from 'got'; +import { logger } from '../../../logger'; /** * Fetch a list of all assessments from the OneTrust client. diff --git a/src/lib/oneTrust/endpoints/getOneTrustAssessment.ts b/src/lib/oneTrust/endpoints/getOneTrustAssessment.ts index 6a02226e..f9440136 100644 --- a/src/lib/oneTrust/endpoints/getOneTrustAssessment.ts +++ b/src/lib/oneTrust/endpoints/getOneTrustAssessment.ts @@ -1,6 +1,6 @@ -import { Got } from 'got'; -import { decodeCodec } from '@transcend-io/type-utils'; import { OneTrustGetAssessmentResponse } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import { Got } from 'got'; /** * Retrieve details about a particular assessment. diff --git a/src/lib/oneTrust/endpoints/getOneTrustRisk.ts b/src/lib/oneTrust/endpoints/getOneTrustRisk.ts index 78184b02..fa1d392e 100644 --- a/src/lib/oneTrust/endpoints/getOneTrustRisk.ts +++ b/src/lib/oneTrust/endpoints/getOneTrustRisk.ts @@ -1,6 +1,6 @@ -import { Got } from 'got'; -import { decodeCodec } from '@transcend-io/type-utils'; import { OneTrustGetRiskResponse } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import { Got } from 'got'; /** * Retrieve details about a particular risk. diff --git a/src/lib/oneTrust/endpoints/getOneTrustUser.ts b/src/lib/oneTrust/endpoints/getOneTrustUser.ts index edbf81da..a2fa931c 100644 --- a/src/lib/oneTrust/endpoints/getOneTrustUser.ts +++ b/src/lib/oneTrust/endpoints/getOneTrustUser.ts @@ -1,6 +1,6 @@ -import { Got } from 'got'; -import { decodeCodec } from '@transcend-io/type-utils'; import { OneTrustGetUserResponse } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import { Got } from 'got'; /** * Retrieve details about a particular user. diff --git a/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts b/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts index 640ffb5a..e8c343b8 100644 --- a/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts +++ b/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts @@ -1,4 +1,3 @@ -import { logger } from '../../../logger'; import colors from 'colors'; import yargs from 'yargs-parser'; import { @@ -6,6 +5,7 @@ import { OneTrustPullResource, OneTrustPullSource, } from '../../../enums'; +import { logger } from '../../../logger'; const VALID_RESOURCES = Object.values(OneTrustPullResource); diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts index 1a00c7c0..89545f17 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts @@ -1,8 +1,8 @@ -import { logger } from '../../../logger'; -import colors from 'colors'; import fs from 'fs'; -import { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson'; import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { logger } from '../../../logger'; +import { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson'; /** * Write the assessment to disk at the specified file path. diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts index 4026ad18..c0bc771b 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts @@ -1,12 +1,12 @@ -import { logger } from '../../../logger'; +import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; import colors from 'colors'; import { GraphQLClient } from 'graphql-request'; +import { ImportOnetrustAssessmentsInput } from '../../../codecs'; +import { logger } from '../../../logger'; import { IMPORT_ONE_TRUST_ASSESSMENT_FORMS, makeGraphQLRequest, } from '../../graphql'; -import { ImportOnetrustAssessmentsInput } from '../../../codecs'; -import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; import { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson'; export interface AssessmentForm { diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts index 1783eb00..cad3b78e 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts @@ -1,12 +1,11 @@ +import { createReadStream } from 'fs'; +import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; import { decodeCodec } from '@transcend-io/type-utils'; import colors from 'colors'; -import { logger } from '../../../logger'; +import { GraphQLClient } from 'graphql-request'; import JSONStream from 'JSONStream'; - -import { createReadStream } from 'fs'; -import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; +import { logger } from '../../../logger'; import { syncOneTrustAssessmentToTranscend } from './syncOneTrustAssessmentToTranscend'; -import { GraphQLClient } from 'graphql-request'; /** * Reads assessments from a file and syncs them to Transcend. diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts index 6792c963..071e4817 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts @@ -1,13 +1,3 @@ -import type { Got } from 'got'; -import colors from 'colors'; -import { - getListOfOneTrustAssessments, - getOneTrustAssessment, - getOneTrustRisk, - getOneTrustUser, -} from '../endpoints'; -import { mapSeries, map } from '../../bluebird-replace'; -import { logger } from '../../../logger'; import { OneTrustAssessmentQuestion, OneTrustAssessmentSection, @@ -15,10 +5,20 @@ import { OneTrustGetRiskResponse, OneTrustGetUserResponse, } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import type { Got } from 'got'; +import { GraphQLClient } from 'graphql-request'; import { uniq } from 'lodash-es'; +import { logger } from '../../../logger'; +import { map, mapSeries } from '../../bluebird-replace'; +import { + getListOfOneTrustAssessments, + getOneTrustAssessment, + getOneTrustRisk, + getOneTrustUser, +} from '../endpoints'; import { enrichOneTrustAssessment } from './enrichOneTrustAssessment'; import { syncOneTrustAssessmentToDisk } from './syncOneTrustAssessmentToDisk'; -import { GraphQLClient } from 'graphql-request'; import { syncOneTrustAssessmentToTranscend } from './syncOneTrustAssessmentToTranscend'; export interface AssessmentForm { diff --git a/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts b/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts index 69269172..0a633c09 100644 --- a/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts +++ b/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts @@ -1,5 +1,4 @@ -import { expect, describe, it } from 'vitest'; - +import { describe, expect, it } from 'vitest'; import { convertToEmptyStrings } from '../convertToEmptyStrings'; describe('buildDefaultCodecWrapper', () => { diff --git a/src/lib/preference-management/getPreferenceUpdatesFromRow.ts b/src/lib/preference-management/getPreferenceUpdatesFromRow.ts index 32d63f68..cabbd8cd 100644 --- a/src/lib/preference-management/getPreferenceUpdatesFromRow.ts +++ b/src/lib/preference-management/getPreferenceUpdatesFromRow.ts @@ -2,10 +2,10 @@ import { PreferenceStorePurposeResponse, PreferenceTopicType, } from '@transcend-io/privacy-types'; -import { PurposeRowMapping } from './codecs'; import { apply } from '@transcend-io/type-utils'; import { PreferenceTopic } from '../graphql'; import { splitCsvToList } from '../requests'; +import { PurposeRowMapping } from './codecs'; /** * Parse an arbitrary object to the Transcend PUT /v1/preference update shape diff --git a/src/lib/preference-management/getPreferencesForIdentifiers.ts b/src/lib/preference-management/getPreferencesForIdentifiers.ts index 6a77d9b8..10bf46f8 100644 --- a/src/lib/preference-management/getPreferencesForIdentifiers.ts +++ b/src/lib/preference-management/getPreferencesForIdentifiers.ts @@ -1,12 +1,12 @@ import { PreferenceQueryResponseItem } from '@transcend-io/privacy-types'; -import type { Got } from 'got'; -import colors from 'colors'; -import cliProgress from 'cli-progress'; -import { chunk } from 'lodash-es'; import { decodeCodec } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import type { Got } from 'got'; import * as t from 'io-ts'; -import { map } from '../bluebird-replace'; +import { chunk } from 'lodash-es'; import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; const PreferenceRecordsQueryResponse = t.intersection([ t.type({ diff --git a/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts b/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts index 7c81c206..fe4b6322 100644 --- a/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts @@ -1,12 +1,12 @@ -import { uniq, difference } from 'lodash-es'; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; import colors from 'colors'; import inquirer from 'inquirer'; -import { FileMetadataState } from './codecs'; +import { difference, uniq } from 'lodash-es'; import { logger } from '../../logger'; import { mapSeries } from '../bluebird-replace'; import { PreferenceTopic } from '../graphql'; -import { PreferenceTopicType } from '@transcend-io/privacy-types'; import { splitCsvToList } from '../requests'; +import { FileMetadataState } from './codecs'; /* eslint-disable no-param-reassign */ diff --git a/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts b/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts index 0c65ccf2..5aea479b 100644 --- a/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts @@ -1,9 +1,9 @@ -import { uniq, groupBy, difference } from 'lodash-es'; import colors from 'colors'; import inquirer from 'inquirer'; -import { FileMetadataState } from './codecs'; +import { difference, groupBy, uniq } from 'lodash-es'; import { logger } from '../../logger'; import { inquirerConfirmBoolean } from '../helpers'; +import { FileMetadataState } from './codecs'; /* eslint-disable no-param-reassign */ diff --git a/src/lib/preference-management/parsePreferenceManagementCsv.ts b/src/lib/preference-management/parsePreferenceManagementCsv.ts index 10b36831..b2db315d 100644 --- a/src/lib/preference-management/parsePreferenceManagementCsv.ts +++ b/src/lib/preference-management/parsePreferenceManagementCsv.ts @@ -1,19 +1,19 @@ import { PersistedState } from '@transcend-io/persisted-state'; +import colors from 'colors'; import type { Got } from 'got'; -import { keyBy } from 'lodash-es'; import * as t from 'io-ts'; -import colors from 'colors'; -import { FileMetadataState, PreferenceState } from './codecs'; +import { keyBy } from 'lodash-es'; import { logger } from '../../logger'; +import { PreferenceTopic } from '../graphql'; import { readCsv } from '../requests'; +import { checkIfPendingPreferenceUpdatesAreNoOp } from './checkIfPendingPreferenceUpdatesAreNoOp'; +import { checkIfPendingPreferenceUpdatesCauseConflict } from './checkIfPendingPreferenceUpdatesCauseConflict'; +import { FileMetadataState, PreferenceState } from './codecs'; import { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers'; -import { PreferenceTopic } from '../graphql'; import { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow'; -import { parsePreferenceTimestampsFromCsv } from './parsePreferenceTimestampsFromCsv'; -import { parsePreferenceIdentifiersFromCsv } from './parsePreferenceIdentifiersFromCsv'; import { parsePreferenceAndPurposeValuesFromCsv } from './parsePreferenceAndPurposeValuesFromCsv'; -import { checkIfPendingPreferenceUpdatesAreNoOp } from './checkIfPendingPreferenceUpdatesAreNoOp'; -import { checkIfPendingPreferenceUpdatesCauseConflict } from './checkIfPendingPreferenceUpdatesCauseConflict'; +import { parsePreferenceIdentifiersFromCsv } from './parsePreferenceIdentifiersFromCsv'; +import { parsePreferenceTimestampsFromCsv } from './parsePreferenceTimestampsFromCsv'; /** * Parse a file into the cache diff --git a/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts b/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts index 6df3ac05..cfe5db8c 100644 --- a/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts @@ -1,8 +1,8 @@ -import { uniq, difference } from 'lodash-es'; import colors from 'colors'; import inquirer from 'inquirer'; -import { FileMetadataState } from './codecs'; +import { difference, uniq } from 'lodash-es'; import { logger } from '../../logger'; +import { FileMetadataState } from './codecs'; export const NONE_PREFERENCE_MAP = '[NONE]'; diff --git a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts index 81dfc6c5..cc467f2c 100644 --- a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts +++ b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts @@ -1,9 +1,8 @@ /* eslint-disable max-lines */ -import { expect, describe, it } from 'vitest'; - -import { checkIfPendingPreferenceUpdatesAreNoOp } from '../index'; import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import { describe, expect, it } from 'vitest'; import { PreferenceTopic } from '../../graphql'; +import { checkIfPendingPreferenceUpdatesAreNoOp } from '../index'; const DEFAULT_VALUES = { userId: 'test@transcend.io', diff --git a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts index 78dfb716..bca6bb10 100644 --- a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts +++ b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts @@ -1,9 +1,8 @@ /* eslint-disable max-lines */ -import { expect, describe, it } from 'vitest'; - -import { checkIfPendingPreferenceUpdatesCauseConflict } from '../index'; import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import { describe, expect, it } from 'vitest'; import { PreferenceTopic } from '../../graphql'; +import { checkIfPendingPreferenceUpdatesCauseConflict } from '../index'; const DEFAULT_VALUES = { userId: 'test@transcend.io', diff --git a/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts b/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts index 53e93a46..0ad9374c 100644 --- a/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts +++ b/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts @@ -1,8 +1,7 @@ /* eslint-disable max-lines */ -import { expect, describe, it } from 'vitest'; - -import { getPreferenceUpdatesFromRow } from '../index'; import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import { describe, expect, it } from 'vitest'; +import { getPreferenceUpdatesFromRow } from '../index'; describe('getPreferenceUpdatesFromRow', () => { it('should parse boolean updates', () => { diff --git a/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts b/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts index 2ae62080..ed8cd7a4 100644 --- a/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts +++ b/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts @@ -1,25 +1,25 @@ +import { PersistedState } from '@transcend-io/persisted-state'; +import { PreferenceUpdateItem } from '@transcend-io/privacy-types'; +import { apply } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { chunk } from 'lodash-es'; +import { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, - fetchAllPurposes, fetchAllPreferenceTopics, + fetchAllPurposes, PreferenceTopic, Purpose, } from '../graphql'; -import colors from 'colors'; -import { map } from '../bluebird-replace'; -import { chunk } from 'lodash-es'; -import { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants'; -import { logger } from '../../logger'; -import cliProgress from 'cli-progress'; import { parseAttributesFromString } from '../requests'; -import { PersistedState } from '@transcend-io/persisted-state'; -import { parsePreferenceManagementCsvWithCache } from './parsePreferenceManagementCsv'; import { PreferenceState } from './codecs'; -import { PreferenceUpdateItem } from '@transcend-io/privacy-types'; -import { apply } from '@transcend-io/type-utils'; -import { NONE_PREFERENCE_MAP } from './parsePreferenceTimestampsFromCsv'; import { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow'; +import { parsePreferenceManagementCsvWithCache } from './parsePreferenceManagementCsv'; +import { NONE_PREFERENCE_MAP } from './parsePreferenceTimestampsFromCsv'; /** * Upload a set of consent preferences diff --git a/src/lib/readTranscendYaml.ts b/src/lib/readTranscendYaml.ts index 4a3428e4..b7a54ee8 100644 --- a/src/lib/readTranscendYaml.ts +++ b/src/lib/readTranscendYaml.ts @@ -1,6 +1,6 @@ +import { readFileSync, writeFileSync } from 'fs'; import { decodeCodec, ObjByString } from '@transcend-io/type-utils'; import yaml from 'js-yaml'; -import { readFileSync, writeFileSync } from 'fs'; import { TranscendInput } from '../codecs'; export const VARIABLE_PARAMETERS_REGEXP = /<>/; diff --git a/src/lib/requests/approvePrivacyRequests.ts b/src/lib/requests/approvePrivacyRequests.ts index 58a62ea8..851fded5 100644 --- a/src/lib/requests/approvePrivacyRequests.ts +++ b/src/lib/requests/approvePrivacyRequests.ts @@ -1,20 +1,20 @@ -import { map } from '../bluebird-replace'; -import colors from 'colors'; -import { logger } from '../../logger'; import { RequestAction, RequestOrigin, RequestStatus, } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { - UPDATE_PRIVACY_REQUEST, + APPROVE_PRIVACY_REQUEST, + buildTranscendGraphQLClient, fetchAllRequests, makeGraphQLRequest, - buildTranscendGraphQLClient, - APPROVE_PRIVACY_REQUEST, + UPDATE_PRIVACY_REQUEST, } from '../graphql'; -import cliProgress from 'cli-progress'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; /** * Approve a set of privacy requests diff --git a/src/lib/requests/bulkRestartRequests.ts b/src/lib/requests/bulkRestartRequests.ts index 9422b390..8f4b4f2f 100644 --- a/src/lib/requests/bulkRestartRequests.ts +++ b/src/lib/requests/bulkRestartRequests.ts @@ -1,19 +1,19 @@ +import { join } from 'path'; import { PersistedState } from '@transcend-io/persisted-state'; import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; -import { map } from '../bluebird-replace'; import cliProgress from 'cli-progress'; import colors from 'colors'; import * as t from 'io-ts'; import { difference } from 'lodash-es'; -import { join } from 'path'; import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequestIdentifiers, fetchAllRequests, } from '../graphql'; -import { logger } from '../../logger'; import { SuccessfulRequest } from './constants'; import { extractClientError } from './extractClientError'; import { restartPrivacyRequest } from './restartPrivacyRequest'; diff --git a/src/lib/requests/bulkRetryEnrichers.ts b/src/lib/requests/bulkRetryEnrichers.ts index 2485e075..e101d8c1 100644 --- a/src/lib/requests/bulkRetryEnrichers.ts +++ b/src/lib/requests/bulkRetryEnrichers.ts @@ -3,18 +3,18 @@ import { RequestEnricherStatus, RequestStatus, } from '@transcend-io/privacy-types'; -import { map } from '../bluebird-replace'; import cliProgress from 'cli-progress'; import colors from 'colors'; import { difference } from 'lodash-es'; import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequestEnrichers, fetchAllRequests, retryRequestEnricher, } from '../graphql'; -import { logger } from '../../logger'; /** * Restart a bunch of request enrichers diff --git a/src/lib/requests/cancelPrivacyRequests.ts b/src/lib/requests/cancelPrivacyRequests.ts index 9bea901f..8f5f96f6 100644 --- a/src/lib/requests/cancelPrivacyRequests.ts +++ b/src/lib/requests/cancelPrivacyRequests.ts @@ -1,18 +1,18 @@ -import { map } from '../bluebird-replace'; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import { map } from '../bluebird-replace'; import { - UPDATE_PRIVACY_REQUEST, - fetchAllRequests, - makeGraphQLRequest, buildTranscendGraphQLClient, CANCEL_PRIVACY_REQUEST, + fetchAllRequests, fetchAllTemplates, + makeGraphQLRequest, Template, + UPDATE_PRIVACY_REQUEST, } from '../graphql'; -import cliProgress from 'cli-progress'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; /** * Cancel a set of privacy requests diff --git a/src/lib/requests/constants.ts b/src/lib/requests/constants.ts index 5720dbaf..716296da 100644 --- a/src/lib/requests/constants.ts +++ b/src/lib/requests/constants.ts @@ -1,11 +1,11 @@ -import { applyEnum, valuesOf } from '@transcend-io/type-utils'; import { LanguageKey } from '@transcend-io/internationalization'; import { CompletedRequestStatus, - RequestAction, IsoCountryCode, IsoCountrySubdivisionCode, + RequestAction, } from '@transcend-io/privacy-types'; +import { applyEnum, valuesOf } from '@transcend-io/type-utils'; import * as t from 'io-ts'; export const NONE = '[NONE]' as const; diff --git a/src/lib/requests/downloadPrivacyRequestFiles.ts b/src/lib/requests/downloadPrivacyRequestFiles.ts index e133730f..2bd300bf 100644 --- a/src/lib/requests/downloadPrivacyRequestFiles.ts +++ b/src/lib/requests/downloadPrivacyRequestFiles.ts @@ -1,18 +1,18 @@ -import { map } from '../bluebird-replace'; import { existsSync, mkdirSync, writeFileSync } from 'fs'; import { dirname, join } from 'path'; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import { map } from '../bluebird-replace'; import { - fetchAllRequests, + APPROVE_PRIVACY_REQUEST, buildTranscendGraphQLClient, createSombraGotInstance, + fetchAllRequests, makeGraphQLRequest, - APPROVE_PRIVACY_REQUEST, } from '../graphql'; -import cliProgress from 'cli-progress'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; import { getFileMetadataForPrivacyRequests } from './getFileMetadataForPrivacyRequests'; import { streamPrivacyRequestFiles } from './streamPrivacyRequestFiles'; diff --git a/src/lib/requests/filterRows.ts b/src/lib/requests/filterRows.ts index f192888c..a68f68a4 100644 --- a/src/lib/requests/filterRows.ts +++ b/src/lib/requests/filterRows.ts @@ -1,6 +1,6 @@ -import inquirer from 'inquirer'; import { ObjByString } from '@transcend-io/type-utils'; import colors from 'colors'; +import inquirer from 'inquirer'; import { uniq } from 'lodash-es'; import { logger } from '../../logger'; import { NONE } from './constants'; diff --git a/src/lib/requests/fuzzyMatchColumns.ts b/src/lib/requests/fuzzyMatchColumns.ts index 31735676..4673f934 100644 --- a/src/lib/requests/fuzzyMatchColumns.ts +++ b/src/lib/requests/fuzzyMatchColumns.ts @@ -1,7 +1,6 @@ -import inquirer from 'inquirer'; -import { NONE, BULK_APPLY } from './constants'; - import fuzzysearch from 'fuzzysearch'; +import inquirer from 'inquirer'; +import { BULK_APPLY, NONE } from './constants'; /** * Check if word1 and word2 are a fuzzy match of each other. diff --git a/src/lib/requests/getFileMetadataForPrivacyRequests.ts b/src/lib/requests/getFileMetadataForPrivacyRequests.ts index 4c2abf3e..c795e0d8 100644 --- a/src/lib/requests/getFileMetadataForPrivacyRequests.ts +++ b/src/lib/requests/getFileMetadataForPrivacyRequests.ts @@ -1,13 +1,12 @@ -import { map } from '../bluebird-replace'; -import colors from 'colors'; +import { TableEncryptionType } from '@transcend-io/privacy-types'; +import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; import cliProgress from 'cli-progress'; - -import { PrivacyRequest } from '../graphql'; -import * as t from 'io-ts'; +import colors from 'colors'; import type { Got } from 'got'; -import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; +import * as t from 'io-ts'; import { logger } from '../../logger'; -import { TableEncryptionType } from '@transcend-io/privacy-types'; +import { map } from '../bluebird-replace'; +import { PrivacyRequest } from '../graphql'; export const IntlMessage = t.type({ /** The message key */ diff --git a/src/lib/requests/mapColumnsToAttributes.ts b/src/lib/requests/mapColumnsToAttributes.ts index 15e65988..c54293f3 100644 --- a/src/lib/requests/mapColumnsToAttributes.ts +++ b/src/lib/requests/mapColumnsToAttributes.ts @@ -1,9 +1,9 @@ +import type { PersistedState } from '@transcend-io/persisted-state'; import type { GraphQLClient } from 'graphql-request'; import inquirer from 'inquirer'; import { AttributeKey } from '../graphql'; import { CachedFileState } from './constants'; import { fuzzyMatchColumns } from './fuzzyMatchColumns'; -import type { PersistedState } from '@transcend-io/persisted-state'; /** * Mapping from attribute name to request input parameter diff --git a/src/lib/requests/mapColumnsToIdentifiers.ts b/src/lib/requests/mapColumnsToIdentifiers.ts index 6bec8709..e65114ad 100644 --- a/src/lib/requests/mapColumnsToIdentifiers.ts +++ b/src/lib/requests/mapColumnsToIdentifiers.ts @@ -1,9 +1,9 @@ +import type { PersistedState } from '@transcend-io/persisted-state'; import type { GraphQLClient } from 'graphql-request'; import inquirer from 'inquirer'; -import { INITIALIZER, makeGraphQLRequest, Initializer } from '../graphql'; +import { INITIALIZER, Initializer, makeGraphQLRequest } from '../graphql'; import { CachedFileState, IDENTIFIER_BLOCK_LIST } from './constants'; import { fuzzyMatchColumns } from './fuzzyMatchColumns'; -import type { PersistedState } from '@transcend-io/persisted-state'; /** * Mapping from identifier name to request input parameter diff --git a/src/lib/requests/mapCsvColumnsToApi.ts b/src/lib/requests/mapCsvColumnsToApi.ts index e920ec66..54f0e794 100644 --- a/src/lib/requests/mapCsvColumnsToApi.ts +++ b/src/lib/requests/mapCsvColumnsToApi.ts @@ -1,12 +1,12 @@ -import { getValues, getEntries } from '@transcend-io/type-utils'; import type { PersistedState } from '@transcend-io/persisted-state'; +import { getEntries, getValues } from '@transcend-io/type-utils'; import inquirer from 'inquirer'; import { startCase } from 'lodash-es'; import { - ColumnName, CachedFileState, - IS_REQUIRED, CAN_APPLY_IN_BULK, + ColumnName, + IS_REQUIRED, } from './constants'; import { fuzzyMatchColumns } from './fuzzyMatchColumns'; diff --git a/src/lib/requests/mapCsvRowsToRequestInputs.ts b/src/lib/requests/mapCsvRowsToRequestInputs.ts index 9659bd94..d014b40e 100644 --- a/src/lib/requests/mapCsvRowsToRequestInputs.ts +++ b/src/lib/requests/mapCsvRowsToRequestInputs.ts @@ -1,31 +1,29 @@ import { LanguageKey } from '@transcend-io/internationalization'; -import { DateFromISOString } from 'io-ts-types'; - -import * as t from 'io-ts'; import type { PersistedState } from '@transcend-io/persisted-state'; import { - NORMALIZE_PHONE_NUMBER, CompletedRequestStatus, - RequestAction, IdentifierType, IsoCountryCode, IsoCountrySubdivisionCode, + NORMALIZE_PHONE_NUMBER, + RequestAction, } from '@transcend-io/privacy-types'; import { ObjByString, valuesOf } from '@transcend-io/type-utils'; - +import * as t from 'io-ts'; +import { DateFromISOString } from 'io-ts-types'; +import { AttributeKey } from '../graphql'; import { - CachedFileState, BLANK, BULK_APPLY, + CachedFileState, ColumnName, NONE, } from './constants'; -import { AttributeKey } from '../graphql'; -import { ColumnNameMap } from './mapCsvColumnsToApi'; -import { splitCsvToList } from './splitCsvToList'; -import { ParsedAttributeInput } from './parseAttributesFromString'; import { AttributeNameMap } from './mapColumnsToAttributes'; import { IdentifierNameMap } from './mapColumnsToIdentifiers'; +import { ColumnNameMap } from './mapCsvColumnsToApi'; +import { ParsedAttributeInput } from './parseAttributesFromString'; +import { splitCsvToList } from './splitCsvToList'; /** * Shape of additional identifiers @@ -117,8 +115,8 @@ export function normalizeIdentifierValue( return !normalized ? '' : normalized.startsWith('+') - ? normalized - : `+${defaultPhoneCountryCode}${normalized}`; + ? normalized + : `+${defaultPhoneCountryCode}${normalized}`; } return identifierValue; } diff --git a/src/lib/requests/mapEnumValues.ts b/src/lib/requests/mapEnumValues.ts index 2c85aaa9..fe85322a 100644 --- a/src/lib/requests/mapEnumValues.ts +++ b/src/lib/requests/mapEnumValues.ts @@ -1,6 +1,6 @@ +import { apply, ObjByString } from '@transcend-io/type-utils'; import inquirer from 'inquirer'; import autoCompletePrompt from 'inquirer-autocomplete-prompt'; -import { apply, ObjByString } from '@transcend-io/type-utils'; import { fuzzySearch } from './fuzzyMatchColumns'; /** diff --git a/src/lib/requests/mapRequestEnumValues.ts b/src/lib/requests/mapRequestEnumValues.ts index 15c4a07c..6d047eeb 100644 --- a/src/lib/requests/mapRequestEnumValues.ts +++ b/src/lib/requests/mapRequestEnumValues.ts @@ -1,20 +1,20 @@ -import { GraphQLClient } from 'graphql-request'; -import colors from 'colors'; +import { LanguageKey } from '@transcend-io/internationalization'; import type { PersistedState } from '@transcend-io/persisted-state'; import { CompletedRequestStatus, - RequestAction, IsoCountryCode, IsoCountrySubdivisionCode, + RequestAction, } from '@transcend-io/privacy-types'; -import { LanguageKey } from '@transcend-io/internationalization'; import { ObjByString } from '@transcend-io/type-utils'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; import { logger } from '../../logger'; -import { makeGraphQLRequest, DataSubject, DATA_SUBJECTS } from '../graphql'; -import { CachedFileState, NONE, ColumnName } from './constants'; -import { mapEnumValues } from './mapEnumValues'; -import { ColumnNameMap } from './mapCsvColumnsToApi'; +import { DATA_SUBJECTS, DataSubject, makeGraphQLRequest } from '../graphql'; +import { CachedFileState, ColumnName, NONE } from './constants'; import { getUniqueValuesForColumn } from './getUniqueValuesForColumn'; +import { ColumnNameMap } from './mapCsvColumnsToApi'; +import { mapEnumValues } from './mapEnumValues'; /** * Map the values in a CSV to the enum values in Transcend diff --git a/src/lib/requests/markSilentPrivacyRequests.ts b/src/lib/requests/markSilentPrivacyRequests.ts index 88a437ce..7c42193c 100644 --- a/src/lib/requests/markSilentPrivacyRequests.ts +++ b/src/lib/requests/markSilentPrivacyRequests.ts @@ -1,15 +1,15 @@ -import { map } from '../bluebird-replace'; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import { map } from '../bluebird-replace'; import { - UPDATE_PRIVACY_REQUEST, + buildTranscendGraphQLClient, fetchAllRequests, makeGraphQLRequest, - buildTranscendGraphQLClient, + UPDATE_PRIVACY_REQUEST, } from '../graphql'; -import cliProgress from 'cli-progress'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; /** * Mark a set of privacy requests to be in silent mode diff --git a/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts b/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts index 59880ad4..2bb4f4db 100644 --- a/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts +++ b/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts @@ -1,16 +1,16 @@ -import { map } from '../bluebird-replace'; +import { RequestAction } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; -import { RequestAction } from '@transcend-io/privacy-types'; +import { map } from '../bluebird-replace'; import { - NOTIFY_ADDITIONAL_TIME, - fetchAllRequests, - makeGraphQLRequest, buildTranscendGraphQLClient, + fetchAllRequests, fetchAllTemplates, + makeGraphQLRequest, + NOTIFY_ADDITIONAL_TIME, } from '../graphql'; -import cliProgress from 'cli-progress'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; /** * Mark a set of privacy requests to be in silent mode. diff --git a/src/lib/requests/pullPrivacyRequests.ts b/src/lib/requests/pullPrivacyRequests.ts index 33653b04..81a9a2b2 100644 --- a/src/lib/requests/pullPrivacyRequests.ts +++ b/src/lib/requests/pullPrivacyRequests.ts @@ -1,18 +1,17 @@ import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; -import { map } from '../bluebird-replace'; import colors from 'colors'; import { groupBy } from 'lodash-es'; - import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { - PrivacyRequest, - RequestIdentifier, buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequestIdentifiers, fetchAllRequests, + PrivacyRequest, + RequestIdentifier, } from '../graphql'; -import { logger } from '../../logger'; export interface ExportedPrivacyRequest extends PrivacyRequest { /** Request identifiers */ diff --git a/src/lib/requests/readCsv.ts b/src/lib/requests/readCsv.ts index fe366e05..df050570 100644 --- a/src/lib/requests/readCsv.ts +++ b/src/lib/requests/readCsv.ts @@ -1,10 +1,9 @@ +import { readFileSync } from 'fs'; +import { decodeCodec } from '@transcend-io/type-utils'; import type { Options } from 'csv-parse'; import { parse } from 'csv-parse/sync'; -import { readFileSync } from 'fs'; import * as t from 'io-ts'; -import { decodeCodec } from '@transcend-io/type-utils'; - /** * Read in a CSV and validate its shape * diff --git a/src/lib/requests/removeUnverifiedRequestIdentifiers.ts b/src/lib/requests/removeUnverifiedRequestIdentifiers.ts index af4ebb36..a93bedb8 100644 --- a/src/lib/requests/removeUnverifiedRequestIdentifiers.ts +++ b/src/lib/requests/removeUnverifiedRequestIdentifiers.ts @@ -1,16 +1,16 @@ -import { map } from '../bluebird-replace'; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import { map } from '../bluebird-replace'; import { - REMOVE_REQUEST_IDENTIFIERS, - fetchAllRequests, + buildTranscendGraphQLClient, fetchAllRequestIdentifierMetadata, + fetchAllRequests, makeGraphQLRequest, - buildTranscendGraphQLClient, + REMOVE_REQUEST_IDENTIFIERS, } from '../graphql'; -import cliProgress from 'cli-progress'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; /** * Remove a set of unverified request identifier diff --git a/src/lib/requests/restartPrivacyRequest.ts b/src/lib/requests/restartPrivacyRequest.ts index 81e58ca0..1e41f41c 100644 --- a/src/lib/requests/restartPrivacyRequest.ts +++ b/src/lib/requests/restartPrivacyRequest.ts @@ -1,8 +1,8 @@ -import * as t from 'io-ts'; -import { groupBy } from 'lodash-es'; -import { apply, decodeCodec } from '@transcend-io/type-utils'; import { IdentifierType } from '@transcend-io/privacy-types'; +import { apply, decodeCodec } from '@transcend-io/type-utils'; import type { Got } from 'got'; +import * as t from 'io-ts'; +import { groupBy } from 'lodash-es'; import { PrivacyRequest, RequestIdentifier } from '../graphql'; import { IDENTIFIER_BLOCK_LIST } from './constants'; import { PrivacyRequestResponse } from './submitPrivacyRequest'; diff --git a/src/lib/requests/retryRequestDataSilos.ts b/src/lib/requests/retryRequestDataSilos.ts index aade236f..48a5f2d2 100644 --- a/src/lib/requests/retryRequestDataSilos.ts +++ b/src/lib/requests/retryRequestDataSilos.ts @@ -1,16 +1,16 @@ -import { map } from '../bluebird-replace'; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import { map } from '../bluebird-replace'; import { - RETRY_REQUEST_DATA_SILO, - fetchRequestDataSilo, + buildTranscendGraphQLClient, fetchAllRequests, + fetchRequestDataSilo, makeGraphQLRequest, - buildTranscendGraphQLClient, + RETRY_REQUEST_DATA_SILO, } from '../graphql'; -import cliProgress from 'cli-progress'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; /** * Retry a set of RequestDataSilos diff --git a/src/lib/requests/skipPreflightJobs.ts b/src/lib/requests/skipPreflightJobs.ts index e303218a..9d903d08 100644 --- a/src/lib/requests/skipPreflightJobs.ts +++ b/src/lib/requests/skipPreflightJobs.ts @@ -1,19 +1,19 @@ -import { mapSeries, map } from '../bluebird-replace'; +import { + RequestEnricherStatus, + RequestStatus, +} from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; import { - makeGraphQLRequest, buildTranscendGraphQLClient, fetchAllRequestEnrichers, fetchAllRequests, + makeGraphQLRequest, SKIP_REQUEST_ENRICHER, } from '../graphql'; -import cliProgress from 'cli-progress'; -import { - RequestEnricherStatus, - RequestStatus, -} from '@transcend-io/privacy-types'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; /** * Given an enricher ID, mark all open request enrichers as skipped diff --git a/src/lib/requests/skipRequestDataSilos.ts b/src/lib/requests/skipRequestDataSilos.ts index 07544e6a..d9c7b21c 100644 --- a/src/lib/requests/skipRequestDataSilos.ts +++ b/src/lib/requests/skipRequestDataSilos.ts @@ -1,15 +1,15 @@ -import { map } from '../bluebird-replace'; +import { RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { - CHANGE_REQUEST_DATA_SILO_STATUS, - makeGraphQLRequest, buildTranscendGraphQLClient, + CHANGE_REQUEST_DATA_SILO_STATUS, fetchRequestDataSilos, + makeGraphQLRequest, } from '../graphql'; -import cliProgress from 'cli-progress'; -import { RequestStatus } from '@transcend-io/privacy-types'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; /** * Given a data silo ID, mark all open request data silos as skipped diff --git a/src/lib/requests/streamPrivacyRequestFiles.ts b/src/lib/requests/streamPrivacyRequestFiles.ts index 4955c13d..98311bdf 100644 --- a/src/lib/requests/streamPrivacyRequestFiles.ts +++ b/src/lib/requests/streamPrivacyRequestFiles.ts @@ -1,8 +1,8 @@ -import { map } from '../bluebird-replace'; import colors from 'colors'; -import { RequestFileMetadata } from './getFileMetadataForPrivacyRequests'; import type { Got } from 'got'; import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { RequestFileMetadata } from './getFileMetadataForPrivacyRequests'; /** * This function will take in a set of file metadata for privacy requests diff --git a/src/lib/requests/submitPrivacyRequest.ts b/src/lib/requests/submitPrivacyRequest.ts index 90d1a439..4f887d97 100644 --- a/src/lib/requests/submitPrivacyRequest.ts +++ b/src/lib/requests/submitPrivacyRequest.ts @@ -1,13 +1,13 @@ -import * as t from 'io-ts'; -import { uniq } from 'lodash-es'; -import { valuesOf, decodeCodec } from '@transcend-io/type-utils'; import { IsoCountryCode, IsoCountrySubdivisionCode, RequestAction, RequestStatus, } from '@transcend-io/privacy-types'; +import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; import type { Got } from 'got'; +import * as t from 'io-ts'; +import { uniq } from 'lodash-es'; import { PrivacyRequestInput } from './mapCsvRowsToRequestInputs'; import { ParsedAttributeInput } from './parseAttributesFromString'; @@ -110,8 +110,8 @@ export async function submitPrivacyRequest( country: input.country, } : input.countrySubDivision - ? { country: input.countrySubDivision.split('-')[0] } - : {}), + ? { country: input.countrySubDivision.split('-')[0] } + : {}), ...(input.countrySubDivision ? { countrySubDivision: input.countrySubDivision } : {}), diff --git a/src/lib/requests/tests/fuzzyMatchColumns.test.ts b/src/lib/requests/tests/fuzzyMatchColumns.test.ts index 48e7edb1..8c4a8942 100644 --- a/src/lib/requests/tests/fuzzyMatchColumns.test.ts +++ b/src/lib/requests/tests/fuzzyMatchColumns.test.ts @@ -1,6 +1,5 @@ -import { expect, describe, it } from 'vitest'; import inquirer from 'inquirer'; - +import { describe, expect, it } from 'vitest'; import { fuzzyMatchColumns, fuzzySearch, NONE } from '../index'; describe('fuzzyMatchColumns', () => { diff --git a/src/lib/requests/tests/getUniqueValuesForColumn.test.ts b/src/lib/requests/tests/getUniqueValuesForColumn.test.ts index 3c9974d0..7992d1e4 100644 --- a/src/lib/requests/tests/getUniqueValuesForColumn.test.ts +++ b/src/lib/requests/tests/getUniqueValuesForColumn.test.ts @@ -1,5 +1,4 @@ -import { expect, describe, it } from 'vitest'; - +import { describe, expect, it } from 'vitest'; import { getUniqueValuesForColumn } from '../index'; describe('getUniqueValuesForColumn', () => { diff --git a/src/lib/requests/tests/parseAttributesFromString.test.ts b/src/lib/requests/tests/parseAttributesFromString.test.ts index 523af9d0..a1a2466c 100644 --- a/src/lib/requests/tests/parseAttributesFromString.test.ts +++ b/src/lib/requests/tests/parseAttributesFromString.test.ts @@ -1,5 +1,4 @@ -import { expect, describe, it } from 'vitest'; - +import { describe, expect, it } from 'vitest'; import { parseAttributesFromString } from '../index'; describe('parseAttributesFromString', () => { diff --git a/src/lib/requests/tests/readCsv.test.ts b/src/lib/requests/tests/readCsv.test.ts index 03d40299..b8c49127 100644 --- a/src/lib/requests/tests/readCsv.test.ts +++ b/src/lib/requests/tests/readCsv.test.ts @@ -1,7 +1,6 @@ -import { expect, describe, it } from 'vitest'; import { join } from 'path'; import * as t from 'io-ts'; - +import { describe, expect, it } from 'vitest'; import { readCsv } from '../index'; describe('readCsv', () => { diff --git a/src/lib/requests/tests/splitCsvToList.test.ts b/src/lib/requests/tests/splitCsvToList.test.ts index 72ac8aff..3387d401 100644 --- a/src/lib/requests/tests/splitCsvToList.test.ts +++ b/src/lib/requests/tests/splitCsvToList.test.ts @@ -1,5 +1,4 @@ -import { expect, describe, it } from 'vitest'; - +import { describe, expect, it } from 'vitest'; import { splitCsvToList } from '../index'; describe('splitCsvToList', () => { diff --git a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts index fdd4e5f0..e5b7216e 100644 --- a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts +++ b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts @@ -1,29 +1,29 @@ /* eslint-disable max-lines */ +import { join } from 'path'; +import { PersistedState } from '@transcend-io/persisted-state'; +import cliProgress from 'cli-progress'; import colors from 'colors'; -import { map } from '../bluebird-replace'; import * as t from 'io-ts'; import { uniq } from 'lodash-es'; -import cliProgress from 'cli-progress'; -import { join } from 'path'; -import { PersistedState } from '@transcend-io/persisted-state'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { - createSombraGotInstance, buildTranscendGraphQLClient, + createSombraGotInstance, fetchAllRequestAttributeKeys, } from '../graphql'; -import { mapRequestEnumValues } from './mapRequestEnumValues'; -import { CachedRequestState, CachedFileState } from './constants'; +import { CachedFileState, CachedRequestState } from './constants'; +import { extractClientError } from './extractClientError'; +import { filterRows } from './filterRows'; +import { mapColumnsToAttributes } from './mapColumnsToAttributes'; +import { mapColumnsToIdentifiers } from './mapColumnsToIdentifiers'; import { mapCsvColumnsToApi } from './mapCsvColumnsToApi'; +import { mapCsvRowsToRequestInputs } from './mapCsvRowsToRequestInputs'; +import { mapRequestEnumValues } from './mapRequestEnumValues'; import { parseAttributesFromString } from './parseAttributesFromString'; import { readCsv } from './readCsv'; import { submitPrivacyRequest } from './submitPrivacyRequest'; -import { mapColumnsToAttributes } from './mapColumnsToAttributes'; -import { mapColumnsToIdentifiers } from './mapColumnsToIdentifiers'; -import { mapCsvRowsToRequestInputs } from './mapCsvRowsToRequestInputs'; -import { filterRows } from './filterRows'; -import { extractClientError } from './extractClientError'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; /** * Upload a set of privacy requests from CSV diff --git a/src/lib/tests/TranscendPromptManager.test.ts b/src/lib/tests/TranscendPromptManager.test.ts index 1344ea54..6602618d 100644 --- a/src/lib/tests/TranscendPromptManager.test.ts +++ b/src/lib/tests/TranscendPromptManager.test.ts @@ -1,5 +1,5 @@ -import { expect, describe, it } from 'vitest'; import * as t from 'io-ts'; +import { describe, expect, it } from 'vitest'; import { TranscendPromptManager } from '../ai/TranscendPromptManager'; describe('TranscendPromptManager', () => { diff --git a/src/lib/tests/filterNullValuesFromObject.test.ts b/src/lib/tests/filterNullValuesFromObject.test.ts index e798be5d..d8a43282 100644 --- a/src/lib/tests/filterNullValuesFromObject.test.ts +++ b/src/lib/tests/filterNullValuesFromObject.test.ts @@ -1,5 +1,4 @@ -import { expect, describe, it } from 'vitest'; - +import { describe, expect, it } from 'vitest'; import { filterNullishValuesFromObject } from '../ai/filterNullishValuesFromObject'; const TEST_DATA = { diff --git a/src/lib/tests/findCodePackagesInFolder.test.ts b/src/lib/tests/findCodePackagesInFolder.test.ts index 471f78f6..59b05dcc 100644 --- a/src/lib/tests/findCodePackagesInFolder.test.ts +++ b/src/lib/tests/findCodePackagesInFolder.test.ts @@ -1,9 +1,8 @@ /* eslint-disable max-lines */ -import { expect, describe, it } from 'vitest'; - -import { findCodePackagesInFolder } from '../code-scanning/findCodePackagesInFolder'; import { join } from 'path'; +import { describe, expect, it } from 'vitest'; import type { CodePackageInput } from '../../codecs'; +import { findCodePackagesInFolder } from '../code-scanning/findCodePackagesInFolder'; const expected: CodePackageInput[] = [ { diff --git a/src/lib/tests/getGitFilesThatChanged.test.ts b/src/lib/tests/getGitFilesThatChanged.test.ts index fab21808..942e61de 100644 --- a/src/lib/tests/getGitFilesThatChanged.test.ts +++ b/src/lib/tests/getGitFilesThatChanged.test.ts @@ -1,7 +1,6 @@ -import { expect, describe, it } from 'vitest'; - -import { getGitFilesThatChanged } from '../ai/getGitFilesThatChanged'; import { join } from 'path'; +import { describe, expect, it } from 'vitest'; +import { getGitFilesThatChanged } from '../ai/getGitFilesThatChanged'; // not easy to test this but can uncomment to run against current commit describe.skip('getGitFilesThatChanged', () => { diff --git a/src/lib/tests/mergeTranscendInputs.test.ts b/src/lib/tests/mergeTranscendInputs.test.ts index 7932cd55..52244b4a 100644 --- a/src/lib/tests/mergeTranscendInputs.test.ts +++ b/src/lib/tests/mergeTranscendInputs.test.ts @@ -1,5 +1,4 @@ -import { expect, describe, it } from 'vitest'; - +import { describe, expect, it } from 'vitest'; import { mergeTranscendInputs, TranscendInput } from '../../index'; describe('mergeTranscendInputs', () => { diff --git a/src/lib/tests/readTranscendYaml.test.ts b/src/lib/tests/readTranscendYaml.test.ts index ffe705c4..7de70005 100644 --- a/src/lib/tests/readTranscendYaml.test.ts +++ b/src/lib/tests/readTranscendYaml.test.ts @@ -1,6 +1,5 @@ -import { expect, describe, it } from 'vitest'; import { join } from 'path'; - +import { describe, expect, it } from 'vitest'; import { readTranscendYaml } from '../../index'; const EXAMPLE_DIR = join(__dirname, '..', '..', '..', 'examples'); diff --git a/src/lib/tests/removeLinks.test.ts b/src/lib/tests/removeLinks.test.ts index 71f80393..63375f1c 100644 --- a/src/lib/tests/removeLinks.test.ts +++ b/src/lib/tests/removeLinks.test.ts @@ -1,5 +1,4 @@ -import { expect, describe, it } from 'vitest'; - +import { describe, expect, it } from 'vitest'; import { removeLinks } from '../ai/removeLinks'; const TEST_DATA = ` diff --git a/vitest.config.ts b/vitest.config.ts index 4a62211c..9ecfec02 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,5 +1,5 @@ -import { defineConfig } from 'vitest/config'; import tsconfigPaths from 'vite-tsconfig-paths'; +import { defineConfig } from 'vitest/config'; export default defineConfig({ plugins: [tsconfigPaths()], From d9765862d26a4d5916a5882b25090bf0b389b6cf Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 20:49:33 -0400 Subject: [PATCH 2/4] eslint fix --- README.md | 296 ++++----- scripts/buildPathfinderJsonSchema.ts | 18 +- scripts/buildTranscendJsonSchema.ts | 22 +- src/codecs.ts | 246 +++---- src/commands/admin/generate-api-keys/impl.ts | 28 +- .../consent/build-xdi-sync-endpoint/impl.ts | 20 +- .../consent/pull-consent-metrics/impl.ts | 98 +-- .../consent/upload-preferences/impl.ts | 58 +- .../impl.ts | 54 +- .../impl.ts | 30 +- .../impl.ts | 126 ++-- .../derive-data-silos-from-data-flows/impl.ts | 44 +- .../inventory/pull-datapoints/impl.ts | 67 +- .../pull-unstructured-discovery-files/impl.ts | 52 +- src/commands/inventory/pull/impl.ts | 68 +- src/commands/inventory/push/impl.ts | 110 ++-- src/commands/inventory/scan-packages/impl.ts | 40 +- src/commands/migration/sync-ot/impl.ts | 49 +- .../request/cron/pull-identifiers/impl.ts | 32 +- .../request/cron/pull-profiles/impl.ts | 62 +- src/commands/request/export/impl.ts | 26 +- src/constants.ts | 92 +-- src/lib/ai/TranscendPromptManager.ts | 194 +++--- src/lib/ai/filterNullishValuesFromObject.ts | 18 +- src/lib/ai/getGitFilesThatChanged.ts | 36 +- src/lib/ai/removeLinks.ts | 2 +- .../api-keys/generateCrossAccountApiKeys.ts | 72 +-- src/lib/api-keys/listDirectories.ts | 6 +- src/lib/api-keys/listFiles.ts | 12 +- src/lib/api-keys/validateTranscendAuth.ts | 18 +- src/lib/bluebird-replace.ts | 16 +- src/lib/code-scanning/constants.ts | 17 +- .../code-scanning/findCodePackagesInFolder.ts | 71 +- src/lib/code-scanning/findFilesToScan.ts | 36 +- .../code-scanning/integrations/cocoaPods.ts | 53 +- .../integrations/composerJson.ts | 28 +- src/lib/code-scanning/integrations/gemfile.ts | 48 +- src/lib/code-scanning/integrations/gradle.ts | 70 +- .../integrations/javascriptPackageJson.ts | 28 +- src/lib/code-scanning/integrations/pubspec.ts | 70 +- .../integrations/pythonRequirementsTxt.ts | 36 +- src/lib/code-scanning/integrations/swift.ts | 20 +- .../consent-manager/buildXdiSyncEndpoint.ts | 43 +- .../consentManagersToBusinessEntities.ts | 30 +- src/lib/consent-manager/createConsentToken.ts | 18 +- .../consent-manager/dataFlowsToDataSilos.ts | 48 +- src/lib/consent-manager/uploadConsents.ts | 82 +-- .../consent-manager/uploadCookiesFromCsv.ts | 52 +- .../consent-manager/uploadDataFlowsFromCsv.ts | 50 +- src/lib/cron/markCronIdentifierCompleted.ts | 18 +- .../cron/markRequestDataSiloIdsCompleted.ts | 36 +- ...ChunkedCustomSiloOutstandingIdentifiers.ts | 63 +- src/lib/cron/pullCronPageOfIdentifiers.ts | 20 +- .../pullCustomSiloOutstandingIdentifiers.ts | 63 +- src/lib/cron/pushCronIdentifiersFromCsv.ts | 56 +- src/lib/cron/writeCsv.ts | 52 +- src/lib/data-inventory/pullAllDatapoints.ts | 110 ++-- ...UnstructuredSubDataPointRecommendations.ts | 48 +- src/lib/graphql/fetchAllAssessments.ts | 12 +- src/lib/graphql/fetchAllRequestIdentifiers.ts | 36 +- src/lib/graphql/fetchAllRequests.ts | 42 +- src/lib/graphql/fetchApiKeys.ts | 46 +- src/lib/graphql/fetchCatalogs.ts | 34 +- src/lib/graphql/fetchDataSubjects.ts | 52 +- src/lib/graphql/fetchIdentifiers.ts | 50 +- src/lib/graphql/fetchPrompts.ts | 32 +- src/lib/graphql/fetchRequestDataSilo.ts | 30 +- src/lib/graphql/formatAttributeValues.ts | 16 +- src/lib/graphql/gqls/consentManager.ts | 4 +- src/lib/graphql/loginUser.ts | 16 +- src/lib/graphql/makeGraphQLRequest.ts | 57 +- src/lib/graphql/pullTranscendConfiguration.ts | 453 +++++++------ src/lib/graphql/syncActionItemCollections.ts | 61 +- src/lib/graphql/syncActionItems.ts | 88 ++- src/lib/graphql/syncAgentFiles.ts | 51 +- src/lib/graphql/syncAgentFunctions.ts | 55 +- src/lib/graphql/syncAgents.ts | 49 +- src/lib/graphql/syncAttribute.ts | 52 +- src/lib/graphql/syncBusinessEntities.ts | 54 +- src/lib/graphql/syncCodePackages.ts | 104 +-- .../graphql/syncConfigurationToTranscend.ts | 239 ++++--- src/lib/graphql/syncConsentManager.ts | 64 +- src/lib/graphql/syncCookies.ts | 32 +- src/lib/graphql/syncDataCategories.ts | 57 +- src/lib/graphql/syncDataFlows.ts | 58 +- src/lib/graphql/syncDataSilos.ts | 282 ++++---- src/lib/graphql/syncDataSubject.ts | 14 +- src/lib/graphql/syncEnrichers.ts | 46 +- src/lib/graphql/syncIdentifier.ts | 14 +- src/lib/graphql/syncIntlMessages.ts | 42 +- src/lib/graphql/syncPartitions.ts | 32 +- src/lib/graphql/syncPolicies.ts | 40 +- src/lib/graphql/syncPrivacyCenter.ts | 24 +- src/lib/graphql/syncProcessingPurposes.ts | 67 +- src/lib/graphql/syncPromptGroups.ts | 70 +- src/lib/graphql/syncPromptPartials.ts | 62 +- src/lib/graphql/syncPrompts.ts | 46 +- src/lib/graphql/syncRepositories.ts | 58 +- .../graphql/syncSoftwareDevelopmentKits.ts | 78 +-- src/lib/graphql/syncTeams.ts | 54 +- src/lib/graphql/syncVendors.ts | 48 +- src/lib/helpers/inquirer.ts | 30 +- src/lib/helpers/parseVariablesFromString.ts | 22 +- .../manual-enrichment/enrichPrivacyRequest.ts | 73 ++- .../pullManualEnrichmentIdentifiersToCsv.ts | 56 +- .../pushManualEnrichmentIdentifiersFromCsv.ts | 28 +- src/lib/mergeTranscendInputs.ts | 8 +- .../oneTrust/helpers/convertToEmptyStrings.ts | 17 +- .../helpers/parseCliSyncOtArguments.ts | 76 ++- .../helpers/syncOneTrustAssessmentToDisk.ts | 18 +- .../syncOneTrustAssessmentToTranscend.ts | 28 +- .../syncOneTrustAssessmentsFromFile.ts | 40 +- .../syncOneTrustAssessmentsFromOneTrust.ts | 75 +-- .../tests/convertToEmptyStrings.test.ts | 52 +- .../checkIfPendingPreferenceUpdatesAreNoOp.ts | 100 +-- ...IfPendingPreferenceUpdatesCauseConflict.ts | 40 +- .../getPreferenceUpdatesFromRow.ts | 267 ++++---- .../getPreferencesForIdentifiers.ts | 50 +- .../parsePreferenceAndPurposeValuesFromCsv.ts | 79 ++- .../parsePreferenceIdentifiersFromCsv.ts | 67 +- .../parsePreferenceManagementCsv.ts | 72 +-- .../parsePreferenceTimestampsFromCsv.ts | 39 +- ...kIfPendingPreferenceUpdatesAreNoOp.test.ts | 242 ++++--- ...dingPreferenceUpdatesCauseConflict.test.ts | 248 ++++--- .../tests/getPreferenceUpdatesFromRow.test.ts | 526 ++++++++------- ...ferenceManagementPreferencesInteractive.ts | 116 ++-- src/lib/readTranscendYaml.ts | 26 +- src/lib/requests/approvePrivacyRequests.ts | 30 +- src/lib/requests/bulkRestartRequests.ts | 88 +-- src/lib/requests/bulkRetryEnrichers.ts | 40 +- src/lib/requests/cancelPrivacyRequests.ts | 38 +- src/lib/requests/constants.ts | 48 +- .../requests/downloadPrivacyRequestFiles.ts | 38 +- src/lib/requests/extractClientError.ts | 4 +- src/lib/requests/filterRows.ts | 30 +- .../getFileMetadataForPrivacyRequests.ts | 51 +- src/lib/requests/getUniqueValuesForColumn.ts | 8 +- src/lib/requests/mapColumnsToAttributes.ts | 34 +- src/lib/requests/mapColumnsToIdentifiers.ts | 36 +- src/lib/requests/mapCsvColumnsToApi.ts | 36 +- src/lib/requests/mapCsvRowsToRequestInputs.ts | 189 +++--- src/lib/requests/mapEnumValues.ts | 34 +- src/lib/requests/mapRequestEnumValues.ts | 97 ++- src/lib/requests/markSilentPrivacyRequests.ts | 28 +- .../notifyPrivacyRequestsAdditionalTime.ts | 38 +- src/lib/requests/pullPrivacyRequests.ts | 72 +-- src/lib/requests/readCsv.ts | 24 +- .../removeUnverifiedRequestIdentifiers.ts | 32 +- src/lib/requests/retryRequestDataSilos.ts | 36 +- src/lib/requests/skipPreflightJobs.ts | 42 +- src/lib/requests/skipRequestDataSilos.ts | 40 +- src/lib/requests/splitCsvToList.ts | 4 +- src/lib/requests/streamPrivacyRequestFiles.ts | 36 +- src/lib/requests/submitPrivacyRequest.ts | 40 +- .../tests/mapCsvRowsToRequestInputs.test.ts | 4 +- src/lib/requests/tests/readCsv.test.ts | 72 +-- .../requests/uploadPrivacyRequestsFromCsv.ts | 144 ++--- .../tests/findCodePackagesInFolder.test.ts | 606 +++++++++--------- src/lib/tests/getGitFilesThatChanged.test.ts | 26 +- src/lib/tests/readTranscendYaml.test.ts | 44 +- 160 files changed, 5250 insertions(+), 5333 deletions(-) diff --git a/README.md b/README.md index 30de3733..925906b0 100644 --- a/README.md +++ b/README.md @@ -10,52 +10,52 @@ - [Installation](#installation) - [transcend.yml](#transcendyml) - [Usage](#usage) - - [`transcend request approve`](#transcend-request-approve) - - [`transcend request upload`](#transcend-request-upload) - - [`transcend request download-files`](#transcend-request-download-files) - - [`transcend request cancel`](#transcend-request-cancel) - - [`transcend request restart`](#transcend-request-restart) - - [`transcend request notify-additional-time`](#transcend-request-notify-additional-time) - - [`transcend request mark-silent`](#transcend-request-mark-silent) - - [`transcend request enricher-restart`](#transcend-request-enricher-restart) - - [`transcend request reject-unverified-identifiers`](#transcend-request-reject-unverified-identifiers) - - [`transcend request export`](#transcend-request-export) - - [`transcend request skip-preflight-jobs`](#transcend-request-skip-preflight-jobs) - - [`transcend request system mark-request-data-silos-completed`](#transcend-request-system-mark-request-data-silos-completed) - - [`transcend request system retry-request-data-silos`](#transcend-request-system-retry-request-data-silos) - - [`transcend request system skip-request-data-silos`](#transcend-request-system-skip-request-data-silos) - - [`transcend request preflight pull-identifiers`](#transcend-request-preflight-pull-identifiers) - - [`transcend request preflight push-identifiers`](#transcend-request-preflight-push-identifiers) - - [`transcend request cron pull-identifiers`](#transcend-request-cron-pull-identifiers) - - [`transcend request cron mark-identifiers-completed`](#transcend-request-cron-mark-identifiers-completed) - - [`transcend consent build-xdi-sync-endpoint`](#transcend-consent-build-xdi-sync-endpoint) - - [`transcend consent pull-consent-metrics`](#transcend-consent-pull-consent-metrics) - - [`transcend consent pull-consent-preferences`](#transcend-consent-pull-consent-preferences) - - [`transcend consent update-consent-manager`](#transcend-consent-update-consent-manager) - - [`transcend consent upload-consent-preferences`](#transcend-consent-upload-consent-preferences) - - [`transcend consent upload-cookies-from-csv`](#transcend-consent-upload-cookies-from-csv) - - [`transcend consent upload-data-flows-from-csv`](#transcend-consent-upload-data-flows-from-csv) - - [`transcend consent upload-preferences`](#transcend-consent-upload-preferences) - - [`transcend inventory pull`](#transcend-inventory-pull) - - [Scopes](#scopes) - - [Usage](#usage-1) - - [`transcend inventory push`](#transcend-inventory-push) - - [Scopes](#scopes-1) - - [Usage](#usage-2) - - [CI Integration](#ci-integration) - - [Dynamic Variables](#dynamic-variables) - - [`transcend inventory scan-packages`](#transcend-inventory-scan-packages) - - [`transcend inventory discover-silos`](#transcend-inventory-discover-silos) - - [Usage](#usage-3) - - [`transcend inventory pull-datapoints`](#transcend-inventory-pull-datapoints) - - [`transcend inventory pull-unstructured-discovery-files`](#transcend-inventory-pull-unstructured-discovery-files) - - [`transcend inventory derive-data-silos-from-data-flows`](#transcend-inventory-derive-data-silos-from-data-flows) - - [`transcend inventory derive-data-silos-from-data-flows-cross-instance`](#transcend-inventory-derive-data-silos-from-data-flows-cross-instance) - - [`transcend inventory consent-manager-service-json-to-yml`](#transcend-inventory-consent-manager-service-json-to-yml) - - [`transcend inventory consent-managers-to-business-entities`](#transcend-inventory-consent-managers-to-business-entities) - - [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys) - - [Usage](#usage-4) - - [`transcend migration sync-ot`](#transcend-migration-sync-ot) + - [`transcend request approve`](#transcend-request-approve) + - [`transcend request upload`](#transcend-request-upload) + - [`transcend request download-files`](#transcend-request-download-files) + - [`transcend request cancel`](#transcend-request-cancel) + - [`transcend request restart`](#transcend-request-restart) + - [`transcend request notify-additional-time`](#transcend-request-notify-additional-time) + - [`transcend request mark-silent`](#transcend-request-mark-silent) + - [`transcend request enricher-restart`](#transcend-request-enricher-restart) + - [`transcend request reject-unverified-identifiers`](#transcend-request-reject-unverified-identifiers) + - [`transcend request export`](#transcend-request-export) + - [`transcend request skip-preflight-jobs`](#transcend-request-skip-preflight-jobs) + - [`transcend request system mark-request-data-silos-completed`](#transcend-request-system-mark-request-data-silos-completed) + - [`transcend request system retry-request-data-silos`](#transcend-request-system-retry-request-data-silos) + - [`transcend request system skip-request-data-silos`](#transcend-request-system-skip-request-data-silos) + - [`transcend request preflight pull-identifiers`](#transcend-request-preflight-pull-identifiers) + - [`transcend request preflight push-identifiers`](#transcend-request-preflight-push-identifiers) + - [`transcend request cron pull-identifiers`](#transcend-request-cron-pull-identifiers) + - [`transcend request cron mark-identifiers-completed`](#transcend-request-cron-mark-identifiers-completed) + - [`transcend consent build-xdi-sync-endpoint`](#transcend-consent-build-xdi-sync-endpoint) + - [`transcend consent pull-consent-metrics`](#transcend-consent-pull-consent-metrics) + - [`transcend consent pull-consent-preferences`](#transcend-consent-pull-consent-preferences) + - [`transcend consent update-consent-manager`](#transcend-consent-update-consent-manager) + - [`transcend consent upload-consent-preferences`](#transcend-consent-upload-consent-preferences) + - [`transcend consent upload-cookies-from-csv`](#transcend-consent-upload-cookies-from-csv) + - [`transcend consent upload-data-flows-from-csv`](#transcend-consent-upload-data-flows-from-csv) + - [`transcend consent upload-preferences`](#transcend-consent-upload-preferences) + - [`transcend inventory pull`](#transcend-inventory-pull) + - [Scopes](#scopes) + - [Usage](#usage-1) + - [`transcend inventory push`](#transcend-inventory-push) + - [Scopes](#scopes-1) + - [Usage](#usage-2) + - [CI Integration](#ci-integration) + - [Dynamic Variables](#dynamic-variables) + - [`transcend inventory scan-packages`](#transcend-inventory-scan-packages) + - [`transcend inventory discover-silos`](#transcend-inventory-discover-silos) + - [Usage](#usage-3) + - [`transcend inventory pull-datapoints`](#transcend-inventory-pull-datapoints) + - [`transcend inventory pull-unstructured-discovery-files`](#transcend-inventory-pull-unstructured-discovery-files) + - [`transcend inventory derive-data-silos-from-data-flows`](#transcend-inventory-derive-data-silos-from-data-flows) + - [`transcend inventory derive-data-silos-from-data-flows-cross-instance`](#transcend-inventory-derive-data-silos-from-data-flows-cross-instance) + - [`transcend inventory consent-manager-service-json-to-yml`](#transcend-inventory-consent-manager-service-json-to-yml) + - [`transcend inventory consent-managers-to-business-entities`](#transcend-inventory-consent-managers-to-business-entities) + - [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys) + - [Usage](#usage-4) + - [`transcend migration sync-ot`](#transcend-migration-sync-ot) - [Prompt Manager](#prompt-manager) - [Proxy usage](#proxy-usage) @@ -190,7 +190,6 @@ data-silos: ## Usage - ### `transcend request approve` ```txt @@ -211,7 +210,6 @@ FLAGS [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] -h --help Print help information and exit ``` - ### `transcend request upload` ```txt @@ -244,7 +242,6 @@ FLAGS [--defaultPhoneCountryCode] When uploading phone numbers, if the phone number is missing a country code, assume this country code [default = 1] -h --help Print help information and exit ``` - ### `transcend request download-files` ```txt @@ -267,7 +264,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend request cancel` ```txt @@ -290,7 +286,6 @@ FLAGS [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] -h --help Print help information and exit ``` - ### `transcend request restart` ```txt @@ -319,7 +314,6 @@ FLAGS [--skipWaitingPeriod] Skip queued state of request and go straight to compiling [default = false] -h --help Print help information and exit ``` - ### `transcend request notify-additional-time` ```txt @@ -342,7 +336,6 @@ FLAGS [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] -h --help Print help information and exit ``` - ### `transcend request mark-silent` ```txt @@ -363,7 +356,6 @@ FLAGS [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] -h --help Print help information and exit ``` - ### `transcend request enricher-restart` ```txt @@ -388,7 +380,6 @@ FLAGS [--createdAtAfter] Restart requests that were submitted after this time -h --help Print help information and exit ``` - ### `transcend request reject-unverified-identifiers` ```txt @@ -405,7 +396,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend request export` ```txt @@ -429,7 +419,6 @@ FLAGS [--pageLimit] The page limit to use when pulling in pages of requests [default = 100] -h --help Print help information and exit ``` - ### `transcend request skip-preflight-jobs` ```txt @@ -445,7 +434,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend request system mark-request-data-silos-completed` ```txt @@ -463,7 +451,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend request system retry-request-data-silos` ```txt @@ -480,7 +467,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend request system skip-request-data-silos` ```txt @@ -498,7 +484,6 @@ FLAGS [--status] The status to set the request data silo job to [SKIPPED|RESOLVED, default = SKIPPED] -h --help Print help information and exit ``` - ### `transcend request preflight pull-identifiers` ```txt @@ -525,7 +510,6 @@ FLAGS [--concurrency] The concurrency to use when uploading requests in parallel [default = 100] -h --help Print help information and exit ``` - ### `transcend request preflight push-identifiers` ```txt @@ -553,7 +537,6 @@ FLAGS [--concurrency] The concurrency to use when uploading requests in parallel [default = 100] -h --help Print help information and exit ``` - ### `transcend request cron pull-identifiers` ```txt @@ -579,7 +562,6 @@ FLAGS [--chunkSize] Maximum number of rows per CSV file. For large datasets, the output will be automatically split into multiple files to avoid file system size limits. Each file will contain at most this many rows [default = 10000] -h --help Print help information and exit ``` - ### `transcend request cron mark-identifiers-completed` ```txt @@ -606,7 +588,6 @@ FLAGS [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra -h --help Print help information and exit ``` - ### `transcend consent build-xdi-sync-endpoint` ```txt @@ -626,7 +607,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend consent pull-consent-metrics` ```txt @@ -653,7 +633,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend consent pull-consent-preferences` ```txt @@ -684,7 +663,7 @@ Each row in the CSV will include: | timestamp | Timestamp for when consent was collected for that user | string - timestamp | N/A | true | | purposes | JSON map from consent purpose name -> boolean indicating whether user has opted in or out of that purpose | {[k in string]: boolean } | {} | true | | airgapVersion | Version of airgap where consent was collected | string | N/A | false | -| [purpose name] | Each consent purpose from `purposes` is also included in a column | boolean | N/A | false | +| [purpose name] | Each consent purpose from `purposes` is also included in a column | boolean | N/A | false | | tcf | IAB TCF string | string - TCF | N/A | false | | gpp | IAB GPP string | string - GPP | N/A | false | @@ -704,7 +683,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend consent upload-consent-preferences` ```txt @@ -737,7 +715,6 @@ Each row in the CSV must include: | gpp | IAB GPP string | string - GPP | N/A | false | An sample CSV can be found [here](./examples/preference-upload.csv). - ### `transcend consent upload-cookies-from-csv` ```txt @@ -760,7 +737,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend consent upload-data-flows-from-csv` ```txt @@ -784,7 +760,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend consent upload-preferences` ```txt @@ -837,7 +812,6 @@ Specifying the backend URL, needed for US hosted backend infrastructure: ```sh transcend consent upload-preferences --auth=$TRANSCEND_API_KEY --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 --consentUrl=https://consent.us.transcend.io ``` - ### `transcend inventory pull` ```txt @@ -874,39 +848,39 @@ FLAGS The API key permissions for this command vary based on the `resources` argument: -| Resource | Description | Scopes | Link | -| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| apiKeys | API Key definitions assigned to Data Silos. API keys cannot be created through the CLI, but you can map API key usage to Data Silos. | View API Keys | [Developer Tools -> API keys](https://app.transcend.io/infrastructure/api-keys) | -| customFields | Custom field definitions that define extra metadata for each table in the Admin Dashboard. | View Global Attributes | [Custom Fields](https://app.transcend.io/infrastructure/attributes) | -| templates | Email templates. Only template titles can be created and mapped to other resources. | View Email Templates | [DSR Automation -> Email Templates](https://app.transcend.io/privacy-requests/email-templates) | -| dataSilos | The Data Silo/Integration definitions. | View Data Map, View Data Subject Request Settings | [Data Inventory -> Data Silos](https://app.transcend.io/data-map/data-inventory/) and [Infrastucture -> Integrations](https://app.transcend.io/infrastructure/integrationsdata-silos) | -| enrichers | The Privacy Request enricher configurations. | View Identity Verification Settings | [DSR Automation -> Identifiers](https://app.transcend.io/privacy-requests/identifiers) | -| dataFlows | Consent Manager Data Flow definitions. | View Data Flows | [Consent Management -> Data Flows](https://app.transcend.io/consent-manager/data-flows/approved) | -| businessEntities | The business entities in the data inventory. | View Data Inventory | [Data Inventory -> Business Entities](https://app.transcend.io/data-map/data-inventory/business-entities) | -| actions | The Privacy Request action settings. | View Data Subject Request Settings | [DSR Automation -> Request Settings](https://app.transcend.io/privacy-requests/settings) | -| dataSubjects | The Privacy Request data subject settings. | View Data Subject Request Settings | [DSR Automation -> Request Settings](https://app.transcend.io/privacy-requests/settings) | -| identifiers | The Privacy Request identifier configurations. | View Identity Verification Settings | [DSR Automation -> Identifiers](https://app.transcend.io/privacy-requests/identifiers) | -| cookies | Consent Manager Cookie definitions. | View Data Flows | [Consent Management -> Cookies](https://app.transcend.io/consent-manager/cookies/approved) | -| consentManager | Consent Manager general settings, including domain list. | View Consent Manager | [Consent Management -> Developer Settings](https://app.transcend.io/consent-manager/developer-settings) | -| partitions | The partitions in the account (often representative of separate data controllers). | View Consent Manager | [Consent Management -> Developer Settings -> Advanced Settings](https://app.transcend.io/consent-manager/developer-settings/advanced-settings) | -| prompts | The Transcend AI prompts | View Prompts | [Prompt Manager -> Browse](https://app.transcend.io/prompts/browse) | -| promptPartials | The Transcend AI prompt partials | View Prompts | [Prompt Manager -> Partials](https://app.transcend.io/prompts/partialss) | -| promptGroups | The Transcend AI prompt groups | View Prompts | [Prompt Manager -> Groups](https://app.transcend.io/prompts/groups) | -| agents | The agents in the prompt manager. | View Prompts | [Prompt Manager -> Agents](https://app.transcend.io/prompts/agents) | -| agentFunctions | The agent functions in the prompt manager. | View Prompts | [Prompt Manager -> Agent Functions](https://app.transcend.io/prompts/agent-functions) | -| agentFiles | The agent files in the prompt manager. | View Prompts | [Prompt Manager -> Agent Files](https://app.transcend.io/prompts/agent-files) | -| vendors | The vendors in the data inventory. | View Data Inventory | [Data Inventory -> Vendors](https://app.transcend.io/data-map/data-inventory/vendors) | -| dataCategories | The data categories in the data inventory. | View Data Inventory | [Data Inventory -> Data Categories](https://app.transcend.io/data-map/data-inventory/data-categories) | -| processingPurposes | The processing purposes in the data inventory. | View Data Inventory | [Data Inventory -> Processing Purposes](https://app.transcend.io/data-map/data-inventory/purposes) | -| actionItems | Onboarding related action items | View All Action Items | [Action Items](https://app.transcend.io/action-items/all) | -| actionItemCollections | Onboarding related action item group names | View All Action Items | [Action Items](https://app.transcend.io/action-items/all) | -| teams | Team definitions of users and scope groupings | View Scopes | [Administration -> Teams](https://app.transcend.io/admin/teams) | -| privacyCenters | The privacy center configurations. | View Privacy Center Layout | [Privacy Center](https://app.transcend.io/privacy-center/general-settings) | -| policies | The privacy center policies. | View Policies | [Privacy Center -> Policies](https://app.transcend.io/privacy-center/policies) | -| messages | Message definitions used across consent, privacy center, email templates and more. | View Internationalization Messages | [Privacy Center -> Messages](https://app.transcend.io/privacy-center/messages-internationalization), [Consent Management -> Display Settings -> Messages](https://app.transcend.io/consent-manager/display-settings/messages) | -| assessments | Assessment responses. | View Assessments | [Assessments -> Assessments](https://app.transcend.io/assessments/groups) | -| assessmentTemplates | Assessment template configurations. | View Assessments | [Assessment -> Templates](https://app.transcend.io/assessments/form-templates) | -| purposes | Consent purposes and related preference management topics. | View Consent Manager, View Preference Store Settings | [Consent Management -> Regional Experiences -> Purposes](https://app.transcend.io/consent-manager/regional-experiences/purposes) | +| Resource | Description | Scopes | Link | +| --- | --- | --- | --- | +| apiKeys | API Key definitions assigned to Data Silos. API keys cannot be created through the CLI, but you can map API key usage to Data Silos. | View API Keys | [Developer Tools -> API keys](https://app.transcend.io/infrastructure/api-keys) | +| customFields | Custom field definitions that define extra metadata for each table in the Admin Dashboard. | View Global Attributes | [Custom Fields](https://app.transcend.io/infrastructure/attributes) | +| templates | Email templates. Only template titles can be created and mapped to other resources. | View Email Templates | [DSR Automation -> Email Templates](https://app.transcend.io/privacy-requests/email-templates) | +| dataSilos | The Data Silo/Integration definitions. | View Data Map, View Data Subject Request Settings | [Data Inventory -> Data Silos](https://app.transcend.io/data-map/data-inventory/) and [Infrastucture -> Integrations](https://app.transcend.io/infrastructure/integrationsdata-silos) | +| enrichers | The Privacy Request enricher configurations. | View Identity Verification Settings | [DSR Automation -> Identifiers](https://app.transcend.io/privacy-requests/identifiers) | +| dataFlows | Consent Manager Data Flow definitions. | View Data Flows | [Consent Management -> Data Flows](https://app.transcend.io/consent-manager/data-flows/approved) | +| businessEntities | The business entities in the data inventory. | View Data Inventory | [Data Inventory -> Business Entities](https://app.transcend.io/data-map/data-inventory/business-entities) | +| actions | The Privacy Request action settings. | View Data Subject Request Settings | [DSR Automation -> Request Settings](https://app.transcend.io/privacy-requests/settings) | +| dataSubjects | The Privacy Request data subject settings. | View Data Subject Request Settings | [DSR Automation -> Request Settings](https://app.transcend.io/privacy-requests/settings) | +| identifiers | The Privacy Request identifier configurations. | View Identity Verification Settings | [DSR Automation -> Identifiers](https://app.transcend.io/privacy-requests/identifiers) | +| cookies | Consent Manager Cookie definitions. | View Data Flows | [Consent Management -> Cookies](https://app.transcend.io/consent-manager/cookies/approved) | +| consentManager | Consent Manager general settings, including domain list. | View Consent Manager | [Consent Management -> Developer Settings](https://app.transcend.io/consent-manager/developer-settings) | +| partitions | The partitions in the account (often representative of separate data controllers). | View Consent Manager | [Consent Management -> Developer Settings -> Advanced Settings](https://app.transcend.io/consent-manager/developer-settings/advanced-settings) | +| prompts | The Transcend AI prompts | View Prompts | [Prompt Manager -> Browse](https://app.transcend.io/prompts/browse) | +| promptPartials | The Transcend AI prompt partials | View Prompts | [Prompt Manager -> Partials](https://app.transcend.io/prompts/partialss) | +| promptGroups | The Transcend AI prompt groups | View Prompts | [Prompt Manager -> Groups](https://app.transcend.io/prompts/groups) | +| agents | The agents in the prompt manager. | View Prompts | [Prompt Manager -> Agents](https://app.transcend.io/prompts/agents) | +| agentFunctions | The agent functions in the prompt manager. | View Prompts | [Prompt Manager -> Agent Functions](https://app.transcend.io/prompts/agent-functions) | +| agentFiles | The agent files in the prompt manager. | View Prompts | [Prompt Manager -> Agent Files](https://app.transcend.io/prompts/agent-files) | +| vendors | The vendors in the data inventory. | View Data Inventory | [Data Inventory -> Vendors](https://app.transcend.io/data-map/data-inventory/vendors) | +| dataCategories | The data categories in the data inventory. | View Data Inventory | [Data Inventory -> Data Categories](https://app.transcend.io/data-map/data-inventory/data-categories) | +| processingPurposes | The processing purposes in the data inventory. | View Data Inventory | [Data Inventory -> Processing Purposes](https://app.transcend.io/data-map/data-inventory/purposes) | +| actionItems | Onboarding related action items | View All Action Items | [Action Items](https://app.transcend.io/action-items/all) | +| actionItemCollections | Onboarding related action item group names | View All Action Items | [Action Items](https://app.transcend.io/action-items/all) | +| teams | Team definitions of users and scope groupings | View Scopes | [Administration -> Teams](https://app.transcend.io/admin/teams) | +| privacyCenters | The privacy center configurations. | View Privacy Center Layout | [Privacy Center](https://app.transcend.io/privacy-center/general-settings) | +| policies | The privacy center policies. | View Policies | [Privacy Center -> Policies](https://app.transcend.io/privacy-center/policies) | +| messages | Message definitions used across consent, privacy center, email templates and more. | View Internationalization Messages | [Privacy Center -> Messages](https://app.transcend.io/privacy-center/messages-internationalization), [Consent Management -> Display Settings -> Messages](https://app.transcend.io/consent-manager/display-settings/messages) | +| assessments | Assessment responses. | View Assessments | [Assessments -> Assessments](https://app.transcend.io/assessments/groups) | +| assessmentTemplates | Assessment template configurations. | View Assessments | [Assessment -> Templates](https://app.transcend.io/assessments/form-templates) | +| purposes | Consent purposes and related preference management topics. | View Consent Manager, View Preference Store Settings | [Consent Management -> Regional Experiences -> Purposes](https://app.transcend.io/consent-manager/regional-experiences/purposes) | #### Usage @@ -1133,32 +1107,32 @@ name: Transcend Data Map Syncing # See https://app.transcend.io/privacy-requests/connected-services on: - push: - branches: - - 'main' + push: + branches: + - 'main' jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 - - name: Setup Node.js - uses: actions/setup-node@v2 - with: - node-version: '16' + - name: Setup Node.js + uses: actions/setup-node@v2 + with: + node-version: '16' - - name: Install Transcend CLI - run: npm install --global @transcend-io/cli + - name: Install Transcend CLI + run: npm install --global @transcend-io/cli - # If you have a script that generates your transcend.yml file from - # an ORM or infrastructure configuration, add that step here - # Leave this step commented out if you want to manage your transcend.yml manually - # - name: Generate transcend.yml - # run: ./scripts/generate_transcend_yml.py + # If you have a script that generates your transcend.yml file from + # an ORM or infrastructure configuration, add that step here + # Leave this step commented out if you want to manage your transcend.yml manually + # - name: Generate transcend.yml + # run: ./scripts/generate_transcend_yml.py - - name: Push Transcend config - run: transcend inventory push --auth=${{ secrets.TRANSCEND_API_KEY }} + - name: Push Transcend config + run: transcend inventory push --auth=${{ secrets.TRANSCEND_API_KEY }} ``` #### Dynamic Variables @@ -1175,32 +1149,32 @@ This command could fill out multiple parameters in a YAML file like [./examples/ ```yml api-keys: - - title: Webhook Key + - title: Webhook Key enrichers: - - title: Basic Identity Enrichment - description: Enrich an email address to the userId and phone number - # The data silo webhook URL is the same in each environment, - # except for the base domain in the webhook URL. - url: https://example.<>/transcend-enrichment-webhook - input-identifier: email - output-identifiers: - - userId - - phone - - myUniqueIdentifier - - title: Fraud Check - description: Ensure the email address is not marked as fraudulent - url: https://example.<>/transcend-fraud-check - input-identifier: email - output-identifiers: - - email - privacy-actions: - - ERASURE + - title: Basic Identity Enrichment + description: Enrich an email address to the userId and phone number + # The data silo webhook URL is the same in each environment, + # except for the base domain in the webhook URL. + url: https://example.<>/transcend-enrichment-webhook + input-identifier: email + output-identifiers: + - userId + - phone + - myUniqueIdentifier + - title: Fraud Check + description: Ensure the email address is not marked as fraudulent + url: https://example.<>/transcend-fraud-check + input-identifier: email + output-identifiers: + - email + privacy-actions: + - ERASURE data-silos: - - title: Redshift Data Warehouse - integrationName: server - description: The mega-warehouse that contains a copy over all SQL backed databases - <> - url: https://example.<>/transcend-webhook - api-key-title: Webhook Key + - title: Redshift Data Warehouse + integrationName: server + description: The mega-warehouse that contains a copy over all SQL backed databases - <> + url: https://example.<>/transcend-webhook + api-key-title: Webhook Key ``` ### `transcend inventory scan-packages` @@ -1236,7 +1210,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend inventory discover-silos` ```txt @@ -1273,7 +1246,6 @@ transcend inventory discover-silos --scanPath=./examples/ --auth=$TRANSCEND_API_ ``` This call will look for all the package.json files that in the scan path `./myJavascriptProject`, parse each of the dependencies into their individual package names, and send it to our Transcend backend for classification. These classifications can then be viewed [here](https://app.transcend.io/data-map/data-inventory/silo-discovery/triage). The process is the same for scanning requirements.txt, podfiles and build.gradle files. - ### `transcend inventory pull-datapoints` ```txt @@ -1294,7 +1266,6 @@ FLAGS [--subCategories]... List of subcategories to filter by [separator = ,] -h --help Print help information and exit ``` - ### `transcend inventory pull-unstructured-discovery-files` ```txt @@ -1314,7 +1285,6 @@ FLAGS [--includeEncryptedSnippets] Whether to include encrypted snippets of the entries classified [default = false] -h --help Print help information and exit ``` - ### `transcend inventory derive-data-silos-from-data-flows` ```txt @@ -1332,7 +1302,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend inventory derive-data-silos-from-data-flows-cross-instance` ```txt @@ -1350,7 +1319,6 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` - ### `transcend inventory consent-manager-service-json-to-yml` ```txt @@ -1371,7 +1339,6 @@ FLAGS [--output] Path to the output transcend.yml to write to [default = ./transcend.yml] -h --help Print help information and exit ``` - ### `transcend inventory consent-managers-to-business-entities` ```txt @@ -1386,7 +1353,6 @@ FLAGS [--output] Path to the output transcend.yml with business entity configuration [default = ./combined-business-entities.yml] -h --help Print help information and exit ``` - ### `transcend admin generate-api-keys` ```txt @@ -1432,12 +1398,12 @@ Filter for only a specific organization by ID, returning all child accounts asso ```gql query { - user { - organization { - id - parentOrganizationId - } - } + user { + organization { + id + parentOrganizationId + } + } } ``` @@ -1462,7 +1428,6 @@ transcend admin generate-api-keys --email=test@transcend.io --password=$TRANSCE --scopes="View Email Templates,View Data Map" --apiKeyTitle="CLI Usage Cross Instance Sync" -file=./working/auth.json \ --deleteExistingApiKey=false ``` - ### `transcend migration sync-ot` ```txt @@ -1491,7 +1456,6 @@ FLAGS [--debug] Whether to print detailed logs in case of error [default = false] -h --help Print help information and exit ``` - ## Prompt Manager diff --git a/scripts/buildPathfinderJsonSchema.ts b/scripts/buildPathfinderJsonSchema.ts index ca0562c8..5db31b12 100644 --- a/scripts/buildPathfinderJsonSchema.ts +++ b/scripts/buildPathfinderJsonSchema.ts @@ -12,16 +12,16 @@ * @see https://github.com/SchemaStore/schemastore */ -import { writeFileSync } from 'fs'; -import { join } from 'path'; -import { toJsonSchema } from '@transcend-io/type-utils'; -import { PathfinderPolicy } from '../src/codecs'; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { toJsonSchema } from "@transcend-io/type-utils"; +import { PathfinderPolicy } from "../src/codecs"; const schemaDefaults = { - $schema: 'http://json-schema.org/draft-07/schema#', - $id: 'https://raw.githubusercontent.com/transcend-io/cli/main/pathfinder-policy-yml-schema.json', - title: 'pathfinder.yml', - description: 'Policies for the Transcend Pathfinder AI governance proxy.', + $schema: "http://json-schema.org/draft-07/schema#", + $id: "https://raw.githubusercontent.com/transcend-io/cli/main/pathfinder-policy-yml-schema.json", + title: "pathfinder.yml", + description: "Policies for the Transcend Pathfinder AI governance proxy.", }; // Build the JSON schema from io-ts codec @@ -30,6 +30,6 @@ const jsonSchema = { ...toJsonSchema(PathfinderPolicy, true), }; -const schemaFilePath = join(process.cwd(), 'pathfinder-policy-yml-schema.json'); +const schemaFilePath = join(process.cwd(), "pathfinder-policy-yml-schema.json"); writeFileSync(schemaFilePath, `${JSON.stringify(jsonSchema, null, 2)}\n`); diff --git a/scripts/buildTranscendJsonSchema.ts b/scripts/buildTranscendJsonSchema.ts index d6b8a897..3087a3fa 100644 --- a/scripts/buildTranscendJsonSchema.ts +++ b/scripts/buildTranscendJsonSchema.ts @@ -12,24 +12,24 @@ * @see https://github.com/SchemaStore/schemastore */ -import { writeFileSync } from 'fs'; -import { join } from 'path'; -import { toJsonSchema } from '@transcend-io/type-utils'; -import * as packageJson from '../package.json'; -import { TranscendInput } from '../src/codecs'; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { toJsonSchema } from "@transcend-io/type-utils"; +import * as packageJson from "../package.json"; +import { TranscendInput } from "../src/codecs"; -const majorVersion = packageJson.version.split('.')[0]; +const majorVersion = packageJson.version.split(".")[0]; // Create a major version JSON schema definition, and update the latest JSON schema definition. -[`v${majorVersion}`, 'latest'].forEach((key) => { +for (const key of [`v${majorVersion}`, "latest"]) { const fileName = `transcend-yml-schema-${key}.json`; const schemaDefaults = { - $schema: 'http://json-schema.org/draft-07/schema#', + $schema: "http://json-schema.org/draft-07/schema#", $id: `https://raw.githubusercontent.com/transcend-io/cli/main/${fileName}`, - title: 'transcend.yml', + title: "transcend.yml", description: - 'Define personal data schema and Transcend config as code with the Transcend CLI.', + "Define personal data schema and Transcend config as code with the Transcend CLI.", }; // Build the JSON schema from io-ts codec @@ -38,4 +38,4 @@ const majorVersion = packageJson.version.split('.')[0]; const schemaFilePath = join(process.cwd(), fileName); writeFileSync(schemaFilePath, `${JSON.stringify(jsonSchema, null, 2)}\n`); -}); +} diff --git a/src/codecs.ts b/src/codecs.ts index 3cab16cf..7fcf8b4f 100644 --- a/src/codecs.ts +++ b/src/codecs.ts @@ -1,12 +1,12 @@ // eslint-disable-next-line eslint-comments/disable-enable-pair -/* eslint-disable max-lines */ + import { BrowserLanguage, InitialViewState, OnConsentExpiry, UserPrivacySignalEnum, -} from '@transcend-io/airgap.js-types'; -import { LanguageKey } from '@transcend-io/internationalization'; +} from "@transcend-io/airgap.js-types"; +import { LanguageKey } from "@transcend-io/internationalization"; import { ActionItemCode, ActionItemPriorityOverride, @@ -58,12 +58,12 @@ import { UnknownRequestPolicy, UnstructuredSubDataPointRecommendationStatus, UspapiOption, -} from '@transcend-io/privacy-types'; -import { applyEnum, valuesOf } from '@transcend-io/type-utils'; -import * as t from 'io-ts'; -import { OpenAIRouteName, PathfinderPolicyName } from './enums'; -import { buildAIIntegrationType } from './lib/helpers/buildAIIntegrationType'; -import { buildEnabledRouteType } from './lib/helpers/buildEnabledRouteType'; +} from "@transcend-io/privacy-types"; +import { applyEnum, valuesOf } from "@transcend-io/type-utils"; +import * as t from "io-ts"; +import { OpenAIRouteName, PathfinderPolicyName } from "./enums"; +import { buildAIIntegrationType } from "./lib/helpers/buildAIIntegrationType"; +import { buildEnabledRouteType } from "./lib/helpers/buildEnabledRouteType"; /** * Input to define email templates that can be used to communicate to end-users @@ -126,11 +126,11 @@ export const TeamInput = t.intersection([ }), t.partial({ /** SSO department for automated provisioning */ - 'sso-department': t.string, + "sso-department": t.string, /** SSO group name for automated provisioning */ - 'sso-group': t.string, + "sso-group": t.string, /** SSO title mapping for automated provisioning */ - 'sso-title': t.string, + "sso-title": t.string, /** List of user emails on the team */ users: t.array(t.string), /** List of scopes that the team should have */ @@ -162,7 +162,7 @@ export const EnricherInput = t.intersection([ * The names of the identifiers that can be resolved by this enricher. * i.e. email -> [userId, phone, advertisingId] */ - 'output-identifiers': t.array(t.string), + "output-identifiers": t.array(t.string), }), t.partial({ /** Internal description for why the enricher is needed */ @@ -176,7 +176,7 @@ export const EnricherInput = t.intersection([ * Whenever a privacy request contains this identifier, the webhook will * be called with the value of that identifier as input */ - 'input-identifier': t.string, + "input-identifier": t.string, /** * A regular expression that can be used to match on for cancelation */ @@ -199,16 +199,16 @@ export const EnricherInput = t.intersection([ phoneNumbers: t.array(t.string), /** The list of regions that should trigger the preflight check */ regionList: t.array( - valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }), + valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }) ), /** * Specify which data subjects the enricher should run for */ - 'data-subjects': t.array(t.string), + "data-subjects": t.array(t.string), /** Headers to include in the webhook */ headers: t.array(WebhookHeader), /** The privacy actions that the enricher should run against */ - 'privacy-actions': t.array(valuesOf(RequestAction)), + "privacy-actions": t.array(valuesOf(RequestAction)), }), ]); @@ -355,7 +355,7 @@ export const AgentInput = t.intersection([ /** Whether the agent has retrieval enabled */ retrievalEnabled: t.boolean, /** Large language model powering the agent */ - 'large-language-model': t.type({ + "large-language-model": t.type({ /** Name of the model */ name: t.string, /** Client of the model */ @@ -383,11 +383,11 @@ export const AgentInput = t.intersection([ /** * The names of the functions that the agent has access to */ - 'agent-functions': t.array(t.string), + "agent-functions": t.array(t.string), /** * The names of the files that the agent has access to for retrieval */ - 'agent-files': t.array(t.string), + "agent-files": t.array(t.string), }), ]); @@ -646,19 +646,19 @@ export const FieldInput = t.intersection([ * * @see https://github.com/transcend-io/privacy-types/blob/main/src/objects.ts */ - 'guessed-categories': t.array(DataCategoryGuessInput), + "guessed-categories": t.array(DataCategoryGuessInput), /** * When true, this subdatapoint should be revealed in a data access request. * When false, this field should be redacted */ - 'access-request-visibility-enabled': t.boolean, + "access-request-visibility-enabled": t.boolean, /** * When true, this subdatapoint should be redacted during an erasure request. * There normally is a choice of enabling hard deletion or redaction at the * datapoint level, but if redaction is enabled, this column can be used * to define which fields should be redacted. */ - 'erasure-request-redaction-enabled': t.boolean, + "erasure-request-redaction-enabled": t.boolean, /** Attributes tagged to subdatapoint */ attributes: t.array(AttributePreview), }), @@ -701,21 +701,21 @@ export const DatapointInput = t.intersection([ * * @see https://docs.transcend.io/docs/privacy-requests/connecting-data-silos/saas-tools#configuring-an-integration */ - 'data-collection-tag': t.string, + "data-collection-tag": t.string, /** * The SQL queries that should be run for that datapoint in a privacy request. * * @see https://github.com/transcend-io/privacy-types/blob/main/src/actions.ts */ - 'privacy-action-queries': t.partial( - applyEnum(RequestActionObjectResolver, () => t.string), + "privacy-action-queries": t.partial( + applyEnum(RequestActionObjectResolver, () => t.string) ), /** * The types of privacy actions that this datapoint can implement * * @see https://github.com/transcend-io/privacy-types/blob/main/src/actions.ts */ - 'privacy-actions': t.array(valuesOf(RequestActionObjectResolver)), + "privacy-actions": t.array(valuesOf(RequestActionObjectResolver)), /** * Provide field-level metadata for this datapoint. * This is often the column metadata @@ -742,30 +742,30 @@ export type DatapointInput = t.TypeOf; export const PromptAVendorEmailSettings = t.partial({ /** The email address of the user to notify when a promptAPerson integration */ - 'notify-email-address': t.string, + "notify-email-address": t.string, /** * The frequency with which we should be sending emails for this data silo, in milliseconds. */ - 'send-frequency': t.number, + "send-frequency": t.number, /** * The type of emails to send for this data silo, i.e. send an email for each DSR, across all open DSRs, * or per profile in a DSR. */ - 'send-type': valuesOf(PromptAVendorEmailSendType), + "send-type": valuesOf(PromptAVendorEmailSendType), /** * Indicates whether prompt-a-vendor emails should include a list of identifiers * in addition to a link to the bulk processing UI. */ - 'include-identifiers-attachment': t.boolean, + "include-identifiers-attachment": t.boolean, /** * Indicates what kind of link to generate as part of the emails sent out for this Prompt-a-Vendor silo. */ - 'completion-link-type': valuesOf(PromptAVendorEmailCompletionLinkType), + "completion-link-type": valuesOf(PromptAVendorEmailCompletionLinkType), /** * The frequency with which we should retry sending emails for this data silo, in milliseconds. * Needs to be a string because the number can be larger than the MAX_INT */ - 'manual-work-retry-frequency': t.string, + "manual-work-retry-frequency": t.string, }); /** Type override */ @@ -975,11 +975,11 @@ export const ActionInput = t.intersection([ regionDetectionMethod: valuesOf(RegionDetectionMethod), /** The list of regions to show in the form */ regionList: t.array( - valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }), + valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }) ), /** The list of regions NOT to show in the form */ regionBlockList: t.array( - valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }), + valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }) ), }), ]); @@ -1128,7 +1128,7 @@ export const ConsentManageExperienceInput = t.intersection([ t.partial({ countrySubDivision: valuesOf(IsoCountrySubdivisionCode), country: valuesOf(IsoCountryCode), - }), + }) ), /** How to handle consent expiry */ onConsentExpiry: valuesOf(OnConsentExpiry), @@ -1145,14 +1145,14 @@ export const ConsentManageExperienceInput = t.intersection([ t.type({ /** Slug of purpose */ trackingType: t.string, - }), + }) ), /** Purposes that are opted out by default in a particular experience */ optedOutPurposes: t.array( t.type({ /** Slug of purpose */ trackingType: t.string, - }), + }) ), /** * Browser languages that define this regional experience @@ -1344,13 +1344,13 @@ export const DataSiloInput = t.intersection([ }), t.partial({ /** For prompt a person or database integrations, the underlying integration name */ - 'outer-type': t.string, + "outer-type": t.string, /** A description for that data silo */ description: t.string, /** The webhook URL to notify for data privacy requests */ url: t.string, /** The title of the API key that will be used to respond to privacy requests */ - 'api-key-title': t.string, + "api-key-title": t.string, /** Custom headers to include in outbound webhook */ headers: t.array(WebhookHeader), /** @@ -1358,18 +1358,18 @@ export const DataSiloInput = t.intersection([ * This field can be omitted, and the default assumption will be that the system may potentially * contain PII for any potential data subject type. */ - 'data-subjects': t.array(t.string), + "data-subjects": t.array(t.string), /** * When this data silo implements a privacy request, these are the identifiers * that should be looked up within this system. */ - 'identity-keys': t.array(t.string), + "identity-keys": t.array(t.string), /** * When a data erasure request is being performed, this data silo should not be deleted from * until all of the following data silos were deleted first. This list can contain other internal * systems defined in this file, as well as any of the SaaS tools connected in your Transcend instance. */ - 'deletion-dependencies': t.array(t.string), + "deletion-dependencies": t.array(t.string), /** * The email addresses of the employees within your company that are the go-to individuals * for managing this data silo @@ -1396,7 +1396,7 @@ export const DataSiloInput = t.intersection([ /** * Configure email notification settings for privacy requests */ - 'email-settings': PromptAVendorEmailSettings, + "email-settings": PromptAVendorEmailSettings, /** Country of data silo hosting */ country: valuesOf(IsoCountryCode), /** Sub-division of data silo hosting */ @@ -1484,13 +1484,13 @@ export type ActionItemInput = t.TypeOf; export const AssessmentRuleInput = t.intersection([ t.type({ /** The reference id of the question whose answer is compared by this rule */ - 'depends-on-question-reference-id': t.string, + "depends-on-question-reference-id": t.string, /** The operator to use when comparing the question answer to the operands */ - 'comparison-operator': valuesOf(ComparisonOperator), + "comparison-operator": valuesOf(ComparisonOperator), }), t.partial({ /** The values to compare the question answer to */ - 'comparison-operands': t.array(t.string), + "comparison-operands": t.array(t.string), }), ]); @@ -1499,28 +1499,28 @@ export type AssessmentRuleInput = t.TypeOf; export interface AssessmentNestedRuleInput { /** The operator to use when comparing the nested rules */ - 'logic-operator': LogicOperator; + "logic-operator": LogicOperator; /** The rules to evaluate and be compared with to other using the LogicOperator */ rules?: AssessmentRuleInput[]; /** The nested rules to add one more level of nesting to the rules. They are also compared to each other. */ - 'nested-rules'?: AssessmentNestedRuleInput[]; + "nested-rules"?: AssessmentNestedRuleInput[]; } export const AssessmentNestedRuleInput: t.RecursiveType< t.Type -> = t.recursion('AssessmentNestedRuleInput', (self) => +> = t.recursion("AssessmentNestedRuleInput", (self) => t.intersection([ t.type({ /** The operator to use when comparing the nested rules */ - 'logic-operator': valuesOf(LogicOperator), + "logic-operator": valuesOf(LogicOperator), }), t.partial({ /** The rules to evaluate and be compared with to other using the LogicOperator */ rules: t.array(AssessmentRuleInput), /** The nested rules to add one more level of nesting to the rules. They are also compared to each other. */ - 'nested-rules': t.array(self), + "nested-rules": t.array(self), }), - ]), + ]) ); export const AssessmentDisplayLogicInput = t.intersection([ @@ -1532,7 +1532,7 @@ export const AssessmentDisplayLogicInput = t.intersection([ /** The rule to evaluate */ rule: AssessmentRuleInput, /** The nested rule to evaluate */ - 'nested-rule': AssessmentNestedRuleInput, + "nested-rule": AssessmentNestedRuleInput, }), ]); @@ -1543,11 +1543,11 @@ export type AssessmentDisplayLogicInput = t.TypeOf< export const RiskAssignmentInput = t.partial({ /** The risk level to assign to the question */ - 'risk-level': t.string, + "risk-level": t.string, /** The risk matrix column to assign to a question. */ - 'risk-matrix-column': t.string, + "risk-matrix-column": t.string, /** The risk matrix row to assign to a question. */ - 'risk-matrix-row': t.string, + "risk-matrix-row": t.string, }); /** Type override */ @@ -1556,17 +1556,17 @@ export type RiskAssignmentInput = t.TypeOf; export const RiskLogicInput = t.intersection([ t.type({ /** The values to compare */ - 'comparison-operands': t.array(t.string), + "comparison-operands": t.array(t.string), /** The operator */ - 'comparison-operator': valuesOf(ComparisonOperator), + "comparison-operator": valuesOf(ComparisonOperator), }), t.partial({ /** The risk level to assign to the question */ - 'risk-level': t.string, + "risk-level": t.string, /** The risk matrix column to assign to a question. */ - 'risk-matrix-column': t.string, + "risk-matrix-column": t.string, /** The risk matrix row to assign to a question. */ - 'risk-matrix-row': t.string, + "risk-matrix-row": t.string, }), ]); @@ -1592,41 +1592,41 @@ export const AssessmentSectionQuestionInput = t.intersection([ }), t.partial({ /** The sub-type of the assessment question */ - 'sub-type': valuesOf(AssessmentQuestionSubType), + "sub-type": valuesOf(AssessmentQuestionSubType), /** The question placeholder */ placeholder: t.string, /** The question description */ description: t.string, /** Whether an answer is required */ - 'is-required': t.boolean, + "is-required": t.boolean, /** Used to identify the question within a form or template so it can be referenced in conditional logic. */ - 'reference-id': t.string, + "reference-id": t.string, /** Display logic for the question */ - 'display-logic': AssessmentDisplayLogicInput, + "display-logic": AssessmentDisplayLogicInput, /** Risk logic for the question */ - 'risk-logic': t.array(RiskLogicInput), + "risk-logic": t.array(RiskLogicInput), /** Risk category titles for the question */ - 'risk-categories': t.array(t.string), + "risk-categories": t.array(t.string), /** Risk framework titles for the question */ - 'risk-framework': t.string, + "risk-framework": t.string, /** Answer options for the question */ - 'answer-options': t.array(AssessmentAnswerOptionInput), + "answer-options": t.array(AssessmentAnswerOptionInput), /** The selected answers to the assessments */ - 'selected-answers': t.array(t.string), + "selected-answers": t.array(t.string), /** Allowed MIME types for the question */ - 'allowed-mime-types': t.array(t.string), + "allowed-mime-types": t.array(t.string), /** Allow selecting other options */ - 'allow-select-other': t.boolean, + "allow-select-other": t.boolean, /** Sync model for the question */ - 'sync-model': valuesOf(AssessmentSyncModel), + "sync-model": valuesOf(AssessmentSyncModel), /** Sync column for the question */ - 'sync-column': valuesOf(AssessmentSyncColumn), + "sync-column": valuesOf(AssessmentSyncColumn), /** Attribute key / custom field name for the question */ - 'attribute-key': t.string, + "attribute-key": t.string, /** Require risk evaluation for the question */ - 'require-risk-evaluation': t.boolean, + "require-risk-evaluation": t.boolean, /** Require risk matrix evaluation for the question */ - 'require-risk-matrix-evaluation': t.boolean, + "require-risk-matrix-evaluation": t.boolean, }), ]); @@ -1646,11 +1646,11 @@ export const AssessmentSectionInput = t.intersection([ /** Email address of those assigned */ assignees: t.array(t.string), /** Email address of those externally assigned */ - 'external-assignees': t.array(t.string), + "external-assignees": t.array(t.string), /** Status of section */ status: t.string, /** Whether assessment is reviewed */ - 'is-reviewed': t.boolean, + "is-reviewed": t.boolean, }), ]); @@ -1661,7 +1661,7 @@ export const AssessmentRetentionScheduleInput = t.type({ /** The retention schedule type */ type: valuesOf(RetentionScheduleType), /** The duration of the retention schedule in days */ - 'duration-days': t.number, + "duration-days": t.number, /** The operation to perform on the retention schedule */ operand: valuesOf(RetentionScheduleOperation), }); @@ -1690,15 +1690,15 @@ export const AssessmentTemplateInput = t.intersection([ /** Whether the template is in a locked status */ locked: t.boolean, /** ID of parent template this was cloned from */ - 'parent-id': t.string, + "parent-id": t.string, /** Whether the template is archived */ archived: t.boolean, /** The date that the assessment was created */ - 'created-at': t.string, + "created-at": t.string, /** The names of the custom fields associated to this assessment template */ - 'attribute-keys': t.array(t.string), + "attribute-keys": t.array(t.string), /** The retention schedule configuration */ - 'retention-schedule': AssessmentRetentionScheduleInput, + "retention-schedule": AssessmentRetentionScheduleInput, /** The titles of the email templates used in the assessment template */ templates: t.array(t.string), }), @@ -1736,7 +1736,7 @@ export const AssessmentInput = t.intersection([ /** The emails of the transcend users assigned to the assessment */ assignees: t.array(t.string), /** The emails of the external emails assigned to the assessment */ - 'external-assignees': t.array(t.string), + "external-assignees": t.array(t.string), /** The emails of the assessment reviewers */ reviewers: t.array(t.string), /** Whether the assessment is in a locked status */ @@ -1748,25 +1748,25 @@ export const AssessmentInput = t.intersection([ /** * Whether the form title is an internal label only, and the group title should be used in communications with assignees */ - 'title-is-internal': t.boolean, + "title-is-internal": t.boolean, /** The date that the assessment is due */ - 'due-date': t.string, + "due-date": t.string, /** The date that the assessment was created */ - 'created-at': t.string, + "created-at": t.string, /** The date that the assessment was assigned at */ - 'assigned-at': t.string, + "assigned-at": t.string, /** The date that the assessment was submitted at */ - 'submitted-at': t.string, + "submitted-at": t.string, /** The date that the assessment was approved at */ - 'approved-at': t.string, + "approved-at": t.string, /** The date that the assessment was rejected at */ - 'rejected-at': t.string, + "rejected-at": t.string, /** The linked data inventory resources */ resources: t.array(AssessmentResourceInput), /** The linked data inventory synced rows */ rows: t.array(AssessmentResourceInput), /** The assessment retention schedule */ - 'retention-schedule': AssessmentRetentionScheduleInput, + "retention-schedule": AssessmentRetentionScheduleInput, /** The assessment custom fields */ attributes: t.array(AttributePreview), }), @@ -1798,9 +1798,9 @@ export const ConsentPreferenceTopic = t.intersection([ }), t.partial({ /** Default value */ - 'default-configuration': t.string, + "default-configuration": t.string, /** Whether the preference topic is shown in privacy center */ - 'show-in-privacy-center': t.boolean, + "show-in-privacy-center": t.boolean, /** The options when type is single or multi select */ options: t.array(ConsentPreferenceTopicOptionValue), }), @@ -1822,23 +1822,23 @@ export const ConsentPurpose = t.intersection([ /** Description of purpose */ description: t.string, /** Whether purpose is active */ - 'is-active': t.boolean, + "is-active": t.boolean, /** Whether purpose is configurable */ configurable: t.boolean, /** Display order of purpose for privacy center */ - 'display-order': t.number, + "display-order": t.number, /** Whether purpose is shown in privacy center */ - 'show-in-privacy-center': t.boolean, + "show-in-privacy-center": t.boolean, /** Whether purpose is show in consent manger */ - 'show-in-consent-manager': t.boolean, + "show-in-consent-manager": t.boolean, /** The preference topics configured for the purpose */ - 'preference-topics': t.array(ConsentPreferenceTopic), + "preference-topics": t.array(ConsentPreferenceTopic), /** Authentication level for purpose on privacy center */ - 'auth-level': valuesOf(PreferenceStoreAuthLevel), + "auth-level": valuesOf(PreferenceStoreAuthLevel), /** Opt out signals that should instantly opt out of this purpose */ - 'opt-out-signals': t.array(valuesOf(UserPrivacySignalEnum)), + "opt-out-signals": t.array(valuesOf(UserPrivacySignalEnum)), /** Default consent value */ - 'default-consent': valuesOf(DefaultConsentOption), + "default-consent": valuesOf(DefaultConsentOption), }), ]); @@ -1849,15 +1849,15 @@ export const TranscendInput = t.partial({ /** * Action items */ - 'action-items': t.array(ActionItemInput), + "action-items": t.array(ActionItemInput), /** * Action item collections */ - 'action-item-collections': t.array(ActionItemCollectionInput), + "action-item-collections": t.array(ActionItemCollectionInput), /** * API key definitions */ - 'api-keys': t.array(ApiKeyInput), + "api-keys": t.array(ApiKeyInput), /** Team definitions */ teams: t.array(TeamInput), /** @@ -1875,7 +1875,7 @@ export const TranscendInput = t.partial({ /** * Business entity definitions */ - 'business-entities': t.array(BusinessEntityInput), + "business-entities": t.array(BusinessEntityInput), /** * Vendor definitions */ @@ -1883,15 +1883,15 @@ export const TranscendInput = t.partial({ /** * Data categories definitions */ - 'data-categories': t.array(DataCategoryInput), + "data-categories": t.array(DataCategoryInput), /** * Vendor definitions */ - 'processing-purposes': t.array(ProcessingPurposeInput), + "processing-purposes": t.array(ProcessingPurposeInput), /** * Data subject definitions */ - 'data-subjects': t.array(DataSubjectInput), + "data-subjects": t.array(DataSubjectInput), /** * Action definitions */ @@ -1903,11 +1903,11 @@ export const TranscendInput = t.partial({ /** * Data silo definitions */ - 'data-silos': t.array(DataSiloInput), + "data-silos": t.array(DataSiloInput), /** * Data flow definitions */ - 'data-flows': t.array(DataFlowInput), + "data-flows": t.array(DataFlowInput), /** * Cookie definitions */ @@ -1915,7 +1915,7 @@ export const TranscendInput = t.partial({ /** * Consent manager definition */ - 'consent-manager': ConsentManagerInput, + "consent-manager": ConsentManagerInput, /** * Prompt definitions */ @@ -1923,11 +1923,11 @@ export const TranscendInput = t.partial({ /** * Prompt partial definitions */ - 'prompt-partials': t.array(PromptPartialInput), + "prompt-partials": t.array(PromptPartialInput), /** * Prompt group definitions */ - 'prompt-groups': t.array(PromptGroupInput), + "prompt-groups": t.array(PromptGroupInput), /** * Agent definitions */ @@ -1935,15 +1935,15 @@ export const TranscendInput = t.partial({ /** * Agent function definitions */ - 'agent-functions': t.array(AgentFunctionInput), + "agent-functions": t.array(AgentFunctionInput), /** * Agent file definitions */ - 'agent-files': t.array(AgentFileInput), + "agent-files": t.array(AgentFileInput), /** * The privacy center configuration */ - 'privacy-center': PrivacyCenterInput, + "privacy-center": PrivacyCenterInput, /** * The policies configuration */ @@ -1957,7 +1957,7 @@ export const TranscendInput = t.partial({ /** * The full list of assessment templates */ - 'assessment-templates': t.array(AssessmentTemplateInput), + "assessment-templates": t.array(AssessmentTemplateInput), /** * The full list of assessment results */ @@ -1992,7 +1992,7 @@ export type StoredApiKey = t.TypeOf; export const DataFlowCsvInput = t.intersection([ t.type({ /** The value of the data flow (host or regex) */ - 'Connections Made To': t.string, + "Connections Made To": t.string, /** The type of the data flow */ Type: valuesOf(DataFlowScope), /** The CSV of purposes mapped to that data flow */ @@ -2058,7 +2058,7 @@ export const ConsentManagerServiceMetadata = t.type({ name: t.string, /** Allowed purposes */ trackingPurposes: t.array(t.string), - }), + }) ), /** Data Flows */ dataFlows: t.array( @@ -2069,7 +2069,7 @@ export const ConsentManagerServiceMetadata = t.type({ type: valuesOf(DataFlowScope), /** Allowed purposes */ trackingPurposes: t.array(t.string), - }), + }) ), }); diff --git a/src/commands/admin/generate-api-keys/impl.ts b/src/commands/admin/generate-api-keys/impl.ts index 8f37de0a..f9c0d278 100644 --- a/src/commands/admin/generate-api-keys/impl.ts +++ b/src/commands/admin/generate-api-keys/impl.ts @@ -1,17 +1,17 @@ -import { writeFileSync } from 'fs'; -import { ScopeName, TRANSCEND_SCOPES } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { keyBy } from 'lodash-es'; -import type { LocalContext } from '../../../context'; -import { generateCrossAccountApiKeys } from '../../../lib/api-keys'; -import { logger } from '../../../logger'; +import { writeFileSync } from "node:fs"; +import { ScopeName, TRANSCEND_SCOPES } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { keyBy } from "lodash-es"; +import type { LocalContext } from "../../../context"; +import { generateCrossAccountApiKeys } from "../../../lib/api-keys"; +import { logger } from "../../../logger"; const SCOPES_BY_TITLE = keyBy( Object.entries(TRANSCEND_SCOPES).map(([name, value]) => ({ ...value, name, })), - 'title', + "title" ); const SCOPE_TITLES = Object.keys(SCOPES_BY_TITLE); @@ -41,25 +41,25 @@ export async function generateApiKeys( createNewApiKey, parentOrganizationId, transcendUrl, - }: GenerateApiKeysCommandFlags, + }: GenerateApiKeysCommandFlags ): Promise { // Validate scopes const splitScopes = scopes.map((x) => x.trim()); const invalidScopes = splitScopes.filter( - (scopeTitle) => !SCOPES_BY_TITLE[scopeTitle], + (scopeTitle) => !SCOPES_BY_TITLE[scopeTitle] ); if (invalidScopes.length > 0) { logger.error( colors.red( - `Failed to parse scopes:"${invalidScopes.join(',')}".\n` + - `Expected one of: \n${SCOPE_TITLES.join('\n')}`, - ), + `Failed to parse scopes:"${invalidScopes.join(",")}".\n` + + `Expected one of: \n${SCOPE_TITLES.join("\n")}` + ) ); process.exit(1); } const scopeNames = splitScopes.map( - (scopeTitle) => SCOPES_BY_TITLE[scopeTitle].name as ScopeName, + (scopeTitle) => SCOPES_BY_TITLE[scopeTitle].name as ScopeName ); // Upload privacy requests diff --git a/src/commands/consent/build-xdi-sync-endpoint/impl.ts b/src/commands/consent/build-xdi-sync-endpoint/impl.ts index a4735488..2b98351a 100644 --- a/src/commands/consent/build-xdi-sync-endpoint/impl.ts +++ b/src/commands/consent/build-xdi-sync-endpoint/impl.ts @@ -1,9 +1,9 @@ -import { writeFileSync } from 'fs'; -import colors from 'colors'; -import type { LocalContext } from '../../../context'; -import { validateTranscendAuth } from '../../../lib/api-keys'; -import { buildXdiSyncEndpoint as buildXdiSyncEndpointHelper } from '../../../lib/consent-manager'; -import { logger } from '../../../logger'; +import { writeFileSync } from "node:fs"; +import colors from "colors"; +import type { LocalContext } from "../../../context"; +import { validateTranscendAuth } from "../../../lib/api-keys"; +import { buildXdiSyncEndpoint as buildXdiSyncEndpointHelper } from "../../../lib/consent-manager"; +import { logger } from "../../../logger"; interface BuildXdiSyncEndpointCommandFlags { auth: string; @@ -25,7 +25,7 @@ export async function buildXdiSyncEndpoint( domainBlockList, xdiAllowedCommands, transcendUrl, - }: BuildXdiSyncEndpointCommandFlags, + }: BuildXdiSyncEndpointCommandFlags ): Promise { // Parse authentication as API key or path to list of API keys const apiKeyOrList = await validateTranscendAuth(auth); @@ -45,9 +45,9 @@ export async function buildXdiSyncEndpoint( `Successfully constructed sync endpoint for sync groups: ${JSON.stringify( syncGroups, null, - 2, - )}`, - ), + 2 + )}` + ) ); // Write to disk diff --git a/src/commands/consent/pull-consent-metrics/impl.ts b/src/commands/consent/pull-consent-metrics/impl.ts index ffba9d54..a7078c79 100644 --- a/src/commands/consent/pull-consent-metrics/impl.ts +++ b/src/commands/consent/pull-consent-metrics/impl.ts @@ -1,17 +1,17 @@ -import fs, { existsSync, mkdirSync } from 'fs'; -import { join } from 'path'; -import colors from 'colors'; -import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; -import type { LocalContext } from '../../../context'; -import { validateTranscendAuth } from '../../../lib/api-keys'; -import { mapSeries } from '../../../lib/bluebird-replace'; -import { pullConsentManagerMetrics } from '../../../lib/consent-manager'; -import { writeCsv } from '../../../lib/cron'; +import fs, { existsSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; +import colors from "colors"; +import { ADMIN_DASH_INTEGRATIONS } from "../../../constants"; +import type { LocalContext } from "../../../context"; +import { validateTranscendAuth } from "../../../lib/api-keys"; +import { mapSeries } from "../../../lib/bluebird-replace"; +import { pullConsentManagerMetrics } from "../../../lib/consent-manager"; +import { writeCsv } from "../../../lib/cron"; import { buildTranscendGraphQLClient, ConsentManagerMetricBin, -} from '../../../lib/graphql'; -import { logger } from '../../../logger'; +} from "../../../lib/graphql"; +import { logger } from "../../../logger"; interface PullConsentMetricsCommandFlags { auth: string; @@ -31,7 +31,7 @@ export async function pullConsentMetrics( folder, bin, transcendUrl, - }: PullConsentMetricsCommandFlags, + }: PullConsentMetricsCommandFlags ): Promise { // Parse authentication as API key or path to list of API keys const apiKeyOrList = await validateTranscendAuth(auth); @@ -40,8 +40,8 @@ export async function pullConsentMetrics( if (fs.existsSync(folder) && !fs.lstatSync(folder).isDirectory()) { logger.error( colors.red( - 'The provided argument "folder" was passed a file. expected: folder="./consent-metrics/"', - ), + 'The provided argument "folder" was passed a file. expected: folder="./consent-metrics/"' + ) ); process.exit(1); } @@ -53,9 +53,9 @@ export async function pullConsentMetrics( colors.red( `Failed to parse argument "bin" with value "${bin}"\n` + `Expected one of: \n${Object.values(ConsentManagerMetricBin).join( - '\n', - )}`, - ), + "\n" + )}` + ) ); process.exit(1); } @@ -66,16 +66,16 @@ export async function pullConsentMetrics( if (Number.isNaN(startDate.getTime())) { logger.error( colors.red( - `Start date provided is invalid date. Got --start="${start}" expected --start="01/01/2023"`, - ), + `Start date provided is invalid date. Got --start="${start}" expected --start="01/01/2023"` + ) ); process.exit(1); } if (Number.isNaN(endDate.getTime())) { logger.error( colors.red( - `End date provided is invalid date. Got --end="${end}" expected --end="01/01/2023"`, - ), + `End date provided is invalid date. Got --end="${end}" expected --end="01/01/2023"` + ) ); process.exit(1); } @@ -83,8 +83,8 @@ export async function pullConsentMetrics( logger.error( colors.red( `Got a start date "${startDate.toISOString()}" that was larger than the end date "${endDate.toISOString()}". ` + - 'Start date must be before end date.', - ), + "Start date must be before end date." + ) ); process.exit(1); } @@ -96,12 +96,12 @@ export async function pullConsentMetrics( logger.info( colors.magenta( - `Pulling consent metrics from start=${startDate.toString()} to end=${endDate.toISOString()} with bin size "${bin}"`, - ), + `Pulling consent metrics from start=${startDate.toString()} to end=${endDate.toISOString()} with bin size "${bin}"` + ) ); // Sync to Disk - if (typeof apiKeyOrList === 'string') { + if (typeof apiKeyOrList === "string") { try { // Create a GraphQL client const client = buildTranscendGraphQLClient(transcendUrl, apiKeyOrList); @@ -114,24 +114,24 @@ export async function pullConsentMetrics( }); // Write to file - Object.entries(configuration).forEach(([metricName, metrics]) => { - metrics.forEach(({ points, name }) => { + for (const [metricName, metrics] of Object.entries(configuration)) { + for (const { points, name } of metrics) { const file = join(folder, `${metricName}_${name}.csv`); logger.info( - colors.magenta(`Writing configuration to file "${file}"...`), + colors.magenta(`Writing configuration to file "${file}"...`) ); writeCsv( file, points.map(({ key, value }) => ({ timestamp: key, value, - })), + })) ); - }); - }); - } catch (err) { + } + } + } catch (error) { logger.error( - colors.red(`An error occurred syncing the schema: ${err.message}`), + colors.red(`An error occurred syncing the schema: ${error.message}`) ); process.exit(1); } @@ -139,8 +139,8 @@ export async function pullConsentMetrics( // Indicate success logger.info( colors.green( - `Successfully synced consent metrics to disk in folder "${folder}"! View at ${ADMIN_DASH_INTEGRATIONS}`, - ), + `Successfully synced consent metrics to disk in folder "${folder}"! View at ${ADMIN_DASH_INTEGRATIONS}` + ) ); } else { const encounteredErrors: string[] = []; @@ -150,8 +150,8 @@ export async function pullConsentMetrics( }] `; logger.info( colors.magenta( - `~~~\n\n${prefix}Attempting to pull consent metrics...\n\n~~~`, - ), + `~~~\n\n${prefix}Attempting to pull consent metrics...\n\n~~~` + ) ); // Create a GraphQL client @@ -171,26 +171,26 @@ export async function pullConsentMetrics( } // Write to file - Object.entries(configuration).forEach(([metricName, metrics]) => { - metrics.forEach(({ points, name }) => { + for (const [metricName, metrics] of Object.entries(configuration)) { + for (const { points, name } of metrics) { const file = join(subFolder, `${metricName}_${name}.csv`); logger.info( - colors.magenta(`Writing configuration to file "${file}"...`), + colors.magenta(`Writing configuration to file "${file}"...`) ); writeCsv( file, points.map(({ key, value }) => ({ timestamp: key, value, - })), + })) ); - }); - }); + } + } logger.info( - colors.green(`${prefix}Successfully pulled configuration!`), + colors.green(`${prefix}Successfully pulled configuration!`) ); - } catch (err) { + } catch { logger.error(colors.red(`${prefix}Failed to sync configuration.`)); encounteredErrors.push(apiKey.organizationName); } @@ -200,9 +200,9 @@ export async function pullConsentMetrics( logger.info( colors.red( `Sync encountered errors for "${encounteredErrors.join( - ',', - )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}`, - ), + "," + )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}` + ) ); process.exit(1); diff --git a/src/commands/consent/upload-preferences/impl.ts b/src/commands/consent/upload-preferences/impl.ts index 9fdf8f55..5d2f61c9 100644 --- a/src/commands/consent/upload-preferences/impl.ts +++ b/src/commands/consent/upload-preferences/impl.ts @@ -1,11 +1,11 @@ -import { readdirSync } from 'fs'; -import { basename, join } from 'path'; -import colors from 'colors'; -import type { LocalContext } from '../../../context'; -import { map } from '../../../lib/bluebird-replace'; -import { uploadPreferenceManagementPreferencesInteractive } from '../../../lib/preference-management'; -import { splitCsvToList } from '../../../lib/requests'; -import { logger } from '../../../logger'; +import { readdirSync } from "node:fs"; +import { basename, join } from "node:path"; +import colors from "colors"; +import type { LocalContext } from "../../../context"; +import { map } from "../../../lib/bluebird-replace"; +import { uploadPreferenceManagementPreferencesInteractive } from "../../../lib/preference-management"; +import { splitCsvToList } from "../../../lib/requests"; +import { logger } from "../../../logger"; interface UploadPreferencesCommandFlags { auth: string; @@ -33,7 +33,7 @@ export async function uploadPreferences( partition, sombraAuth, consentUrl, - file = '', + file = "", directory, dryRun, skipExistingRecordCheck, @@ -44,13 +44,13 @@ export async function uploadPreferences( isSilent, attributes, concurrency, - }: UploadPreferencesCommandFlags, + }: UploadPreferencesCommandFlags ): Promise { if (!!directory && !!file) { logger.error( colors.red( - 'Cannot provide both a directory and a file. Please provide only one.', - ), + "Cannot provide both a directory and a file. Please provide only one." + ) ); process.exit(1); } @@ -58,8 +58,8 @@ export async function uploadPreferences( if (!file && !directory) { logger.error( colors.red( - 'A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences', - ), + "A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences" + ) ); process.exit(1); } @@ -69,56 +69,56 @@ export async function uploadPreferences( if (directory) { try { const filesInDirectory = readdirSync(directory); - const csvFiles = filesInDirectory.filter((file) => file.endsWith('.csv')); + const csvFiles = filesInDirectory.filter((file) => file.endsWith(".csv")); if (csvFiles.length === 0) { logger.error( - colors.red(`No CSV files found in directory: ${directory}`), + colors.red(`No CSV files found in directory: ${directory}`) ); process.exit(1); } // Add full paths for each CSV file files.push(...csvFiles.map((file) => join(directory, file))); - } catch (err) { + } catch (error) { logger.error(colors.red(`Failed to read directory: ${directory}`)); - logger.error(colors.red((err as Error).message)); + logger.error(colors.red((error as Error).message)); process.exit(1); } } else { try { // Verify file exists and is a CSV - if (!file.endsWith('.csv')) { - logger.error(colors.red('File must be a CSV file')); + if (!file.endsWith(".csv")) { + logger.error(colors.red("File must be a CSV file")); process.exit(1); } files.push(file); - } catch (err) { + } catch (error) { logger.error(colors.red(`Failed to access file: ${file}`)); - logger.error(colors.red((err as Error).message)); + logger.error(colors.red((error as Error).message)); process.exit(1); } } logger.info( colors.green( - `Processing ${files.length} consent preferences files for partition: ${partition}`, - ), + `Processing ${files.length} consent preferences files for partition: ${partition}` + ) ); - logger.debug(`Files to process: ${files.join(', ')}`); + logger.debug(`Files to process: ${files.join(", ")}`); if (skipExistingRecordCheck) { logger.info( colors.bgYellow( - `Skipping existing record check: ${skipExistingRecordCheck}`, - ), + `Skipping existing record check: ${skipExistingRecordCheck}` + ) ); } await map( files, async (filePath) => { - const fileName = basename(filePath).replace('.csv', ''); + const fileName = basename(filePath).replace(".csv", ""); await uploadPreferenceManagementPreferencesInteractive({ receiptFilepath: join(receiptFileDir, `${fileName}-receipts.json`), auth, @@ -135,6 +135,6 @@ export async function uploadPreferences( forceTriggerWorkflows, }); }, - { concurrency }, + { concurrency } ); } diff --git a/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts b/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts index bd02f44c..ad41b2e9 100644 --- a/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts +++ b/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts @@ -1,19 +1,19 @@ -import { existsSync, readFileSync } from 'fs'; +import { existsSync, readFileSync } from "node:fs"; import { ConsentTrackerStatus, DataFlowScope, -} from '@transcend-io/privacy-types'; -import { decodeCodec } from '@transcend-io/type-utils'; -import colors from 'colors'; -import * as t from 'io-ts'; +} from "@transcend-io/privacy-types"; +import { decodeCodec } from "@transcend-io/type-utils"; +import colors from "colors"; +import * as t from "io-ts"; import { ConsentManagerServiceMetadata, CookieInput, DataFlowInput, -} from '../../../codecs'; -import type { LocalContext } from '../../../context'; -import { writeTranscendYaml } from '../../../lib/readTranscendYaml'; -import { logger } from '../../../logger'; +} from "../../../codecs"; +import type { LocalContext } from "../../../context"; +import { writeTranscendYaml } from "../../../lib/readTranscendYaml"; +import { logger } from "../../../logger"; interface ConsentManagerServiceJsonToYmlCommandFlags { file: string; @@ -22,7 +22,7 @@ interface ConsentManagerServiceJsonToYmlCommandFlags { export function consentManagerServiceJsonToYml( this: LocalContext, - { file, output }: ConsentManagerServiceJsonToYmlCommandFlags, + { file, output }: ConsentManagerServiceJsonToYmlCommandFlags ): void { // Ensure files exist if (!existsSync(file)) { @@ -33,42 +33,42 @@ export function consentManagerServiceJsonToYml( // Read in each consent manager configuration const services = decodeCodec( t.array(ConsentManagerServiceMetadata), - readFileSync(file, 'utf-8'), + readFileSync(file, "utf-8") ); // Create data flows and cookie configurations const dataFlows: DataFlowInput[] = []; const cookies: CookieInput[] = []; - services.forEach((service) => { - service.dataFlows - .filter(({ type }) => type !== DataFlowScope.CSP) - .forEach((dataFlow) => { - dataFlows.push({ - value: dataFlow.value, - type: dataFlow.type, - status: ConsentTrackerStatus.Live, - trackingPurposes: dataFlow.trackingPurposes, - }); + for (const service of services) { + for (const dataFlow of service.dataFlows.filter( + ({ type }) => type !== DataFlowScope.CSP + )) { + dataFlows.push({ + value: dataFlow.value, + type: dataFlow.type, + status: ConsentTrackerStatus.Live, + trackingPurposes: dataFlow.trackingPurposes, }); + } - service.cookies.forEach((cookie) => { + for (const cookie of service.cookies) { cookies.push({ name: cookie.name, status: ConsentTrackerStatus.Live, trackingPurposes: cookie.trackingPurposes, }); - }); - }); + } + } // write to disk writeTranscendYaml(output, { - 'data-flows': dataFlows, + "data-flows": dataFlows, cookies, }); logger.info( colors.green( - `Successfully wrote ${dataFlows.length} data flows and ${cookies.length} cookies to file "${output}"`, - ), + `Successfully wrote ${dataFlows.length} data flows and ${cookies.length} cookies to file "${output}"` + ) ); } diff --git a/src/commands/inventory/consent-managers-to-business-entities/impl.ts b/src/commands/inventory/consent-managers-to-business-entities/impl.ts index 090b203b..a504d375 100644 --- a/src/commands/inventory/consent-managers-to-business-entities/impl.ts +++ b/src/commands/inventory/consent-managers-to-business-entities/impl.ts @@ -1,14 +1,14 @@ -import { existsSync, lstatSync } from 'fs'; -import { join } from 'path'; -import colors from 'colors'; -import type { LocalContext } from '../../../context'; -import { listFiles } from '../../../lib/api-keys'; -import { consentManagersToBusinessEntities as consentManagersToBusinessEntitiesHelper } from '../../../lib/consent-manager'; +import { existsSync, lstatSync } from "node:fs"; +import { join } from "node:path"; +import colors from "colors"; +import type { LocalContext } from "../../../context"; +import { listFiles } from "../../../lib/api-keys"; +import { consentManagersToBusinessEntities as consentManagersToBusinessEntitiesHelper } from "../../../lib/consent-manager"; import { readTranscendYaml, writeTranscendYaml, -} from '../../../lib/readTranscendYaml'; -import { logger } from '../../../logger'; +} from "../../../lib/readTranscendYaml"; +import { logger } from "../../../logger"; interface ConsentManagersToBusinessEntitiesCommandFlags { consentManagerYmlFolder: string; @@ -20,7 +20,7 @@ export function consentManagersToBusinessEntities( { consentManagerYmlFolder, output, - }: ConsentManagersToBusinessEntitiesCommandFlags, + }: ConsentManagersToBusinessEntitiesCommandFlags ): void { // Ensure folder is passed if ( @@ -28,15 +28,15 @@ export function consentManagersToBusinessEntities( !lstatSync(consentManagerYmlFolder).isDirectory() ) { logger.error( - colors.red(`Folder does not exist: "${consentManagerYmlFolder}"`), + colors.red(`Folder does not exist: "${consentManagerYmlFolder}"`) ); process.exit(1); } // Read in each consent manager configuration const inputs = listFiles(consentManagerYmlFolder).map((directory) => { - const { 'consent-manager': consentManager } = readTranscendYaml( - join(consentManagerYmlFolder, directory), + const { "consent-manager": consentManager } = readTranscendYaml( + join(consentManagerYmlFolder, directory) ); return { name: directory, input: consentManager }; }); @@ -46,12 +46,12 @@ export function consentManagersToBusinessEntities( // write to disk writeTranscendYaml(output, { - 'business-entities': businessEntities, + "business-entities": businessEntities, }); logger.info( colors.green( - `Successfully wrote ${businessEntities.length} business entities to file "${output}"`, - ), + `Successfully wrote ${businessEntities.length} business entities to file "${output}"` + ) ); } diff --git a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts index 9ed62e77..0002d8f0 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts @@ -1,20 +1,20 @@ -import { existsSync, lstatSync } from 'fs'; -import { join } from 'path'; -import colors from 'colors'; -import { difference } from 'lodash-es'; -import { DataFlowInput } from '../../../codecs'; -import type { LocalContext } from '../../../context'; -import { listFiles } from '../../../lib/api-keys'; -import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; +import { existsSync, lstatSync } from "node:fs"; +import { join } from "node:path"; +import colors from "colors"; +import { difference } from "lodash-es"; +import { DataFlowInput } from "../../../codecs"; +import type { LocalContext } from "../../../context"; +import { listFiles } from "../../../lib/api-keys"; +import { dataFlowsToDataSilos } from "../../../lib/consent-manager/dataFlowsToDataSilos"; import { buildTranscendGraphQLClient, fetchAndIndexCatalogs, -} from '../../../lib/graphql'; +} from "../../../lib/graphql"; import { readTranscendYaml, writeTranscendYaml, -} from '../../../lib/readTranscendYaml'; -import { logger } from '../../../logger'; +} from "../../../lib/readTranscendYaml"; +import { logger } from "../../../logger"; interface DeriveDataSilosFromDataFlowsCrossInstanceCommandFlags { auth: string; @@ -32,14 +32,14 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( output, ignoreYmls = [], transcendUrl, - }: DeriveDataSilosFromDataFlowsCrossInstanceCommandFlags, + }: DeriveDataSilosFromDataFlowsCrossInstanceCommandFlags ): Promise { // Ensure folder is passed to dataFlowsYmlFolder if (!dataFlowsYmlFolder) { logger.error( colors.red( - 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', - ), + "Missing required arg: --dataFlowsYmlFolder=./working/data-flows/" + ) ); process.exit(1); } @@ -54,58 +54,58 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( } // Ignore the data flows in these yml files - const instancesToIgnore = ignoreYmls.map((x) => x.split('.')[0]); + const instancesToIgnore = ignoreYmls.map((x) => x.split(".")[0]); // Map over each data flow yml file and convert to data silo configurations const dataSiloInputs = listFiles(dataFlowsYmlFolder).map((directory) => { // read in the data flows for a specific instance - const { 'data-flows': dataFlows = [] } = readTranscendYaml( - join(dataFlowsYmlFolder, directory), + const { "data-flows": dataFlows = [] } = readTranscendYaml( + join(dataFlowsYmlFolder, directory) ); // map the data flows to data silos const { adTechDataSilos, siteTechDataSilos } = dataFlowsToDataSilos( - dataFlows as DataFlowInput[], + dataFlows, { serviceToSupportedIntegration, serviceToTitle, - }, + } ); return { adTechDataSilos, siteTechDataSilos, - organizationName: directory.split('.')[0], + organizationName: directory.split(".")[0], }; }); // Mapping from service name to instances that have that service - const serviceToInstance: { [k in string]: string[] } = {}; - dataSiloInputs.forEach( - ({ adTechDataSilos, siteTechDataSilos, organizationName }) => { - const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos]; - allDataSilos.forEach((dataSilo) => { - const service = dataSilo['outer-type'] || dataSilo.integrationName; - // create mapping to instance - if (!serviceToInstance[service]) { - serviceToInstance[service] = []; - } - serviceToInstance[service]!.push(organizationName); - serviceToInstance[service] = [...new Set(serviceToInstance[service])]; - }); - }, - ); + const serviceToInstance: Record = {}; + for (const { + adTechDataSilos, + siteTechDataSilos, + organizationName, + } of dataSiloInputs) { + const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos]; + for (const dataSilo of allDataSilos) { + const service = dataSilo["outer-type"] || dataSilo.integrationName; + // create mapping to instance + if (!serviceToInstance[service]) { + serviceToInstance[service] = []; + } + serviceToInstance[service].push(organizationName); + serviceToInstance[service] = [...new Set(serviceToInstance[service])]; + } + } // List of ad tech integrations const adTechIntegrations = [ ...new Set( - dataSiloInputs - .map(({ adTechDataSilos }) => - adTechDataSilos.map( - (silo) => silo['outer-type'] || silo.integrationName, - ), + dataSiloInputs.flatMap(({ adTechDataSilos }) => + adTechDataSilos.map( + (silo) => silo["outer-type"] || silo.integrationName ) - .flat(), + ) ), ]; @@ -113,37 +113,35 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( const siteTechIntegrations = difference( [ ...new Set( - dataSiloInputs - .map(({ siteTechDataSilos }) => - siteTechDataSilos.map( - (silo) => silo['outer-type'] || silo.integrationName, - ), + dataSiloInputs.flatMap(({ siteTechDataSilos }) => + siteTechDataSilos.map( + (silo) => silo["outer-type"] || silo.integrationName ) - .flat(), + ) ), ], - adTechIntegrations, + adTechIntegrations ); // Mapping from service name to list of - const serviceToFoundOnDomain: { [k in string]: string[] } = {}; - dataSiloInputs.forEach(({ adTechDataSilos, siteTechDataSilos }) => { + const serviceToFoundOnDomain: Record = {}; + for (const { adTechDataSilos, siteTechDataSilos } of dataSiloInputs) { const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos]; - allDataSilos.forEach((dataSilo) => { - const service = dataSilo['outer-type'] || dataSilo.integrationName; + for (const dataSilo of allDataSilos) { + const service = dataSilo["outer-type"] || dataSilo.integrationName; const foundOnDomain = dataSilo.attributes?.find( - (attr) => attr.key === 'Found On Domain', + (attribute) => attribute.key === "Found On Domain" ); // create mapping to instance if (!serviceToFoundOnDomain[service]) { serviceToFoundOnDomain[service] = []; } - serviceToFoundOnDomain[service]!.push(...(foundOnDomain?.values || [])); + serviceToFoundOnDomain[service].push(...(foundOnDomain?.values || [])); serviceToFoundOnDomain[service] = [ ...new Set(serviceToFoundOnDomain[service]), ]; - }); - }); + } + } // Fetch all integrations in the catalog const client = buildTranscendGraphQLClient(transcendUrl, auth); @@ -156,25 +154,25 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( title: serviceToTitle[service], ...(serviceToSupportedIntegration[service] ? { integrationName: service } - : { integrationName: 'promptAPerson', 'outer-type': service }), + : { integrationName: "promptAPerson", "outer-type": service }), attributes: [ { - key: 'Tech Type', - values: ['Ad Tech'], + key: "Tech Type", + values: ["Ad Tech"], }, { - key: 'Business Units', + key: "Business Units", values: difference( serviceToInstance[service] || [], - instancesToIgnore, + instancesToIgnore ), }, { - key: 'Found On Domain', + key: "Found On Domain", values: serviceToFoundOnDomain[service] || [], }, ], - }), + }) ); // Log output @@ -184,6 +182,6 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( // Write to yaml writeTranscendYaml(output, { - 'data-silos': dataSilos, + "data-silos": dataSilos, }); } diff --git a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts index 7f7112fa..5c6e642e 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts @@ -1,19 +1,19 @@ -import { existsSync, lstatSync } from 'fs'; -import { join } from 'path'; -import colors from 'colors'; -import { DataFlowInput } from '../../../codecs'; -import type { LocalContext } from '../../../context'; -import { listFiles } from '../../../lib/api-keys'; -import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; +import { existsSync, lstatSync } from "node:fs"; +import { join } from "node:path"; +import colors from "colors"; +import { DataFlowInput } from "../../../codecs"; +import type { LocalContext } from "../../../context"; +import { listFiles } from "../../../lib/api-keys"; +import { dataFlowsToDataSilos } from "../../../lib/consent-manager/dataFlowsToDataSilos"; import { buildTranscendGraphQLClient, fetchAndIndexCatalogs, -} from '../../../lib/graphql'; +} from "../../../lib/graphql"; import { readTranscendYaml, writeTranscendYaml, -} from '../../../lib/readTranscendYaml'; -import { logger } from '../../../logger'; +} from "../../../lib/readTranscendYaml"; +import { logger } from "../../../logger"; interface DeriveDataSilosFromDataFlowsCommandFlags { auth: string; @@ -31,14 +31,14 @@ export async function deriveDataSilosFromDataFlows( dataSilosYmlFolder, ignoreYmls = [], transcendUrl, - }: DeriveDataSilosFromDataFlowsCommandFlags, + }: DeriveDataSilosFromDataFlowsCommandFlags ): Promise { // Ensure folder is passed to dataFlowsYmlFolder if (!dataFlowsYmlFolder) { logger.error( colors.red( - 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', - ), + "Missing required arg: --dataFlowsYmlFolder=./working/data-flows/" + ) ); process.exit(1); } @@ -56,8 +56,8 @@ export async function deriveDataSilosFromDataFlows( if (!dataSilosYmlFolder) { logger.error( colors.red( - 'Missing required arg: --dataSilosYmlFolder=./working/data-silos/', - ), + "Missing required arg: --dataSilosYmlFolder=./working/data-silos/" + ) ); process.exit(1); } @@ -77,19 +77,19 @@ export async function deriveDataSilosFromDataFlows( await fetchAndIndexCatalogs(client); // List of each data flow yml file - listFiles(dataFlowsYmlFolder).forEach((directory) => { + for (const directory of listFiles(dataFlowsYmlFolder)) { // read in the data flows for a specific instance - const { 'data-flows': dataFlows = [] } = readTranscendYaml( - join(dataFlowsYmlFolder, directory), + const { "data-flows": dataFlows = [] } = readTranscendYaml( + join(dataFlowsYmlFolder, directory) ); // map the data flows to data silos const { adTechDataSilos, siteTechDataSilos } = dataFlowsToDataSilos( - dataFlows as DataFlowInput[], + dataFlows, { serviceToSupportedIntegration, serviceToTitle, - }, + } ); // combine and write to yml file @@ -98,7 +98,7 @@ export async function deriveDataSilosFromDataFlows( logger.log(`Ad Tech Services: ${adTechDataSilos.length}`); logger.log(`Site Tech Services: ${siteTechDataSilos.length}`); writeTranscendYaml(join(dataSilosYmlFolder, directory), { - 'data-silos': ignoreYmls.includes(directory) ? [] : dataSilos, + "data-silos": ignoreYmls.includes(directory) ? [] : dataSilos, }); - }); + } } diff --git a/src/commands/inventory/pull-datapoints/impl.ts b/src/commands/inventory/pull-datapoints/impl.ts index 0bab667d..fa968a75 100644 --- a/src/commands/inventory/pull-datapoints/impl.ts +++ b/src/commands/inventory/pull-datapoints/impl.ts @@ -1,12 +1,12 @@ -import { DataCategoryType } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { groupBy, uniq } from 'lodash-es'; -import { ADMIN_DASH_DATAPOINTS } from '../../../constants'; -import type { LocalContext } from '../../../context'; -import { writeCsv } from '../../../lib/cron'; -import { pullAllDatapoints } from '../../../lib/data-inventory'; -import { buildTranscendGraphQLClient } from '../../../lib/graphql'; -import { logger } from '../../../logger'; +import { DataCategoryType } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { groupBy, uniq } from "lodash-es"; +import { ADMIN_DASH_DATAPOINTS } from "../../../constants"; +import type { LocalContext } from "../../../context"; +import { writeCsv } from "../../../lib/cron"; +import { pullAllDatapoints } from "../../../lib/data-inventory"; +import { buildTranscendGraphQLClient } from "../../../lib/graphql"; +import { logger } from "../../../logger"; interface PullDatapointsCommandFlags { auth: string; @@ -30,7 +30,7 @@ export async function pullDatapoints( includeGuessedCategories, parentCategories, subCategories = [], - }: PullDatapointsCommandFlags, + }: PullDatapointsCommandFlags ): Promise { try { // Create a GraphQL client @@ -48,43 +48,38 @@ export async function pullDatapoints( let headers: string[] = []; const inputs = dataPoints.map((point) => { const result = { - 'Property ID': point.id, - 'Data Silo': point.dataSilo.title, + "Property ID": point.id, + "Data Silo": point.dataSilo.title, Object: point.dataPoint.name, - 'Object Path': point.dataPoint.path.join('.'), + "Object Path": point.dataPoint.path.join("."), Property: point.name, - 'Property Description': point.description, - 'Data Categories': point.categories + "Property Description": point.description, + "Data Categories": point.categories .map((category) => `${category.category}:${category.name}`) - .join(', '), - 'Guessed Category': point.pendingCategoryGuesses?.[0] - ? `${point.pendingCategoryGuesses![0]!.category.category}:${ - point.pendingCategoryGuesses![0]!.category.name - }` - : '', - 'Processing Purposes': point.purposes + .join(", "), + "Guessed Category": point.pendingCategoryGuesses?.[0] + ? `${point.pendingCategoryGuesses[0].category.category}:${point.pendingCategoryGuesses[0].category.name}` + : "", + "Processing Purposes": point.purposes .map((purpose) => `${purpose.purpose}:${purpose.name}`) - .join(', '), + .join(", "), ...Object.entries( groupBy( point.attributeValues || [], - ({ attributeKey }) => attributeKey.name, - ), - ).reduce( - (acc, [key, values]) => { - acc[key] = values.map((value) => value.name).join(','); - return acc; - }, - {} as Record, - ), + ({ attributeKey }) => attributeKey.name + ) + ).reduce>((accumulator, [key, values]) => { + accumulator[key] = values.map((value) => value.name).join(","); + return accumulator; + }, {}), }; headers = uniq([...headers, ...Object.keys(result)]); return result; }); writeCsv(file, inputs, headers); - } catch (err) { + } catch (error) { logger.error( - colors.red(`An error occurred syncing the datapoints: ${err.message}`), + colors.red(`An error occurred syncing the datapoints: ${error.message}`) ); process.exit(1); } @@ -92,7 +87,7 @@ export async function pullDatapoints( // Indicate success logger.info( colors.green( - `Successfully synced datapoints to disk at ${file}! View at ${ADMIN_DASH_DATAPOINTS}`, - ), + `Successfully synced datapoints to disk at ${file}! View at ${ADMIN_DASH_DATAPOINTS}` + ) ); } diff --git a/src/commands/inventory/pull-unstructured-discovery-files/impl.ts b/src/commands/inventory/pull-unstructured-discovery-files/impl.ts index a66d0358..001ed4b0 100644 --- a/src/commands/inventory/pull-unstructured-discovery-files/impl.ts +++ b/src/commands/inventory/pull-unstructured-discovery-files/impl.ts @@ -1,11 +1,11 @@ -import type { UnstructuredSubDataPointRecommendationStatus } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { uniq } from 'lodash-es'; -import type { LocalContext } from '../../../context'; -import { writeCsv } from '../../../lib/cron'; -import { pullUnstructuredSubDataPointRecommendations } from '../../../lib/data-inventory'; -import { buildTranscendGraphQLClient } from '../../../lib/graphql'; -import { logger } from '../../../logger'; +import type { UnstructuredSubDataPointRecommendationStatus } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { uniq } from "lodash-es"; +import type { LocalContext } from "../../../context"; +import { writeCsv } from "../../../lib/cron"; +import { pullUnstructuredSubDataPointRecommendations } from "../../../lib/data-inventory"; +import { buildTranscendGraphQLClient } from "../../../lib/graphql"; +import { logger } from "../../../logger"; interface PullUnstructuredDiscoveryFilesCommandFlags { auth: string; @@ -27,7 +27,7 @@ export async function pullUnstructuredDiscoveryFiles( subCategories, status, includeEncryptedSnippets, - }: PullUnstructuredDiscoveryFilesCommandFlags, + }: PullUnstructuredDiscoveryFilesCommandFlags ): Promise { try { // Create a GraphQL client @@ -42,34 +42,34 @@ export async function pullUnstructuredDiscoveryFiles( logger.info( colors.magenta( - `Writing unstructured discovery files to file "${file}"...`, - ), + `Writing unstructured discovery files to file "${file}"...` + ) ); let headers: string[] = []; const inputs = entries.map((entry) => { const result = { - 'Entry ID': entry.id, - 'Data Silo ID': entry.dataSiloId, - 'Object Path ID': entry.scannedObjectPathId, - 'Object ID': entry.scannedObjectId, + "Entry ID": entry.id, + "Data Silo ID": entry.dataSiloId, + "Object Path ID": entry.scannedObjectPathId, + "Object ID": entry.scannedObjectId, ...(includeEncryptedSnippets - ? { Entry: entry.name, 'Context Snippet': entry.contextSnippet } + ? { Entry: entry.name, "Context Snippet": entry.contextSnippet } : {}), - 'Data Category': `${entry.dataSubCategory.category}:${entry.dataSubCategory.name}`, - 'Classification Status': entry.status, - 'Confidence Score': entry.confidence, - 'Classification Method': entry.classificationMethod, - 'Classifier Version': entry.classifierVersion, + "Data Category": `${entry.dataSubCategory.category}:${entry.dataSubCategory.name}`, + "Classification Status": entry.status, + "Confidence Score": entry.confidence, + "Classification Method": entry.classificationMethod, + "Classifier Version": entry.classifierVersion, }; headers = uniq([...headers, ...Object.keys(result)]); return result; }); writeCsv(file, inputs, headers); - } catch (err) { + } catch (error) { logger.error( colors.red( - `An error occurred syncing the unstructured discovery files: ${err.message}`, - ), + `An error occurred syncing the unstructured discovery files: ${error.message}` + ) ); process.exit(1); } @@ -77,7 +77,7 @@ export async function pullUnstructuredDiscoveryFiles( // Indicate success logger.info( colors.green( - `Successfully synced unstructured discovery files to disk at ${file}!`, - ), + `Successfully synced unstructured discovery files to disk at ${file}!` + ) ); } diff --git a/src/commands/inventory/pull/impl.ts b/src/commands/inventory/pull/impl.ts index 1bfa37ab..799d574b 100644 --- a/src/commands/inventory/pull/impl.ts +++ b/src/commands/inventory/pull/impl.ts @@ -1,26 +1,26 @@ -import fs from 'fs'; -import { join } from 'path'; -import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; -import type { LocalContext } from '../../../context'; -import { TranscendPullResource } from '../../../enums'; -import { validateTranscendAuth } from '../../../lib/api-keys'; -import { mapSeries } from '../../../lib/bluebird-replace'; +import fs from "node:fs"; +import { join } from "node:path"; +import { ConsentTrackerStatus } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { ADMIN_DASH_INTEGRATIONS } from "../../../constants"; +import type { LocalContext } from "../../../context"; +import { TranscendPullResource } from "../../../enums"; +import { validateTranscendAuth } from "../../../lib/api-keys"; +import { mapSeries } from "../../../lib/bluebird-replace"; import { buildTranscendGraphQLClient, pullTranscendConfiguration, -} from '../../../lib/graphql'; -import { writeTranscendYaml } from '../../../lib/readTranscendYaml'; -import { logger } from '../../../logger'; +} from "../../../lib/graphql"; +import { writeTranscendYaml } from "../../../lib/readTranscendYaml"; +import { logger } from "../../../logger"; import { DEFAULT_CONSENT_TRACKER_STATUSES, DEFAULT_TRANSCEND_PULL_RESOURCES, -} from './command'; +} from "./command"; interface PullCommandFlags { auth: string; - resources?: (TranscendPullResource | 'all')[]; + resources?: (TranscendPullResource | "all")[]; file: string; transcendUrl: string; dataSiloIds?: string[]; @@ -48,17 +48,17 @@ export async function pull( skipSubDatapoints, includeGuessedCategories, debug, - }: PullCommandFlags, + }: PullCommandFlags ): Promise { // Parse authentication as API key or path to list of API keys const apiKeyOrList = await validateTranscendAuth(auth); - const resourcesToPull: TranscendPullResource[] = resources.includes('all') + const resourcesToPull: TranscendPullResource[] = resources.includes("all") ? Object.values(TranscendPullResource) : (resources as TranscendPullResource[]); // Sync to Disk - if (typeof apiKeyOrList === 'string') { + if (typeof apiKeyOrList === "string") { try { // Create a GraphQL client const client = buildTranscendGraphQLClient(transcendUrl, apiKeyOrList); @@ -77,13 +77,13 @@ export async function pull( logger.info(colors.magenta(`Writing configuration to file "${file}"...`)); writeTranscendYaml(file, configuration); - } catch (err) { + } catch (error) { logger.error( colors.red( `An error occurred syncing the schema: ${ - debug ? err.stack : err.message - }`, - ), + debug ? error.stack : error.message + }` + ) ); process.exit(1); } @@ -91,13 +91,13 @@ export async function pull( // Indicate success logger.info( colors.green( - `Successfully synced yaml file to disk at ${file}! View at ${ADMIN_DASH_INTEGRATIONS}`, - ), + `Successfully synced yaml file to disk at ${file}! View at ${ADMIN_DASH_INTEGRATIONS}` + ) ); } else { if (!fs.lstatSync(file).isDirectory()) { throw new Error( - 'File is expected to be a folder when passing in a list of API keys to pull from. e.g. --file=./working/', + "File is expected to be a folder when passing in a list of API keys to pull from. e.g. --file=./working/" ); } @@ -108,8 +108,8 @@ export async function pull( }] `; logger.info( colors.magenta( - `~~~\n\n${prefix}Attempting to pull configuration...\n\n~~~`, - ), + `~~~\n\n${prefix}Attempting to pull configuration...\n\n~~~` + ) ); // Create a GraphQL client @@ -130,16 +130,18 @@ export async function pull( const filePath = join(file, `${apiKey.organizationName}.yml`); logger.info( - colors.magenta(`Writing configuration to file "${filePath}"...`), + colors.magenta(`Writing configuration to file "${filePath}"...`) ); writeTranscendYaml(filePath, configuration); logger.info( - colors.green(`${prefix}Successfully pulled configuration!`), + colors.green(`${prefix}Successfully pulled configuration!`) ); - } catch (err) { + } catch (error) { logger.error( - colors.red(`${prefix}Failed to sync configuration. - ${err.message}`), + colors.red( + `${prefix}Failed to sync configuration. - ${error.message}` + ) ); encounteredErrors.push(apiKey.organizationName); } @@ -149,9 +151,9 @@ export async function pull( logger.info( colors.red( `Sync encountered errors for "${encounteredErrors.join( - ',', - )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}`, - ), + "," + )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}` + ) ); process.exit(1); diff --git a/src/commands/inventory/push/impl.ts b/src/commands/inventory/push/impl.ts index 7bac297a..d95b7633 100644 --- a/src/commands/inventory/push/impl.ts +++ b/src/commands/inventory/push/impl.ts @@ -1,19 +1,19 @@ -import { existsSync, lstatSync } from 'fs'; -import { join } from 'path'; -import colors from 'colors'; -import { TranscendInput } from '../../../codecs'; -import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; -import type { LocalContext } from '../../../context'; -import { listFiles, validateTranscendAuth } from '../../../lib/api-keys'; -import { mapSeries } from '../../../lib/bluebird-replace'; +import { existsSync, lstatSync } from "node:fs"; +import { join } from "node:path"; +import colors from "colors"; +import { TranscendInput } from "../../../codecs"; +import { ADMIN_DASH_INTEGRATIONS } from "../../../constants"; +import type { LocalContext } from "../../../context"; +import { listFiles, validateTranscendAuth } from "../../../lib/api-keys"; +import { mapSeries } from "../../../lib/bluebird-replace"; import { buildTranscendGraphQLClient, syncConfigurationToTranscend, -} from '../../../lib/graphql'; -import { parseVariablesFromString } from '../../../lib/helpers/parseVariablesFromString'; -import { mergeTranscendInputs } from '../../../lib/mergeTranscendInputs'; -import { readTranscendYaml } from '../../../lib/readTranscendYaml'; -import { logger } from '../../../logger'; +} from "../../../lib/graphql"; +import { parseVariablesFromString } from "../../../lib/helpers/parseVariablesFromString"; +import { mergeTranscendInputs } from "../../../lib/mergeTranscendInputs"; +import { readTranscendYaml } from "../../../lib/readTranscendYaml"; +import { logger } from "../../../logger"; /** * Sync configuration to Transcend @@ -57,14 +57,14 @@ async function syncConfiguration({ publishToPrivacyCenter, classifyService, deleteExtraAttributeValues, - }, + } ); return !encounteredError; - } catch (err) { + } catch (error) { logger.error( colors.red( - `An unexpected error occurred syncing the schema: ${err.message}`, - ), + `An unexpected error occurred syncing the schema: ${error.message}` + ) ); return false; } @@ -84,7 +84,7 @@ interface PushCommandFlags { export async function push( this: LocalContext, { - file = './transcend.yml', + file = "./transcend.yml", transcendUrl, auth, variables, @@ -92,61 +92,59 @@ export async function push( publishToPrivacyCenter, classifyService, deleteExtraAttributeValues, - }: PushCommandFlags, + }: PushCommandFlags ): Promise { // Parse authentication as API key or path to list of API keys const apiKeyOrList = await validateTranscendAuth(auth); // Parse out the variables - const vars = parseVariablesFromString(variables); + const variables_ = parseVariablesFromString(variables); // check if we are being passed a list of API keys and a list of files let fileList: string[]; - if (Array.isArray(apiKeyOrList) && lstatSync(file).isDirectory()) { - fileList = listFiles(file).map((filePath) => join(file, filePath)); - } else { - fileList = file.split(','); - } + fileList = + Array.isArray(apiKeyOrList) && lstatSync(file).isDirectory() + ? listFiles(file).map((filePath) => join(file, filePath)) + : file.split(","); // Ensure at least one file is parsed - if (fileList.length < 1) { - throw new Error('No file specified!'); + if (fileList.length === 0) { + throw new Error("No file specified!"); } - // eslint-disable-next-line array-callback-return,consistent-return const transcendInputs = fileList.map((filePath) => { // Ensure yaml file exists on disk - if (!existsSync(filePath)) { + if (existsSync(filePath)) { + logger.info(colors.magenta(`Reading file "${filePath}"...`)); + } else { logger.error( colors.red( - `The file path does not exist on disk: ${filePath}. You can specify the filepath using --file=./examples/transcend.yml`, - ), + `The file path does not exist on disk: ${filePath}. You can specify the filepath using --file=./examples/transcend.yml` + ) ); process.exit(1); - } else { - logger.info(colors.magenta(`Reading file "${filePath}"...`)); } try { // Read in the yaml file and validate it's shape - const newContents = readTranscendYaml(filePath, vars); + const newContents = readTranscendYaml(filePath, variables_); logger.info(colors.green(`Successfully read in "${filePath}"`)); return { content: newContents, - name: filePath.split('/').pop()!.replace('.yml', ''), + name: filePath.split("/").pop()!.replace(".yml", ""), }; - } catch (err) { + } catch (error) { logger.error( colors.red( - `The shape of your yaml file is invalid with the following errors: ${err.message}`, - ), + `The shape of your yaml file is invalid with the following errors: ${error.message}` + ) ); process.exit(1); } }); // process a single API key - if (typeof apiKeyOrList === 'string') { + if (typeof apiKeyOrList === "string") { // if passed multiple inputs, merge them together const [base, ...rest] = transcendInputs.map(({ content }) => content); const contents = mergeTranscendInputs(base, ...rest); @@ -166,8 +164,8 @@ export async function push( if (!success) { logger.info( colors.red( - `Sync encountered errors. View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}`, - ), + `Sync encountered errors. View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}` + ) ); process.exit(1); @@ -179,12 +177,12 @@ export async function push( transcendInputs.length !== apiKeyOrList.length ) { throw new Error( - 'Expected list of yml files to be equal to the list of API keys.' + + "Expected list of yml files to be equal to the list of API keys." + `Got ${transcendInputs.length} YML file${ - transcendInputs.length === 1 ? '' : 's' + transcendInputs.length === 1 ? "" : "s" } and ${apiKeyOrList.length} API key${ - apiKeyOrList.length === 1 ? '' : 's' - }`, + apiKeyOrList.length === 1 ? "" : "s" + }` ); } @@ -195,8 +193,8 @@ export async function push( }] `; logger.info( colors.magenta( - `~~~\n\n${prefix}Attempting to push configuration...\n\n~~~`, - ), + `~~~\n\n${prefix}Attempting to push configuration...\n\n~~~` + ) ); // use the merged contents if 1 yml passed, else use the contents that map to that organization @@ -204,15 +202,15 @@ export async function push( transcendInputs.length === 1 ? transcendInputs[0].content : transcendInputs.find( - (input) => input.name === apiKey.organizationName, + (input) => input.name === apiKey.organizationName )?.content; // Throw error if cannot find a yml file matching that organization name if (!useContents) { logger.error( colors.red( - `${prefix}Failed to find transcend.yml file for organization: "${apiKey.organizationName}".`, - ), + `${prefix}Failed to find transcend.yml file for organization: "${apiKey.organizationName}".` + ) ); encounteredErrors.push(apiKey.organizationName); return; @@ -230,7 +228,7 @@ export async function push( if (success) { logger.info( - colors.green(`${prefix}Successfully pushed configuration!`), + colors.green(`${prefix}Successfully pushed configuration!`) ); } else { logger.error(colors.red(`${prefix}Failed to sync configuration.`)); @@ -242,9 +240,9 @@ export async function push( logger.info( colors.red( `Sync encountered errors for "${encounteredErrors.join( - ',', - )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}`, - ), + "," + )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}` + ) ); process.exit(1); @@ -254,7 +252,7 @@ export async function push( // Indicate success logger.info( colors.green( - `Successfully synced yaml file to Transcend! View at ${ADMIN_DASH_INTEGRATIONS}`, - ), + `Successfully synced yaml file to Transcend! View at ${ADMIN_DASH_INTEGRATIONS}` + ) ); } diff --git a/src/commands/inventory/scan-packages/impl.ts b/src/commands/inventory/scan-packages/impl.ts index e06466f1..c2198f90 100644 --- a/src/commands/inventory/scan-packages/impl.ts +++ b/src/commands/inventory/scan-packages/impl.ts @@ -1,17 +1,17 @@ -import { execSync } from 'child_process'; -import colors from 'colors'; -import { ADMIN_DASH } from '../../../constants'; -import type { LocalContext } from '../../../context'; -import { findCodePackagesInFolder } from '../../../lib/code-scanning'; +import { execSync } from "node:child_process"; +import colors from "colors"; +import { ADMIN_DASH } from "../../../constants"; +import type { LocalContext } from "../../../context"; +import { findCodePackagesInFolder } from "../../../lib/code-scanning"; import { buildTranscendGraphQLClient, syncCodePackages, -} from '../../../lib/graphql'; -import { logger } from '../../../logger'; +} from "../../../lib/graphql"; +import { logger } from "../../../logger"; const REPO_ERROR = - 'A repository name must be provided. ' + - 'You can specify using --repositoryName=$REPO_NAME or by ensuring the ' + + "A repository name must be provided. " + + "You can specify using --repositoryName=$REPO_NAME or by ensuring the " + 'command "git config --get remote.origin.url" returns the name of the repository'; interface ScanPackagesCommandFlags { @@ -30,26 +30,26 @@ export async function scanPackages( ignoreDirs, repositoryName, transcendUrl, - }: ScanPackagesCommandFlags, + }: ScanPackagesCommandFlags ): Promise { // Ensure repository name is specified let gitRepositoryName = repositoryName; if (!gitRepositoryName) { try { const name = execSync( - `cd ${scanPath} && git config --get remote.origin.url`, + `cd ${scanPath} && git config --get remote.origin.url` ); // Trim and parse the URL - const url = name.toString('utf-8').trim(); - [gitRepositoryName] = !url.includes('https:') - ? (url.split(':').pop() || '').split('.') - : url.split('/').slice(3).join('/').split('.'); + const url = name.toString("utf-8").trim(); + [gitRepositoryName] = url.includes("https:") + ? url.split("/").slice(3).join("/").split(".") + : (url.split(":").pop() || "").split("."); if (!gitRepositoryName) { logger.error(colors.red(REPO_ERROR)); process.exit(1); } - } catch (err) { - logger.error(colors.red(`${REPO_ERROR} - Got error: ${err.message}`)); + } catch (error) { + logger.error(colors.red(`${REPO_ERROR} - Got error: ${error.message}`)); process.exit(1); } } @@ -68,13 +68,13 @@ export async function scanPackages( await syncCodePackages(client, results); const newUrl = new URL(ADMIN_DASH); - newUrl.pathname = '/code-scanning/code-packages'; + newUrl.pathname = "/code-scanning/code-packages"; // Indicate success logger.info( colors.green( `Scan found ${results.length} packages at ${scanPath}! ` + - `View results at '${newUrl.href}'`, - ), + `View results at '${newUrl.href}'` + ) ); } diff --git a/src/commands/migration/sync-ot/impl.ts b/src/commands/migration/sync-ot/impl.ts index a2ed2d4e..443e91e1 100644 --- a/src/commands/migration/sync-ot/impl.ts +++ b/src/commands/migration/sync-ot/impl.ts @@ -1,17 +1,17 @@ -import colors from 'colors'; -import type { LocalContext } from '../../../context'; +import colors from "colors"; +import type { LocalContext } from "../../../context"; import { OneTrustFileFormat, OneTrustPullResource, OneTrustPullSource, -} from '../../../enums'; -import { buildTranscendGraphQLClient } from '../../../lib/graphql'; -import { createOneTrustGotInstance } from '../../../lib/oneTrust'; +} from "../../../enums"; +import { buildTranscendGraphQLClient } from "../../../lib/graphql"; +import { createOneTrustGotInstance } from "../../../lib/oneTrust"; import { syncOneTrustAssessmentsFromFile, syncOneTrustAssessmentsFromOneTrust, -} from '../../../lib/oneTrust/helpers'; -import { logger } from '../../../logger'; +} from "../../../lib/oneTrust/helpers"; +import { logger } from "../../../logger"; // Command flag interface interface SyncOtCommandFlags { @@ -39,35 +39,34 @@ export async function syncOt( file, dryRun, debug, - }: SyncOtCommandFlags, + }: SyncOtCommandFlags ): Promise { // Must be able to authenticate to transcend to sync resources to it if (!dryRun && !transcendAuth) { throw new Error( - // eslint-disable-next-line no-template-curly-in-string - 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}', + 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}' ); } // If trying to sync to disk, must specify a file path if (dryRun && !file) { throw new Error( - 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json', + 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json' ); } if (file) { - const splitFile = file.split('.'); + const splitFile = file.split("."); if (splitFile.length < 2) { throw new Error( - 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.', + 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.' ); } if (splitFile.at(-1) !== OneTrustFileFormat.Json) { throw new Error( `Expected the format of the "file" parameters '${file}' to be '${ OneTrustFileFormat.Json - }', but got '${splitFile.at(-1)}'.`, + }', but got '${splitFile.at(-1)}'.` ); } } @@ -77,28 +76,28 @@ export async function syncOt( // must specify the OneTrust hostname if (!hostname) { throw new Error( - 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com', + 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com' ); } // must specify the OneTrust auth if (!oneTrustAuth) { throw new Error( - 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN', + 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN' ); } } else { // if reading the assessments from a file, must specify a file to read from if (!file) { throw new Error( - 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json', + 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json' ); } // Cannot try reading from file and save assessments to a file simultaneously if (dryRun) { throw new Error( - 'Cannot read and write to a file simultaneously.' + - ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.`, + "Cannot read and write to a file simultaneously." + + ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.` ); } } @@ -131,11 +130,11 @@ export async function syncOt( await syncOneTrustAssessmentsFromFile({ file, transcend }); } } - } catch (err) { + } catch (error) { throw new Error( `An error occurred syncing the resource ${resource} from OneTrust: ${ - debug ? err.stack : err.message - }`, + debug ? error.stack : error.message + }` ); } @@ -143,8 +142,8 @@ export async function syncOt( logger.info( colors.green( `Successfully synced OneTrust ${resource} to ${ - dryRun ? `disk at "${file}"` : 'Transcend' - }!`, - ), + dryRun ? `disk at "${file}"` : "Transcend" + }!` + ) ); } diff --git a/src/commands/request/cron/pull-identifiers/impl.ts b/src/commands/request/cron/pull-identifiers/impl.ts index 49ea95c2..140efaf2 100644 --- a/src/commands/request/cron/pull-identifiers/impl.ts +++ b/src/commands/request/cron/pull-identifiers/impl.ts @@ -1,14 +1,14 @@ -import { RequestAction } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { uniq } from 'lodash-es'; -import type { LocalContext } from '../../../../context'; +import { RequestAction } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { uniq } from "lodash-es"; +import type { LocalContext } from "../../../../context"; import { CsvFormattedIdentifier, parseFilePath, pullChunkedCustomSiloOutstandingIdentifiers, writeCsv, -} from '../../../../lib/cron'; -import { logger } from '../../../../logger'; +} from "../../../../lib/cron"; +import { logger } from "../../../../logger"; interface PullIdentifiersCommandFlags { file: string; @@ -34,13 +34,13 @@ export async function pullIdentifiers( pageLimit, skipRequestCount, chunkSize, - }: PullIdentifiersCommandFlags, + }: PullIdentifiersCommandFlags ): Promise { if (skipRequestCount) { logger.info( colors.yellow( - 'Skipping request count as requested. This may help speed up the call.', - ), + "Skipping request count as requested. This may help speed up the call." + ) ); } @@ -51,8 +51,8 @@ export async function pullIdentifiers( ) { logger.error( colors.red( - `Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${pageLimit}.`, - ), + `Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${pageLimit}.` + ) ); process.exit(1); } @@ -64,16 +64,16 @@ export async function pullIdentifiers( const numberedFileName = `${baseName}-${fileCount}${extension}`; logger.info( colors.blue( - `Saving ${chunk.length} identifiers to file "${numberedFileName}"`, - ), + `Saving ${chunk.length} identifiers to file "${numberedFileName}"` + ) ); - const headers = uniq(chunk.map((d) => Object.keys(d)).flat()); + const headers = uniq(chunk.flatMap((d) => Object.keys(d))); writeCsv(numberedFileName, chunk, headers); logger.info( colors.green( - `Successfully wrote ${chunk.length} identifiers to file "${numberedFileName}"`, - ), + `Successfully wrote ${chunk.length} identifiers to file "${numberedFileName}"` + ) ); fileCount += 1; return Promise.resolve(); diff --git a/src/commands/request/cron/pull-profiles/impl.ts b/src/commands/request/cron/pull-profiles/impl.ts index 1d791cb6..3f800a1d 100644 --- a/src/commands/request/cron/pull-profiles/impl.ts +++ b/src/commands/request/cron/pull-profiles/impl.ts @@ -1,19 +1,19 @@ -import type { RequestAction } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { uniq } from 'lodash-es'; -import type { LocalContext } from '../../../../context'; -import { map } from '../../../../lib/bluebird-replace'; +import type { RequestAction } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { uniq } from "lodash-es"; +import type { LocalContext } from "../../../../context"; +import { map } from "../../../../lib/bluebird-replace"; import { parseFilePath, pullChunkedCustomSiloOutstandingIdentifiers, writeCsv, type CsvFormattedIdentifier, -} from '../../../../lib/cron'; +} from "../../../../lib/cron"; import { buildTranscendGraphQLClient, fetchRequestFilesForRequest, -} from '../../../../lib/graphql'; -import { logger } from '../../../../logger'; +} from "../../../../lib/graphql"; +import { logger } from "../../../../logger"; interface PullProfilesCommandFlags { file: string; @@ -43,13 +43,13 @@ export async function pullProfiles( skipRequestCount, pageLimit, chunkSize, - }: PullProfilesCommandFlags, + }: PullProfilesCommandFlags ): Promise { if (skipRequestCount) { logger.info( colors.yellow( - 'Skipping request count as requested. This may help speed up the call.', - ), + "Skipping request count as requested. This may help speed up the call." + ) ); } @@ -60,8 +60,8 @@ export async function pullProfiles( ) { logger.error( colors.red( - `Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${pageLimit}.`, - ), + `Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${pageLimit}.` + ) ); process.exit(1); } @@ -95,52 +95,52 @@ export async function pullProfiles( return results.map(({ fileName, remoteId }) => { if (!remoteId) { throw new Error( - `Failed to find remoteId for ${fileName} request: ${requestId}`, + `Failed to find remoteId for ${fileName} request: ${requestId}` ); } return { RecordId: remoteId, Object: fileName - .replace('.json', '') - .split('/') + .replace(".json", "") + .split("/") .pop() - ?.replace(' Information', ''), + ?.replace(" Information", ""), Comment: - 'Customer data deletion request submitted via transcend.io', + "Customer data deletion request submitted via transcend.io", }; }); }, { concurrency: 10, - }, + } ); allTargetIdentifiersCount += results.flat().length; // Write the identifiers and target identifiers to CSV - const headers = uniq(chunk.map((d) => Object.keys(d)).flat()); + const headers = uniq(chunk.flatMap((d) => Object.keys(d))); const numberedFileName = `${baseName}-${fileCount}${extension}`; const numberedFileNameTarget = `${baseNameTarget}-${fileCount}${extensionTarget}`; writeCsv(numberedFileName, chunk, headers); logger.info( colors.green( - `Successfully wrote ${chunk.length} identifiers to file "${file}"`, - ), + `Successfully wrote ${chunk.length} identifiers to file "${file}"` + ) ); const targetIdentifiers = results.flat(); - const headers2 = uniq(targetIdentifiers.map((d) => Object.keys(d)).flat()); + const headers2 = uniq(targetIdentifiers.flatMap((d) => Object.keys(d))); writeCsv(numberedFileNameTarget, targetIdentifiers, headers2); logger.info( colors.green( - `Successfully wrote ${targetIdentifiers.length} identifiers to file "${fileTarget}"`, - ), + `Successfully wrote ${targetIdentifiers.length} identifiers to file "${fileTarget}"` + ) ); logger.info( colors.blue( - `Processed chunk of ${chunk.length} identifiers, found ${targetIdentifiers.length} target identifiers`, - ), + `Processed chunk of ${chunk.length} identifiers, found ${targetIdentifiers.length} target identifiers` + ) ); fileCount += 1; }; @@ -160,12 +160,12 @@ export async function pullProfiles( logger.info( colors.green( - `Successfully wrote ${allIdentifiersCount} identifiers to file "${file}"`, - ), + `Successfully wrote ${allIdentifiersCount} identifiers to file "${file}"` + ) ); logger.info( colors.green( - `Successfully wrote ${allTargetIdentifiersCount} identifiers to file "${fileTarget}"`, - ), + `Successfully wrote ${allTargetIdentifiersCount} identifiers to file "${fileTarget}"` + ) ); } diff --git a/src/commands/request/export/impl.ts b/src/commands/request/export/impl.ts index 0b1a939d..c40bb266 100644 --- a/src/commands/request/export/impl.ts +++ b/src/commands/request/export/impl.ts @@ -1,10 +1,10 @@ -import type { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { uniq } from 'lodash-es'; -import type { LocalContext } from '../../../context'; -import { writeCsv } from '../../../lib/cron'; -import { pullPrivacyRequests } from '../../../lib/requests'; -import { logger } from '../../../logger'; +import type { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { uniq } from "lodash-es"; +import type { LocalContext } from "../../../context"; +import { writeCsv } from "../../../lib/cron"; +import { pullPrivacyRequests } from "../../../lib/requests"; +import { logger } from "../../../logger"; interface ExportCommandFlags { auth: string; @@ -21,7 +21,7 @@ interface ExportCommandFlags { } // `export` is a reserved keyword, so we need to prefix it with an underscore -// eslint-disable-next-line no-underscore-dangle + export async function _export( this: LocalContext, { @@ -35,7 +35,7 @@ export async function _export( createdAtBefore, createdAtAfter, showTests, - }: ExportCommandFlags, + }: ExportCommandFlags ): Promise { const { requestsFormattedForCsv } = await pullPrivacyRequests({ transcendUrl, @@ -50,13 +50,11 @@ export async function _export( }); // Write to CSV - const headers = uniq( - requestsFormattedForCsv.map((d) => Object.keys(d)).flat(), - ); + const headers = uniq(requestsFormattedForCsv.flatMap((d) => Object.keys(d))); writeCsv(file, requestsFormattedForCsv, headers); logger.info( colors.green( - `Successfully wrote ${requestsFormattedForCsv.length} requests to file "${file}"`, - ), + `Successfully wrote ${requestsFormattedForCsv.length} requests to file "${file}"` + ) ); } diff --git a/src/constants.ts b/src/constants.ts index ba8b68ea..42fd2711 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,11 +1,11 @@ -import { ScopeName } from '@transcend-io/privacy-types'; -import { TranscendInput } from './codecs'; -import { TranscendPullResource } from './enums'; +import { ScopeName } from "@transcend-io/privacy-types"; +import { TranscendInput } from "./codecs"; +import { TranscendPullResource } from "./enums"; -export { description, version } from '../package.json'; -export const name = 'transcend'; +export { description, version } from "../package.json"; +export const name = "transcend"; -export const ADMIN_DASH = 'https://app.transcend.io'; +export const ADMIN_DASH = "https://app.transcend.io"; export const ADMIN_DASH_INTEGRATIONS = `${ADMIN_DASH}/infrastructure/integrations`; export const ADMIN_DASH_DATAPOINTS = `${ADMIN_DASH}/data-map/data-inventory/data-points`; @@ -15,21 +15,22 @@ export const ADMIN_DASH_DATAPOINTS = `${ADMIN_DASH}/data-map/data-inventory/data * TRANSCEND_API_URL=https://api.us.transcend.io transcend ... */ export const DEFAULT_TRANSCEND_API = - process.env.TRANSCEND_API_URL || 'https://api.transcend.io'; + process.env.TRANSCEND_API_URL || "https://api.transcend.io"; /** * Override default transcend API url using * TRANSCEND_CONSENT_API_URL=https://consent.us.transcend.io transcend ... */ export const DEFAULT_TRANSCEND_CONSENT_API = - process.env.TRANSCEND_CONSENT_API_URL || 'https://consent.transcend.io'; + process.env.TRANSCEND_CONSENT_API_URL || "https://consent.transcend.io"; /** * Mapping between resource type and scopes required for cli */ -export const TR_PUSH_RESOURCE_SCOPE_MAP: { - [k in TranscendPullResource]: ScopeName[]; -} = { +export const TR_PUSH_RESOURCE_SCOPE_MAP: Record< + TranscendPullResource, + ScopeName[] +> = { [TranscendPullResource.ApiKeys]: [ScopeName.ViewApiKeys], [TranscendPullResource.Templates]: [ScopeName.ManageEmailTemplates], [TranscendPullResource.DataSilos]: [ @@ -83,9 +84,10 @@ export const TR_PUSH_RESOURCE_SCOPE_MAP: { /** * Mapping between resource type and scopes required for cli */ -export const TR_PULL_RESOURCE_SCOPE_MAP: { - [k in TranscendPullResource]: ScopeName[]; -} = { +export const TR_PULL_RESOURCE_SCOPE_MAP: Record< + TranscendPullResource, + ScopeName[] +> = { [TranscendPullResource.ApiKeys]: [ScopeName.ViewApiKeys], [TranscendPullResource.Templates]: [ScopeName.ViewEmailTemplates], [TranscendPullResource.DataSilos]: [ @@ -131,35 +133,35 @@ export const TR_YML_RESOURCE_TO_FIELD_NAME: Record< TranscendPullResource, keyof TranscendInput > = { - [TranscendPullResource.ApiKeys]: 'api-keys', - [TranscendPullResource.Attributes]: 'attributes', - [TranscendPullResource.DataFlows]: 'data-flows', - [TranscendPullResource.Cookies]: 'cookies', - [TranscendPullResource.ConsentManager]: 'consent-manager', - [TranscendPullResource.Partitions]: 'partitions', - [TranscendPullResource.Actions]: 'actions', - [TranscendPullResource.DataSubjects]: 'data-subjects', - [TranscendPullResource.BusinessEntities]: 'business-entities', - [TranscendPullResource.Identifiers]: 'identifiers', - [TranscendPullResource.Enrichers]: 'enrichers', - [TranscendPullResource.DataSilos]: 'data-silos', - [TranscendPullResource.Templates]: 'templates', - [TranscendPullResource.Prompts]: 'prompts', - [TranscendPullResource.PromptPartials]: 'prompt-partials', - [TranscendPullResource.PromptGroups]: 'prompt-groups', - [TranscendPullResource.Agents]: 'agents', - [TranscendPullResource.AgentFunctions]: 'agent-functions', - [TranscendPullResource.AgentFiles]: 'agent-files', - [TranscendPullResource.Vendors]: 'vendors', - [TranscendPullResource.DataCategories]: 'data-categories', - [TranscendPullResource.ProcessingPurposes]: 'processing-purposes', - [TranscendPullResource.ActionItems]: 'action-items', - [TranscendPullResource.ActionItemCollections]: 'action-item-collections', - [TranscendPullResource.Teams]: 'teams', - [TranscendPullResource.Messages]: 'messages', - [TranscendPullResource.PrivacyCenters]: 'privacy-center', - [TranscendPullResource.Policies]: 'policies', - [TranscendPullResource.Assessments]: 'assessments', - [TranscendPullResource.AssessmentTemplates]: 'assessment-templates', - [TranscendPullResource.Purposes]: 'purposes', + [TranscendPullResource.ApiKeys]: "api-keys", + [TranscendPullResource.Attributes]: "attributes", + [TranscendPullResource.DataFlows]: "data-flows", + [TranscendPullResource.Cookies]: "cookies", + [TranscendPullResource.ConsentManager]: "consent-manager", + [TranscendPullResource.Partitions]: "partitions", + [TranscendPullResource.Actions]: "actions", + [TranscendPullResource.DataSubjects]: "data-subjects", + [TranscendPullResource.BusinessEntities]: "business-entities", + [TranscendPullResource.Identifiers]: "identifiers", + [TranscendPullResource.Enrichers]: "enrichers", + [TranscendPullResource.DataSilos]: "data-silos", + [TranscendPullResource.Templates]: "templates", + [TranscendPullResource.Prompts]: "prompts", + [TranscendPullResource.PromptPartials]: "prompt-partials", + [TranscendPullResource.PromptGroups]: "prompt-groups", + [TranscendPullResource.Agents]: "agents", + [TranscendPullResource.AgentFunctions]: "agent-functions", + [TranscendPullResource.AgentFiles]: "agent-files", + [TranscendPullResource.Vendors]: "vendors", + [TranscendPullResource.DataCategories]: "data-categories", + [TranscendPullResource.ProcessingPurposes]: "processing-purposes", + [TranscendPullResource.ActionItems]: "action-items", + [TranscendPullResource.ActionItemCollections]: "action-item-collections", + [TranscendPullResource.Teams]: "teams", + [TranscendPullResource.Messages]: "messages", + [TranscendPullResource.PrivacyCenters]: "privacy-center", + [TranscendPullResource.Policies]: "policies", + [TranscendPullResource.Assessments]: "assessments", + [TranscendPullResource.AssessmentTemplates]: "assessment-templates", + [TranscendPullResource.Purposes]: "purposes", }; diff --git a/src/lib/ai/TranscendPromptManager.ts b/src/lib/ai/TranscendPromptManager.ts index 5b56750c..7fed5f5d 100644 --- a/src/lib/ai/TranscendPromptManager.ts +++ b/src/lib/ai/TranscendPromptManager.ts @@ -1,29 +1,28 @@ -/* eslint-disable max-lines */ -import type { Handlebars } from '@transcend-io/handlebars-utils'; +import type { Handlebars } from "@transcend-io/handlebars-utils"; import { createHandlebarsWithHelpers, HandlebarsInput, -} from '@transcend-io/handlebars-utils'; +} from "@transcend-io/handlebars-utils"; import { ChatCompletionRole, LargeLanguageModelClient, PromptRunProductArea, PromptStatus, QueueStatus, -} from '@transcend-io/privacy-types'; -import { Secret } from '@transcend-io/secret-value'; +} from "@transcend-io/privacy-types"; +import { Secret } from "@transcend-io/secret-value"; import { apply, decodeCodec, getValues, Optionalize, Requirize, -} from '@transcend-io/type-utils'; -import { GraphQLClient } from 'graphql-request'; -import * as t from 'io-ts'; -import { chunk, groupBy, keyBy, uniq } from 'lodash-es'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { mapSeries } from '../bluebird-replace'; +} from "@transcend-io/type-utils"; +import { GraphQLClient } from "graphql-request"; +import * as t from "io-ts"; +import { chunk, groupBy, keyBy, uniq } from "lodash-es"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { mapSeries } from "../bluebird-replace"; import { Agent, AgentFile, @@ -33,27 +32,27 @@ import { fetchAllAgents, reportPromptRun, ReportPromptRunInput, -} from '../graphql'; +} from "../graphql"; import { fetchAllLargeLanguageModels, LargeLanguageModel, -} from '../graphql/fetchLargeLanguageModels'; +} from "../graphql/fetchLargeLanguageModels"; import { fetchPromptsWithVariables, TranscendPromptsAndVariables, TranscendPromptTemplated, -} from '../graphql/fetchPrompts'; +} from "../graphql/fetchPrompts"; import { fetchAllPromptThreads, PromptThread, -} from '../graphql/fetchPromptThreads'; +} from "../graphql/fetchPromptThreads"; /** * An LLM Prompt definition */ export type TranscendPrompt< - TInputParams extends t.Any, - TOutputCodec extends t.Any, + TInputParameters extends t.Any, + TOutputCodec extends t.Any > = ( | { /** ID of the prompt */ @@ -71,7 +70,7 @@ export type TranscendPrompt< /** The names of the agents that should be loaded along with the prompt */ agentNames?: string[]; /** Codec to validate runtime input shape */ - paramCodec: TInputParams; + paramCodec: TInputParameters; /** Codec to validate output response */ outputCodec: TOutputCodec; /** @@ -109,7 +108,7 @@ export function createRegexForTag(tagName: string): RegExp { */ export function defineTranscendPrompts< TPromptNames extends string, - TPrompts extends { [k in TPromptNames]: TranscendPrompt }, + TPrompts extends Record> >(prompts: TPrompts): TPrompts { return prompts; } @@ -119,24 +118,24 @@ export function defineTranscendPrompts< */ export type GetPromptParamType< TPromptName extends keyof TPrompts, - TPrompts extends { [k in TPromptName]: TranscendPrompt }, -> = t.TypeOf; + TPrompts extends Record> +> = t.TypeOf; /** * Helper to get the type of the parameter for a given prompt */ export type GetPromptResponseType< TPromptName extends keyof TPrompts, - TPrompts extends { [k in TPromptName]: TranscendPrompt }, -> = t.TypeOf; + TPrompts extends Record> +> = t.TypeOf; /** * Input for reporting a prompt run */ export interface ReportPromptRunOptions extends Optionalize< - Omit, - 'name' | 'productArea' + Omit, + "name" | "productArea" > { /** The large language model being run. Either the ID of the LLM or the client/name pairing */ largeLanguageModel: @@ -149,11 +148,11 @@ export interface ReportPromptRunOptions }; } -const jsonParseSafe = (obj: string): unknown => { +const jsonParseSafe = (object: string): unknown => { try { - return JSON.parse(obj); - } catch (e) { - return obj; + return JSON.parse(object); + } catch { + return object; } }; @@ -163,7 +162,7 @@ const jsonParseSafe = (obj: string): unknown => { */ export class TranscendPromptManager< TPromptNames extends string, - TPrompts extends { [k in TPromptNames]: TranscendPrompt }, + TPrompts extends Record> > { /** Prompt definitions */ public prompts: TPrompts; @@ -172,28 +171,28 @@ export class TranscendPromptManager< public handlebarsOptions!: HandlebarsInput; /** Prompt name -> content map, populated by call to Transcend API */ - public promptContentMap?: { [k in TPromptNames]: TranscendPromptTemplated }; + public promptContentMap?: Record; /** The large language models that are registered to this organization for reporting */ public largeLanguageModels: LargeLanguageModel[] = []; /** The agent definitions registered to this organization */ - private agentsByName: { [name in string]: Agent } = {}; + private agentsByName: Record = {}; /** The agent definitions registered to this organization */ - private agentsByAgentId: { [id in string]: Agent } = {}; + private agentsByAgentId: Record = {}; /** The GraphQL client that can be used to call Transcend */ public graphQLClient: GraphQLClient; /** The set of variables to expose in handlebars context specified at class initiation */ - public defaultVariables: { [k in string]: unknown }; + public defaultVariables: Record; /** * The set of variables to expose in handlebars context, * merges defaults with calculated variables from the inventory */ - public variables: { [k in string]: unknown }; + public variables: Record; /** Handlebars compiler */ public handlebars: typeof Handlebars; @@ -245,7 +244,7 @@ export class TranscendPromptManager< /** When true, throw an error if the prompt is not approved */ requireApproval?: boolean; /** The set of variables to expose in handlebars context specified at class initiation */ - defaultVariables?: { [k in string]: unknown }; + defaultVariables?: Record; /** * The cache duration in ms for how long prompts and associated metadata should be cached * When undefined - prompts are cached indefinitely unless explicitly re-requested @@ -261,9 +260,9 @@ export class TranscendPromptManager< this.defaultVariables = defaultVariables; this.graphQLClient = buildTranscendGraphQLClient( transcendUrl, - typeof transcendApiKey === 'object' + typeof transcendApiKey === "object" ? transcendApiKey.release() - : transcendApiKey, + : transcendApiKey ); this.requireApproval = requireApproval; this.cacheDuration = cacheDuration; @@ -278,10 +277,7 @@ export class TranscendPromptManager< */ async fetchPromptsAndMetadata(): Promise { // Determine what to fetch - const promptDefinitions = getValues(this.prompts) as TranscendPrompt< - t.Any, - t.Any - >[]; + const promptDefinitions = getValues(this.prompts); const promptIds = promptDefinitions .map(({ id }) => id) .filter((x): x is string => !!x); @@ -289,7 +285,7 @@ export class TranscendPromptManager< .map(({ title }) => title) .filter((x): x is string => !!x); const agentNames = uniq( - promptDefinitions.map(({ agentNames }) => agentNames || []).flat(), + promptDefinitions.flatMap(({ agentNames }) => agentNames || []) ); // Fetch prompts and data @@ -301,24 +297,23 @@ export class TranscendPromptManager< fetchAllLargeLanguageModels(this.graphQLClient), fetchAllAgents(this.graphQLClient, { names: agentNames }), ]); - this.agentsByName = keyBy(agents, 'name'); - this.agentsByAgentId = keyBy(agents, 'agentId'); + this.agentsByName = keyBy(agents, "name"); + this.agentsByAgentId = keyBy(agents, "agentId"); this.largeLanguageModels = largeLanguageModels.filter( - (model) => model.isTranscendHosted === false, + (model) => !model.isTranscendHosted ); // Lookup prompts by id/title - const promptByTitle = keyBy(response.prompts, 'title'); - const promptById = keyBy(response.prompts, 'id'); + const promptByTitle = keyBy(response.prompts, "title"); + const promptById = keyBy(response.prompts, "id"); // Update variables this.variables = { - ...response.calculatedVariables.reduce( - (acc, v) => - Object.assign(acc, { - [v.name]: v.data ? JSON.parse(v.data) : v.data, - }), - {}, + ...Object.fromEntries( + response.calculatedVariables.map((v) => [ + v.name, + v.data ? JSON.parse(v.data) : v.data, + ]) ), ...this.defaultVariables, }; @@ -340,11 +335,11 @@ export class TranscendPromptManager< const result = id ? promptById[id] : title - ? promptByTitle[title] - : undefined; + ? promptByTitle[title] + : undefined; if (!result) { throw new Error( - `Failed to find prompt with title: "${title}" and id: "${id}"`, + `Failed to find prompt with title: "${title}" and id: "${id}"` ); } return result; @@ -385,7 +380,7 @@ export class TranscendPromptManager< * @returns Large language model configuration */ async getPromptThreadBySlackTs( - ts: string, + ts: string ): Promise { const [thread] = await fetchAllPromptThreads(this.graphQLClient, { slackMessageTs: [ts], @@ -402,11 +397,11 @@ export class TranscendPromptManager< * @returns The agents that were found matching the names */ async getAgentsByName(names: string[]): Promise { - if (names.length < 1) { - throw new Error('Expected at least one name to be provided'); + if (names.length === 0) { + throw new Error("Expected at least one name to be provided"); } const { hasCache = [], missingCache = [] } = groupBy(names, (name) => - this.agentsByName[name] ? 'hasCache' : 'missingCache', + this.agentsByName[name] ? "hasCache" : "missingCache" ); const cachedAgents = hasCache.map((name) => this.agentsByName[name]); if (missingCache.length === 0) { @@ -418,10 +413,10 @@ export class TranscendPromptManager< const pageOfAgents = await fetchAllAgents(this.graphQLClient, { names: chunkedName, }); - pageOfAgents.forEach((agent) => { + for (const agent of pageOfAgents) { this.agentsByName[agent.name] = agent; this.agentsByAgentId[agent.agentId] = agent; - }); + } remoteAgents.push(...pageOfAgents); }); return [...cachedAgents, ...remoteAgents]; @@ -444,21 +439,21 @@ export class TranscendPromptManager< * @returns Large language model configuration */ getLargeLanguageModel( - largeLanguageModel: ReportPromptRunOptions['largeLanguageModel'], + largeLanguageModel: ReportPromptRunOptions["largeLanguageModel"] ): LargeLanguageModel { const matching = this.largeLanguageModels.find((model) => - typeof largeLanguageModel === 'string' + typeof largeLanguageModel === "string" ? model.id === largeLanguageModel : model.name === largeLanguageModel.name && - model.client === largeLanguageModel.client, + model.client === largeLanguageModel.client ); if (!matching) { throw new Error( `Failed to find model matching: ${ - typeof largeLanguageModel === 'string' + typeof largeLanguageModel === "string" ? largeLanguageModel : JSON.stringify(largeLanguageModel) - }`, + }` ); } return matching; @@ -471,7 +466,7 @@ export class TranscendPromptManager< * @returns Parsed content */ async getPromptDefinition( - promptName: TPromptName, + promptName: TPromptName ): Promise { // Determine if prompts need to be fetched if ( @@ -489,12 +484,12 @@ export class TranscendPromptManager< // Lookup prompt const { promptContentMap } = this; if (!promptContentMap) { - throw new Error('Expected this.promptContentMap to be defined'); + throw new Error("Expected this.promptContentMap to be defined"); } const promptTemplate = promptContentMap[promptName]; if (!promptTemplate) { throw new Error( - `Expected this.promptContentMap[${promptName}] to be defined`, + `Expected this.promptContentMap[${promptName}] to be defined` ); } return promptTemplate; @@ -509,7 +504,7 @@ export class TranscendPromptManager< */ async compilePrompt( promptName: TPromptName, - params: t.TypeOf, + parameters: t.TypeOf ): Promise { // Grab the prompt const promptTemplate = await this.getPromptDefinition(promptName); @@ -524,26 +519,26 @@ export class TranscendPromptManager< promptTemplate.status !== PromptStatus.Approved ) { throw new Error( - `Assessment "${promptTemplate.title}" cannot be used because its in status: "${promptTemplate.status}"`, + `Assessment "${promptTemplate.title}" cannot be used because its in status: "${promptTemplate.status}"` ); } // If prompt is rejected, throw error if (promptTemplate.status === PromptStatus.Rejected) { throw new Error( - `Assessment "${promptTemplate.title}" cannot be used because it's in status: "${promptTemplate.status}"`, + `Assessment "${promptTemplate.title}" cannot be used because it's in status: "${promptTemplate.status}"` ); } // Validate params - decodeCodec(promptInput.paramCodec, params); + decodeCodec(promptInput.paramCodec, parameters); // Compile prompt and template return this.handlebars.compile(promptTemplate.content)({ // template in currentDate by default currentDate: new Date().toISOString(), ...this.variables, - ...params, + ...parameters, }); } @@ -556,8 +551,8 @@ export class TranscendPromptManager< */ parseAiResponse( promptName: TPromptName, - response: string, - ): t.TypeOf { + response: string + ): t.TypeOf { // Look up prompt info const promptInput = this.prompts[promptName]; if (!promptInput) { @@ -574,7 +569,7 @@ export class TranscendPromptManager< return decodeCodec( promptInput.outputCodec, jsonParseSafe(extracted), - false, + false ); } @@ -587,11 +582,11 @@ export class TranscendPromptManager< */ async reportAndParsePromptRun( promptName: TPromptName, - { largeLanguageModel, ...options }: ReportPromptRunOptions, + { largeLanguageModel, ...options }: ReportPromptRunOptions ): Promise< PromptRunResult & { /** Resulting prompt run */ - result: t.TypeOf; + result: t.TypeOf; } > { // Determine if prompts need to be fetched @@ -612,7 +607,7 @@ export class TranscendPromptManager< `@transcend-io/cli-prompt-run-${new Date().toISOString()}`; if (!this.promptContentMap) { - throw new Error('Expected this.promptContentMap to be defined'); + throw new Error("Expected this.promptContentMap to be defined"); } // Look up prompt info const promptInput = this.promptContentMap[promptName]; @@ -622,38 +617,36 @@ export class TranscendPromptManager< // Ensure the first message in `promptRunMessages` is of type=system if (options.promptRunMessages.length === 0) { - throw new Error('promptRunMessages is expected to have length > 0'); + throw new Error("promptRunMessages is expected to have length > 0"); } if (options.promptRunMessages[0].role !== ChatCompletionRole.System) { throw new Error( - `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}`, + `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}` ); } if ( - options.promptRunMessages[options.promptRunMessages.length - 1].role !== - ChatCompletionRole.Assistant + options.promptRunMessages.at(-1).role !== ChatCompletionRole.Assistant ) { throw new Error( `promptRunMessages[${ options.promptRunMessages.length - 1 - }].role is expected to be = ${ChatCompletionRole.Assistant}`, + }].role is expected to be = ${ChatCompletionRole.Assistant}` ); } - const response = - options.promptRunMessages[options.promptRunMessages.length - 1].content; + const response = options.promptRunMessages.at(-1).content; - let parsed: t.TypeOf; + let parsed: t.TypeOf; try { // Parse the response parsed = this.parseAiResponse(promptName, response); - } catch (err) { + } catch (error) { await reportPromptRun(this.graphQLClient, { productArea: PromptRunProductArea.PromptManager, ...options, name, - error: err.message, + error: error.message, status: QueueStatus.Error, - ...(typeof largeLanguageModel === 'string' + ...(typeof largeLanguageModel === "string" ? { largeLanguageModelId: largeLanguageModel } : { largeLanguageModelName: largeLanguageModel.name, @@ -665,7 +658,7 @@ export class TranscendPromptManager< ...(ind === 0 ? { template: promptInput.content } : {}), })), }); - throw err; + throw error; } // report successful run @@ -674,7 +667,7 @@ export class TranscendPromptManager< ...options, name, status: QueueStatus.Resolved, - ...(typeof largeLanguageModel === 'string' + ...(typeof largeLanguageModel === "string" ? { largeLanguageModelId: largeLanguageModel } : { largeLanguageModelName: largeLanguageModel.name, @@ -706,7 +699,7 @@ export class TranscendPromptManager< { largeLanguageModel, ...options - }: Requirize, + }: Requirize ): Promise { // Determine if prompts need to be fetched if ( @@ -726,7 +719,7 @@ export class TranscendPromptManager< `@transcend-io/cli-prompt-run-${new Date().toISOString()}`; if (!this.promptContentMap) { - throw new Error('Expected this.promptContentMap to be defined'); + throw new Error("Expected this.promptContentMap to be defined"); } // Look up prompt info const promptInput = this.promptContentMap[promptName]; @@ -736,11 +729,11 @@ export class TranscendPromptManager< // Ensure the first message in `promptRunMessages` is of type=system if (options.promptRunMessages.length === 0) { - throw new Error('promptRunMessages is expected to have length > 0'); + throw new Error("promptRunMessages is expected to have length > 0"); } if (options.promptRunMessages[0].role !== ChatCompletionRole.System) { throw new Error( - `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}`, + `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}` ); } @@ -749,7 +742,7 @@ export class TranscendPromptManager< ...options, name, status: QueueStatus.Error, - ...(typeof largeLanguageModel === 'string' + ...(typeof largeLanguageModel === "string" ? { largeLanguageModelId: largeLanguageModel } : { largeLanguageModelName: largeLanguageModel.name, @@ -768,4 +761,3 @@ export class TranscendPromptManager< }; } } -/* eslint-enable max-lines */ diff --git a/src/lib/ai/filterNullishValuesFromObject.ts b/src/lib/ai/filterNullishValuesFromObject.ts index 3dac2dfa..ea36f39c 100644 --- a/src/lib/ai/filterNullishValuesFromObject.ts +++ b/src/lib/ai/filterNullishValuesFromObject.ts @@ -1,4 +1,4 @@ -import { ObjByString } from '@transcend-io/type-utils'; +import { ObjByString } from "@transcend-io/type-utils"; /** * Given an object, remove all keys that are null-ish @@ -7,17 +7,17 @@ import { ObjByString } from '@transcend-io/type-utils'; * @returns Object with null-ish values removed */ export function filterNullishValuesFromObject( - obj: T, + object: T ): T { - return Object.entries(obj).reduce( - (acc, [k, v]) => + return Object.entries(object).reduce( + (accumulator, [k, v]) => v !== null && v !== undefined && - v !== '' && + v !== "" && !(Array.isArray(v) && v.length === 0) && - !(typeof v === 'object' && Object.keys(v).length === 0) - ? Object.assign(acc, { [k]: v }) - : acc, - {} as T, + !(typeof v === "object" && Object.keys(v).length === 0) + ? Object.assign(accumulator, { [k]: v }) + : accumulator, + {} as T ); } diff --git a/src/lib/ai/getGitFilesThatChanged.ts b/src/lib/ai/getGitFilesThatChanged.ts index 717b3933..af27c02b 100644 --- a/src/lib/ai/getGitFilesThatChanged.ts +++ b/src/lib/ai/getGitFilesThatChanged.ts @@ -1,6 +1,6 @@ -import { execSync } from 'child_process'; -import fastGlob from 'fast-glob'; -import { difference } from 'lodash-es'; +import { execSync } from "node:child_process"; +import fastGlob from "fast-glob"; +import { difference } from "lodash-es"; /** * Function thats gets the git files that have changed @@ -34,7 +34,7 @@ export function getGitFilesThatChanged({ /** Current commit */ commit: string; /** File diffs */ - fileDiffs: { [k in string]: string }; + fileDiffs: Record; } { // Pull base branch execSync(`git fetch origin ${baseBranch}`); @@ -42,17 +42,17 @@ export function getGitFilesThatChanged({ // Latest commit on base branch. If we are on the base branch, we take the prior commit const latestBasedCommit = execSync( `git ls-remote ${githubRepo} "refs/heads/${baseBranch}" | cut -f 1`, - { encoding: 'utf-8' }, - ).split('\n')[0]; + { encoding: "utf-8" } + ).split("\n")[0]; // This commit - const latestThisCommit = execSync('git rev-parse HEAD', { - encoding: 'utf-8', - }).split('\n')[0]; + const latestThisCommit = execSync("git rev-parse HEAD", { + encoding: "utf-8", + }).split("\n")[0]; // Ensure commits are present if (!latestBasedCommit || !latestThisCommit) { - throw new Error('FAILED TO FIND COMMIT RANGE'); + throw new Error("FAILED TO FIND COMMIT RANGE"); } // Get the diff between the given branch and base branch @@ -60,13 +60,13 @@ export function getGitFilesThatChanged({ `git fetch && git diff --name-only "${ baseBranch || latestBasedCommit }...${latestThisCommit}" -- ${rootDirectory}`, - { encoding: 'utf-8' }, + { encoding: "utf-8" } ); // Filter out block list const changedFiles = difference( - diff.split('\n').filter((f) => f), - fileBlockList, + diff.split("\n").filter(Boolean), + fileBlockList ); // Filter out globs @@ -76,16 +76,16 @@ export function getGitFilesThatChanged({ : changedFiles; // Get the contents of only the changed files - const fileDiffs: { [k in string]: string } = {}; - filteredChanges.forEach((file) => { + const fileDiffs: Record = {}; + for (const file of filteredChanges) { const contents = execSync(`git show ${latestThisCommit}:${file}`, { - encoding: 'utf-8', + encoding: "utf-8", }); fileDiffs[file] = contents; - }); + } // Pull the github repo name - const repoName = githubRepo.split('/').pop()!.split('.')[0]; + const repoName = githubRepo.split("/").pop()!.split(".")[0]; return { changedFiles, diff --git a/src/lib/ai/removeLinks.ts b/src/lib/ai/removeLinks.ts index 88fd67f9..dcbe2bee 100644 --- a/src/lib/ai/removeLinks.ts +++ b/src/lib/ai/removeLinks.ts @@ -6,5 +6,5 @@ */ export function removeLinks(inputString: string): string { const regex = /(https?:\/\/[^\s]+)/g; - return inputString.replace(regex, ''); + return inputString.replaceAll(regex, ""); } diff --git a/src/lib/api-keys/generateCrossAccountApiKeys.ts b/src/lib/api-keys/generateCrossAccountApiKeys.ts index 420465ba..d189b540 100644 --- a/src/lib/api-keys/generateCrossAccountApiKeys.ts +++ b/src/lib/api-keys/generateCrossAccountApiKeys.ts @@ -1,9 +1,9 @@ -import { ScopeName } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { StoredApiKey } from '../../codecs'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; +import { ScopeName } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { StoredApiKey } from "../../codecs"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; import { assumeRole, buildTranscendGraphQLClientGeneric, @@ -11,7 +11,7 @@ import { deleteApiKey, fetchAllApiKeys, loginUser, -} from '../graphql'; +} from "../graphql"; export interface ApiKeyGenerateError { /** Name of instance */ @@ -64,14 +64,14 @@ export async function generateCrossAccountApiKeys({ const client = await buildTranscendGraphQLClientGeneric(transcendUrl, {}); // Login the user - logger.info(colors.magenta('Logging in using email and password.')); + logger.info(colors.magenta("Logging in using email and password.")); const { roles, loginCookie } = await loginUser(client, { email, password }); logger.info( colors.green( `Successfully logged in and found ${roles.length} role${ - roles.length === 1 ? '' : 's' - }!`, - ), + roles.length === 1 ? "" : "s" + }!` + ) ); // Filter down by parentOrganizationId @@ -79,7 +79,7 @@ export async function generateCrossAccountApiKeys({ ? roles.filter( (role) => role.organization.id === parentOrganizationId || - role.organization.parentOrganizationId === parentOrganizationId, + role.organization.parentOrganizationId === parentOrganizationId ) : roles; @@ -96,9 +96,9 @@ export async function generateCrossAccountApiKeys({ logger.info( colors.magenta( `Generating API keys with title: ${apiKeyTitle}, scopes: ${scopes.join( - ',', - )}.`, - ), + "," + )}.` + ) ); // Map over each role @@ -110,8 +110,8 @@ export async function generateCrossAccountApiKeys({ // Grab API keys with that title logger.info( colors.magenta( - `Checking if API key already exists in organization "${role.organization.name}" with title: "${apiKeyTitle}".`, - ), + `Checking if API key already exists in organization "${role.organization.name}" with title: "${apiKeyTitle}".` + ) ); // Delete existing API key @@ -119,14 +119,14 @@ export async function generateCrossAccountApiKeys({ if (apiKeyWithTitle && deleteExistingApiKey) { logger.info( colors.yellow( - `Deleting existing API key in "${role.organization.name}" with title: "${apiKeyTitle}".`, - ), + `Deleting existing API key in "${role.organization.name}" with title: "${apiKeyTitle}".` + ) ); await deleteApiKey(client, apiKeyWithTitle.id); logger.info( colors.green( - `Successfully deleted API key in "${role.organization.name}" with title: "${apiKeyTitle}".`, - ), + `Successfully deleted API key in "${role.organization.name}" with title: "${apiKeyTitle}".` + ) ); } else if (apiKeyWithTitle) { // throw error if one exists but not configured to delete @@ -137,8 +137,8 @@ export async function generateCrossAccountApiKeys({ if (createNewApiKey) { logger.info( colors.magenta( - `Creating API key in "${role.organization.name}" with title: "${apiKeyTitle}".`, - ), + `Creating API key in "${role.organization.name}" with title: "${apiKeyTitle}".` + ) ); const { apiKey } = await createApiKey(client, { title: apiKeyTitle, @@ -151,45 +151,45 @@ export async function generateCrossAccountApiKeys({ }); logger.info( colors.green( - `Successfully created API key in "${role.organization.name}" with title: "${apiKeyTitle}".`, - ), + `Successfully created API key in "${role.organization.name}" with title: "${apiKeyTitle}".` + ) ); } else { // Delete only results.push({ organizationName: role.organization.name, organizationId: role.organization.id, - apiKey: '', + apiKey: "", }); } - } catch (err) { + } catch (error) { logger.error( colors.red( - `Failed to create API key in organization "${role.organization.name}"! - ${err.message}`, - ), + `Failed to create API key in organization "${role.organization.name}"! - ${error.message}` + ) ); errors.push({ organizationName: role.organization.name, organizationId: role.organization.id, - error: err.message, + error: error.message, }); } }); logger.info( colors.green( `Successfully created ${results.length} API key${ - results.length === 1 ? '' : 's' - }`, - ), + results.length === 1 ? "" : "s" + }` + ) ); if (errors.length > 0) { logger.error( colors.red( `Failed to create ${errors.length} API key${ - errors.length === 1 ? '' : 's' - }!`, - ), + errors.length === 1 ? "" : "s" + }!` + ) ); } diff --git a/src/lib/api-keys/listDirectories.ts b/src/lib/api-keys/listDirectories.ts index 966ca7a9..72d69c8a 100644 --- a/src/lib/api-keys/listDirectories.ts +++ b/src/lib/api-keys/listDirectories.ts @@ -1,5 +1,5 @@ -import { readdirSync, statSync } from 'fs'; -import { join } from 'path'; +import { readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; /** * List the folders in a directory @@ -9,6 +9,6 @@ import { join } from 'path'; */ export function listDirectories(startDir: string): string[] { return readdirSync(startDir).filter((entryName) => - statSync(join(startDir, entryName)).isDirectory(), + statSync(join(startDir, entryName)).isDirectory() ); } diff --git a/src/lib/api-keys/listFiles.ts b/src/lib/api-keys/listFiles.ts index fd5e38af..b3000f77 100644 --- a/src/lib/api-keys/listFiles.ts +++ b/src/lib/api-keys/listFiles.ts @@ -1,4 +1,4 @@ -import { existsSync, readdirSync } from 'fs'; +import { existsSync, readdirSync } from "node:fs"; /** * List the files in a directory @@ -18,7 +18,7 @@ import { existsSync, readdirSync } from 'fs'; export function listFiles( directory: string, validExtensions?: string[], - removeExtensions = false, + removeExtensions = false ): string[] { if (!existsSync(directory)) { return []; @@ -27,12 +27,12 @@ export function listFiles( const files = readdirSync(directory) .filter((fil) => validExtensions - ? validExtensions.filter((ext) => fil.endsWith(ext)).length - : true, + ? validExtensions.filter((extension) => fil.endsWith(extension)).length + : true ) - .filter((fil) => fil.indexOf('.') > 0); + .filter((fil) => fil.indexOf(".") > 0); return removeExtensions - ? files.map((fil) => fil.replace(/\.[^/.]+$/, '')) + ? files.map((fil) => fil.replace(/\.[^/.]+$/, "")) : files; } diff --git a/src/lib/api-keys/validateTranscendAuth.ts b/src/lib/api-keys/validateTranscendAuth.ts index 0db9b392..45761e1e 100644 --- a/src/lib/api-keys/validateTranscendAuth.ts +++ b/src/lib/api-keys/validateTranscendAuth.ts @@ -1,9 +1,9 @@ -import { existsSync, readFileSync } from 'fs'; -import { decodeCodec } from '@transcend-io/type-utils'; -import colors from 'colors'; -import * as t from 'io-ts'; -import { StoredApiKey } from '../../codecs'; -import { logger } from '../../logger'; +import { existsSync, readFileSync } from "node:fs"; +import { decodeCodec } from "@transcend-io/type-utils"; +import colors from "colors"; +import * as t from "io-ts"; +import { StoredApiKey } from "../../codecs"; +import { logger } from "../../logger"; /** * Determine if the `--auth` parameter is an API key or a path to a JSON @@ -17,8 +17,8 @@ export function validateTranscendAuth(auth: string): string | StoredApiKey[] { if (!auth) { logger.error( colors.red( - 'A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY', - ), + "A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY" + ) ); process.exit(1); } @@ -26,7 +26,7 @@ export function validateTranscendAuth(auth: string): string | StoredApiKey[] { // Read from disk if (existsSync(auth)) { // validate that file is a list of API keys - return decodeCodec(t.array(StoredApiKey), readFileSync(auth, 'utf-8')); + return decodeCodec(t.array(StoredApiKey), readFileSync(auth, "utf-8")); } // Return as single API key diff --git a/src/lib/bluebird-replace.ts b/src/lib/bluebird-replace.ts index 0b82e55a..46c2fa2e 100644 --- a/src/lib/bluebird-replace.ts +++ b/src/lib/bluebird-replace.ts @@ -7,11 +7,11 @@ */ export async function mapSeries( array: R[], - iterator: (item: R, index: number, arrayLength: number) => Promise, + iterator: (item: R, index: number, arrayLength: number) => Promise ): Promise { const results = []; - for (let i = 0; i < array.length; i += 1) { - results.push(await iterator(array[i], i, array.length)); + for (let index = 0; index < array.length; index += 1) { + results.push(await iterator(array[index], index, array.length)); } return results; } @@ -30,10 +30,10 @@ export async function map( options: { /** Concurrency level for the Promise.all call */ concurrency?: number; - } = {}, + } = {} ): Promise { const { concurrency = Infinity } = options; - const results: U[] = new Array(array.length); + const results: U[] = Array.from({ length: array.length }); const executing: Promise[] = []; let nextIndex = 0; @@ -46,7 +46,7 @@ export async function map( const promise = iterator( array[currentIndex], currentIndex, - array.length, + array.length ).then((result) => { results[currentIndex] = result; }); @@ -56,7 +56,7 @@ export async function map( // Remove the completed promise from executing array const index = executing.indexOf(promise); - if (index > -1) { + if (index !== -1) { executing.splice(index, 1); } }; @@ -64,7 +64,7 @@ export async function map( // Start initial batch of promises up to concurrency limit const initialBatch = Math.min(concurrency, array.length); const initialPromises = []; - for (let i = 0; i < initialBatch; i += 1) { + for (let index = 0; index < initialBatch; index += 1) { initialPromises.push(executeNext()); } diff --git a/src/lib/code-scanning/constants.ts b/src/lib/code-scanning/constants.ts index a7225a3a..092aee78 100644 --- a/src/lib/code-scanning/constants.ts +++ b/src/lib/code-scanning/constants.ts @@ -1,4 +1,4 @@ -import { CodePackageType } from '@transcend-io/privacy-types'; +import { CodePackageType } from "@transcend-io/privacy-types"; import { cocoaPods, composerJson, @@ -8,15 +8,13 @@ import { pubspec, pythonRequirementsTxt, swift, -} from './integrations'; -import { CodeScanningConfig } from './types'; +} from "./integrations"; +import { CodeScanningConfig } from "./types"; /** * @deprecated TODO: https://transcend.height.app/T-32325 - use code scanning instead */ -export const SILO_DISCOVERY_CONFIGS: { - [k in string]: CodeScanningConfig; -} = { +export const SILO_DISCOVERY_CONFIGS: Record = { cocoaPods, gradle, javascriptPackageJson, @@ -26,9 +24,10 @@ export const SILO_DISCOVERY_CONFIGS: { swift, }; -export const CODE_SCANNING_CONFIGS: { - [k in CodePackageType]: CodeScanningConfig; -} = { +export const CODE_SCANNING_CONFIGS: Record< + CodePackageType, + CodeScanningConfig +> = { [CodePackageType.CocoaPods]: cocoaPods, [CodePackageType.Gradle]: gradle, [CodePackageType.PackageJson]: javascriptPackageJson, diff --git a/src/lib/code-scanning/findCodePackagesInFolder.ts b/src/lib/code-scanning/findCodePackagesInFolder.ts index 77c0c58c..9abd8f34 100644 --- a/src/lib/code-scanning/findCodePackagesInFolder.ts +++ b/src/lib/code-scanning/findCodePackagesInFolder.ts @@ -1,9 +1,9 @@ -import { getEntries } from '@transcend-io/type-utils'; -import colors from 'colors'; -import fastGlob from 'fast-glob'; -import { CodePackageInput } from '../../codecs'; -import { logger } from '../../logger'; -import { CODE_SCANNING_CONFIGS } from './constants'; +import { getEntries } from "@transcend-io/type-utils"; +import colors from "colors"; +import fastGlob from "fast-glob"; +import { CodePackageInput } from "../../codecs"; +import { logger } from "../../logger"; +import { CODE_SCANNING_CONFIGS } from "./constants"; /** * Helper to scan and discovery all of the code packages within a folder @@ -13,7 +13,7 @@ import { CODE_SCANNING_CONFIGS } from './constants'; */ export async function findCodePackagesInFolder({ scanPath, - ignoreDirs = [], + ignoreDirs: ignoreDirectories = [], repositoryName, }: { /** The name of the github repository reporting packages for */ @@ -26,60 +26,59 @@ export async function findCodePackagesInFolder({ const allCodePackages = await Promise.all( getEntries(CODE_SCANNING_CONFIGS).map(async ([codePackageType, config]) => { const { - ignoreDirs: configIgnoreDirs, + ignoreDirs: configIgnoreDirectories, supportedFiles, scanFunction, } = config; - const dirsToIgnore = [...ignoreDirs, ...configIgnoreDirs].filter( - (dir) => dir.length > 0, - ); + const directoriesToIgnore = [ + ...ignoreDirectories, + ...configIgnoreDirectories, + ].filter((dir) => dir.length > 0); try { const filesToScan: string[] = await fastGlob( - `${scanPath}/**/${supportedFiles.join('|')}`, + `${scanPath}/**/${supportedFiles.join("|")}`, { - ignore: dirsToIgnore.map((dir: string) => `${scanPath}/**/${dir}`), + ignore: directoriesToIgnore.map( + (dir: string) => `${scanPath}/**/${dir}` + ), unique: true, onlyFiles: true, - }, + } ); logger.info( colors.magenta( - `Scanning: ${filesToScan.length} files of type ${codePackageType}`, - ), - ); - const allPackages = filesToScan - .map((filePath) => - scanFunction(filePath).map((result) => ({ - ...result, - relativePath: filePath.replace(`${scanPath}/`, ''), - })), + `Scanning: ${filesToScan.length} files of type ${codePackageType}` ) - .flat(); + ); + const allPackages = filesToScan.flatMap((filePath) => + scanFunction(filePath).map((result) => ({ + ...result, + relativePath: filePath.replace(`${scanPath}/`, ""), + })) + ); logger.info( colors.green( `Found: ${allPackages.length} packages and ${ - allPackages - .map( - ({ softwareDevelopmentKits = [] }) => softwareDevelopmentKits, - ) - .flat().length - } sdks`, - ), + allPackages.flatMap( + ({ softwareDevelopmentKits = [] }) => softwareDevelopmentKits + ).length + } sdks` + ) ); return allPackages.map( - (pkg): CodePackageInput => ({ - ...pkg, + (package_): CodePackageInput => ({ + ...package_, type: codePackageType, repositoryName, - }), + }) ); } catch (error) { throw new Error( - `Error scanning globs ${supportedFiles} with error: ${error}`, + `Error scanning globs ${supportedFiles} with error: ${error}` ); } - }), + }) ); return allCodePackages.flat(); diff --git a/src/lib/code-scanning/findFilesToScan.ts b/src/lib/code-scanning/findFilesToScan.ts index 13aaba41..4ffe20d4 100644 --- a/src/lib/code-scanning/findFilesToScan.ts +++ b/src/lib/code-scanning/findFilesToScan.ts @@ -1,6 +1,6 @@ -import fastGlob from 'fast-glob'; -import { logger } from '../../logger'; -import { CodeScanningConfig } from './types'; +import fastGlob from "fast-glob"; +import { logger } from "../../logger"; +import { CodeScanningConfig } from "./types"; export interface SiloDiscoveryRawResults { /** The name of the potential data silo entry */ @@ -37,28 +37,30 @@ export async function findFilesToScan({ }): Promise { const { ignoreDirs: IGNORE_DIRS, supportedFiles, scanFunction } = config; const globsToSupport = - fileGlobs === '' + fileGlobs === "" ? supportedFiles - : supportedFiles.concat(fileGlobs.split(',')); - const dirsToIgnore = [...ignoreDirs.split(','), ...IGNORE_DIRS].filter( - (dir) => dir.length > 0, + : supportedFiles.concat(fileGlobs.split(",")); + const directoriesToIgnore = [...ignoreDirs.split(","), ...IGNORE_DIRS].filter( + (dir) => dir.length > 0 ); try { const filesToScan: string[] = await fastGlob( - `${scanPath}/**/${globsToSupport.join('|')}`, + `${scanPath}/**/${globsToSupport.join("|")}`, { - ignore: dirsToIgnore.map((dir: string) => `${scanPath}/**/${dir}`), + ignore: directoriesToIgnore.map( + (dir: string) => `${scanPath}/**/${dir}` + ), unique: true, onlyFiles: true, - }, + } ); logger.info(`Scanning: ${filesToScan.length} files`); - const allPackages = filesToScan - .map((filePath: string) => scanFunction(filePath)) - .flat(); - const allSdks = allPackages - .map((appPackage) => appPackage.softwareDevelopmentKits || []) - .flat(); + const allPackages = filesToScan.flatMap((filePath: string) => + scanFunction(filePath) + ); + const allSdks = allPackages.flatMap( + (appPackage) => appPackage.softwareDevelopmentKits || [] + ); const uniqueDeps = new Set(allSdks.map((sdk) => sdk.name)); const deps = [...uniqueDeps]; logger.info(`Found: ${deps.length} unique dependencies`); @@ -69,7 +71,7 @@ export async function findFilesToScan({ })); } catch (error) { throw new Error( - `Error scanning globs ${findFilesToScan} with error: ${error}`, + `Error scanning globs ${findFilesToScan} with error: ${error}` ); } } diff --git a/src/lib/code-scanning/integrations/cocoaPods.ts b/src/lib/code-scanning/integrations/cocoaPods.ts index b0fcaa38..9a2d57c9 100644 --- a/src/lib/code-scanning/integrations/cocoaPods.ts +++ b/src/lib/code-scanning/integrations/cocoaPods.ts @@ -1,39 +1,39 @@ -import { readFileSync } from 'fs'; -import { CodePackageType } from '@transcend-io/privacy-types'; -import { findAllWithRegex } from '@transcend-io/type-utils'; -import { CodePackageSdk } from '../../../codecs'; -import { CodeScanningConfig } from '../types'; +import { readFileSync } from "node:fs"; +import { CodePackageType } from "@transcend-io/privacy-types"; +import { findAllWithRegex } from "@transcend-io/type-utils"; +import { CodePackageSdk } from "../../../codecs"; +import { CodeScanningConfig } from "../types"; const POD_TARGET_REGEX = /target ('|")(.*?)('|")/; const POD_PACKAGE_REGEX = /pod ('|")(.*?)('|")(, ('|")~> (.+?)('|")|)/; export const cocoaPods: CodeScanningConfig = { - supportedFiles: ['Podfile'], - ignoreDirs: ['Pods'], + supportedFiles: ["Podfile"], + ignoreDirs: ["Pods"], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, "utf-8"); const targets = findAllWithRegex( { - value: new RegExp(POD_TARGET_REGEX, 'g'), - matches: ['quote1', 'name', 'quote2'], + value: new RegExp(POD_TARGET_REGEX, "g"), + matches: ["quote1", "name", "quote2"], }, - fileContents, + fileContents ); const packages = findAllWithRegex( { - value: new RegExp(POD_PACKAGE_REGEX, 'g'), + value: new RegExp(POD_PACKAGE_REGEX, "g"), matches: [ - 'quote1', - 'name', - 'quote2', - 'extra', - 'quote3', - 'version', - 'quote4', + "quote1", + "name", + "quote2", + "extra", + "quote3", + "version", + "quote4", ], }, - fileContents, + fileContents ); const deps: CodePackageSdk[] = targets.map((target, ind) => ({ @@ -41,13 +41,14 @@ export const cocoaPods: CodeScanningConfig = { type: CodePackageType.CocoaPods, softwareDevelopmentKits: packages .filter( - (pkg) => - pkg.matchIndex > target.matchIndex && - (!targets[ind + 1] || pkg.matchIndex < targets[ind + 1].matchIndex), + (package_) => + package_.matchIndex > target.matchIndex && + (!targets[ind + 1] || + package_.matchIndex < targets[ind + 1].matchIndex) ) - .map((pkg) => ({ - name: pkg.name, - version: pkg.version, + .map((package_) => ({ + name: package_.name, + version: package_.version, })), })); diff --git a/src/lib/code-scanning/integrations/composerJson.ts b/src/lib/code-scanning/integrations/composerJson.ts index 82f7f54c..96eb782f 100644 --- a/src/lib/code-scanning/integrations/composerJson.ts +++ b/src/lib/code-scanning/integrations/composerJson.ts @@ -1,39 +1,39 @@ -import { readFileSync } from 'fs'; -import { dirname } from 'path'; -import { CodePackageSdk } from '../../../codecs'; -import { CodeScanningConfig } from '../types'; +import { readFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { CodePackageSdk } from "../../../codecs"; +import { CodeScanningConfig } from "../types"; export const composerJson: CodeScanningConfig = { - supportedFiles: ['composer.json'], - ignoreDirs: ['vendor', 'node_modules', 'cache', 'build', 'dist'], + supportedFiles: ["composer.json"], + ignoreDirs: ["vendor", "node_modules", "cache", "build", "dist"], scanFunction: (filePath) => { - const file = readFileSync(filePath, 'utf-8'); + const file = readFileSync(filePath, "utf-8"); const directory = dirname(filePath); const asJson = JSON.parse(file); const { name, description, require: requireDependencies = {}, - 'require-dev': requiredDevDependencies = {}, + "require-dev": requiredDevelopmentDependencies = {}, } = asJson; return [ { // name of the package - name: name || directory.split('/').pop()!, + name: name || directory.split("/").pop()!, description, softwareDevelopmentKits: [ ...Object.entries(requireDependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === 'string' ? version : undefined, - }), + version: typeof version === "string" ? version : undefined, + }) ), - ...Object.entries(requiredDevDependencies).map( + ...Object.entries(requiredDevelopmentDependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === 'string' ? version : undefined, + version: typeof version === "string" ? version : undefined, isDevDependency: true, - }), + }) ), ], }, diff --git a/src/lib/code-scanning/integrations/gemfile.ts b/src/lib/code-scanning/integrations/gemfile.ts index e9b6ca3a..1d16ce8d 100644 --- a/src/lib/code-scanning/integrations/gemfile.ts +++ b/src/lib/code-scanning/integrations/gemfile.ts @@ -1,9 +1,9 @@ -import { readFileSync } from 'fs'; -import { dirname } from 'path'; -import { CodePackageType } from '@transcend-io/privacy-types'; -import { findAllWithRegex } from '@transcend-io/type-utils'; -import { listFiles } from '../../api-keys'; -import { CodeScanningConfig } from '../types'; +import { readFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { CodePackageType } from "@transcend-io/privacy-types"; +import { findAllWithRegex } from "@transcend-io/type-utils"; +import { listFiles } from "../../api-keys"; +import { CodeScanningConfig } from "../types"; const GEM_PACKAGE_REGEX = /gem *('|")(.+?)('|")(, *('|")(.+?)('|")|)/; const GEMFILE_PACKAGE_NAME_REGEX = /spec\.name *= *('|")(.+?)('|")/; @@ -12,17 +12,17 @@ const GEMFILE_PACKAGE_DESCRIPTION_REGEX = const GEMFILE_PACKAGE_SUMMARY_REGEX = /spec\.summary *= *('|")(.+?)('|")/; export const gemfile: CodeScanningConfig = { - supportedFiles: ['Gemfile'], - ignoreDirs: ['bin'], + supportedFiles: ["Gemfile"], + ignoreDirs: ["bin"], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, "utf-8"); const directory = dirname(filePath); const filesInFolder = listFiles(directory); // parse gemspec file for name - const gemspec = filesInFolder.find((file) => file === '.gemspec'); + const gemspec = filesInFolder.find((file) => file === ".gemspec"); const gemspecContents = gemspec - ? readFileSync(gemspec, 'utf-8') + ? readFileSync(gemspec, "utf-8") : undefined; const gemfileName = gemspecContents ? (GEMFILE_PACKAGE_NAME_REGEX.exec(gemspecContents) || [])[2] @@ -35,28 +35,28 @@ export const gemfile: CodeScanningConfig = { const targets = findAllWithRegex( { - value: new RegExp(GEM_PACKAGE_REGEX, 'g'), + value: new RegExp(GEM_PACKAGE_REGEX, "g"), matches: [ - 'quote1', - 'name', - 'quote2', - 'hasVersion', - 'quote3', - 'version', - 'quote4', + "quote1", + "name", + "quote2", + "hasVersion", + "quote3", + "version", + "quote4", ], }, - fileContents, + fileContents ); return [ { - name: gemfileName || directory.split('/').pop()!, + name: gemfileName || directory.split("/").pop()!, description: gemfileDescription || undefined, type: CodePackageType.RequirementsTxt, - softwareDevelopmentKits: targets.map((pkg) => ({ - name: pkg.name, - version: pkg.version, + softwareDevelopmentKits: targets.map((package_) => ({ + name: package_.name, + version: package_.version, })), }, ]; diff --git a/src/lib/code-scanning/integrations/gradle.ts b/src/lib/code-scanning/integrations/gradle.ts index a29431ef..aa871cee 100644 --- a/src/lib/code-scanning/integrations/gradle.ts +++ b/src/lib/code-scanning/integrations/gradle.ts @@ -1,7 +1,7 @@ -import { readFileSync } from 'fs'; -import { dirname } from 'path'; -import { findAllWithRegex } from '@transcend-io/type-utils'; -import { CodeScanningConfig } from '../types'; +import { readFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { findAllWithRegex } from "@transcend-io/type-utils"; +import { CodeScanningConfig } from "../types"; const GRADLE_IMPLEMENTATION_REGEX = /implementation( *)('|")(.+?):(.+?):(.+?|)('|")/; @@ -21,58 +21,58 @@ const GRADLE_APPLICATION_NAME_REGEX = /applicationId( *)"(.+?)"/; * single and double quotes are both recognized */ export const gradle: CodeScanningConfig = { - supportedFiles: ['build.gradle**'], + supportedFiles: ["build.gradle**"], ignoreDirs: [ - 'gradle-app.setting', - 'gradle-wrapper.jar', - 'gradle-wrapper.properties', + "gradle-app.setting", + "gradle-wrapper.jar", + "gradle-wrapper.properties", ], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, "utf-8"); const directory = dirname(filePath); const targets = findAllWithRegex( { - value: new RegExp(GRADLE_IMPLEMENTATION_REGEX, 'g'), - matches: ['space', 'quote1', 'name', 'path', 'version', 'quote2'], + value: new RegExp(GRADLE_IMPLEMENTATION_REGEX, "g"), + matches: ["space", "quote1", "name", "path", "version", "quote2"], }, - fileContents, + fileContents ); const targetPlugins = findAllWithRegex( { - value: new RegExp(GRADLE_PLUGIN_REGEX, 'g'), - matches: ['quote1', 'name', 'group', 'version', 'quote2'], + value: new RegExp(GRADLE_PLUGIN_REGEX, "g"), + matches: ["quote1", "name", "group", "version", "quote2"], }, - fileContents, + fileContents ); const targetGroups = findAllWithRegex( { - value: new RegExp(GRADLE_IMPLEMENTATION_GROUP_REGEX, 'g'), + value: new RegExp(GRADLE_IMPLEMENTATION_GROUP_REGEX, "g"), matches: [ - 'space1', - 'quote1', - 'group', - 'quote2', - 'space2', - 'space3', - 'quote3', - 'name', - 'quote4', - 'space4', - 'space5', - 'quote5', - 'version', - 'quote6', + "space1", + "quote1", + "group", + "quote2", + "space2", + "space3", + "quote3", + "name", + "quote4", + "space4", + "space5", + "quote5", + "version", + "quote6", ], }, - fileContents, + fileContents ); const applications = findAllWithRegex( { - value: new RegExp(GRADLE_APPLICATION_NAME_REGEX, 'g'), - matches: ['space', 'name'], + value: new RegExp(GRADLE_APPLICATION_NAME_REGEX, "g"), + matches: ["space", "name"], }, - fileContents, + fileContents ); if (applications.length > 1) { throw new Error(`Expected only one applicationId per file: ${filePath}`); @@ -80,7 +80,7 @@ export const gradle: CodeScanningConfig = { return [ { - name: applications[0]?.name || directory.split('/').pop()!, + name: applications[0]?.name || directory.split("/").pop()!, softwareDevelopmentKits: [ ...targets, ...targetGroups, diff --git a/src/lib/code-scanning/integrations/javascriptPackageJson.ts b/src/lib/code-scanning/integrations/javascriptPackageJson.ts index 99ae0164..b7464e3c 100644 --- a/src/lib/code-scanning/integrations/javascriptPackageJson.ts +++ b/src/lib/code-scanning/integrations/javascriptPackageJson.ts @@ -1,13 +1,13 @@ -import { readFileSync } from 'fs'; -import { dirname } from 'path'; -import { CodePackageSdk } from '../../../codecs'; -import { CodeScanningConfig } from '../types'; +import { readFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { CodePackageSdk } from "../../../codecs"; +import { CodeScanningConfig } from "../types"; export const javascriptPackageJson: CodeScanningConfig = { - supportedFiles: ['package.json'], - ignoreDirs: ['node_modules', 'serverless-build', 'lambda-build'], + supportedFiles: ["package.json"], + ignoreDirs: ["node_modules", "serverless-build", "lambda-build"], scanFunction: (filePath) => { - const file = readFileSync(filePath, 'utf-8'); + const file = readFileSync(filePath, "utf-8"); const directory = dirname(filePath); const asJson = JSON.parse(file); const { @@ -20,27 +20,27 @@ export const javascriptPackageJson: CodeScanningConfig = { return [ { // name of the package - name: name || directory.split('/').pop()!, + name: name || directory.split("/").pop()!, description, softwareDevelopmentKits: [ ...Object.entries(dependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === 'string' ? version : undefined, - }), + version: typeof version === "string" ? version : undefined, + }) ), ...Object.entries(devDependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === 'string' ? version : undefined, + version: typeof version === "string" ? version : undefined, isDevDependency: true, - }), + }) ), ...Object.entries(optionalDependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === 'string' ? version : undefined, - }), + version: typeof version === "string" ? version : undefined, + }) ), ], }, diff --git a/src/lib/code-scanning/integrations/pubspec.ts b/src/lib/code-scanning/integrations/pubspec.ts index 0ef1ccd5..d2edf734 100644 --- a/src/lib/code-scanning/integrations/pubspec.ts +++ b/src/lib/code-scanning/integrations/pubspec.ts @@ -1,8 +1,8 @@ -import { readFileSync } from 'fs'; -import { dirname } from 'path'; -import { CodePackageType } from '@transcend-io/privacy-types'; -import yaml from 'js-yaml'; -import { CodeScanningConfig } from '../types'; +import { readFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { CodePackageType } from "@transcend-io/privacy-types"; +import yaml from "js-yaml"; +import { CodeScanningConfig } from "../types"; /** * Remove YAML comments from a string @@ -12,35 +12,33 @@ import { CodeScanningConfig } from '../types'; */ function removeYAMLComments(yamlString: string): string { return yamlString - .split('\n') + .split("\n") .map((line) => { // Remove inline comments - const commentIndex = line.indexOf('#'); - if (commentIndex > -1) { - // Check if '#' is not inside a string - if ( - !line.substring(0, commentIndex).includes('"') && - !line.substring(0, commentIndex).includes("'") - ) { - return line.substring(0, commentIndex).trim(); - } + const commentIndex = line.indexOf("#"); + if ( + commentIndex !== -1 && // Check if '#' is not inside a string + !line.slice(0, Math.max(0, commentIndex)).includes('"') && + !line.slice(0, Math.max(0, commentIndex)).includes("'") + ) { + return line.slice(0, Math.max(0, commentIndex)).trim(); } return line; }) .filter((line) => line.length > 0) - .join('\n'); + .join("\n"); } export const pubspec: CodeScanningConfig = { - supportedFiles: ['pubspec.yml'], - ignoreDirs: ['build'], + supportedFiles: ["pubspec.yml"], + ignoreDirs: ["build"], scanFunction: (filePath) => { const directory = dirname(filePath); - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, "utf-8"); const { name, description, - dev_dependencies = {}, + dev_dependencies: development_dependencies = {}, dependencies = {}, } = yaml.load(removeYAMLComments(fileContents)) as { /** Name */ @@ -48,35 +46,37 @@ export const pubspec: CodeScanningConfig = { /** Description */ description?: string; /** Dev dependencies */ - dev_dependencies?: { [k in string]: number | Record }; + dev_dependencies?: Record>; /** Dependencies */ - dependencies?: { [k in string]: number | Record }; + dependencies?: Record>; }; return [ { - name: name || directory.split('/').pop()!, + name: name || directory.split("/").pop()!, description, type: CodePackageType.RequirementsTxt, softwareDevelopmentKits: [ ...Object.entries(dependencies).map(([name, version]) => ({ name, version: - typeof version === 'string' + typeof version === "string" ? version - : typeof version === 'number' - ? version.toString() - : version?.sdk, + : typeof version === "number" + ? version.toString() + : version?.sdk, })), - ...Object.entries(dev_dependencies).map(([name, version]) => ({ - name, - version: - typeof version === 'string' - ? version - : typeof version === 'number' + ...Object.entries(development_dependencies).map( + ([name, version]) => ({ + name, + version: + typeof version === "string" + ? version + : typeof version === "number" ? version.toString() : version?.sdk, - isDevDependency: true, - })), + isDevDependency: true, + }) + ), ], }, ]; diff --git a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts index 237a46b8..97659eab 100644 --- a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts +++ b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts @@ -1,26 +1,26 @@ -import { readFileSync } from 'fs'; -import { dirname, join } from 'path'; -import { CodePackageType } from '@transcend-io/privacy-types'; -import { findAllWithRegex } from '@transcend-io/type-utils'; -import { listFiles } from '../../api-keys'; -import { CodeScanningConfig } from '../types'; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { CodePackageType } from "@transcend-io/privacy-types"; +import { findAllWithRegex } from "@transcend-io/type-utils"; +import { listFiles } from "../../api-keys"; +import { CodeScanningConfig } from "../types"; const REQUIREMENTS_PACKAGE_MATCH = /(.+?)(=+)(.+)/; const PACKAGE_NAME = /name *= *('|")(.+?)('|")/; const PACKAGE_DESCRIPTION = /description *= *('|")(.+?)('|")/; export const pythonRequirementsTxt: CodeScanningConfig = { - supportedFiles: ['requirements.txt'], - ignoreDirs: ['build', 'lib', 'lib64'], + supportedFiles: ["requirements.txt"], + ignoreDirs: ["build", "lib", "lib64"], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, "utf-8"); const directory = dirname(filePath); const filesInFolder = listFiles(directory); // parse setup file for name - const setupFile = filesInFolder.find((file) => file === 'setup.py'); + const setupFile = filesInFolder.find((file) => file === "setup.py"); const setupFileContents = setupFile - ? readFileSync(join(directory, setupFile), 'utf-8') + ? readFileSync(join(directory, setupFile), "utf-8") : undefined; const packageName = setupFileContents ? (PACKAGE_NAME.exec(setupFileContents) || [])[2] @@ -31,20 +31,20 @@ export const pythonRequirementsTxt: CodeScanningConfig = { const targets = findAllWithRegex( { - value: new RegExp(REQUIREMENTS_PACKAGE_MATCH, 'g'), - matches: ['name', 'equals', 'version'], + value: new RegExp(REQUIREMENTS_PACKAGE_MATCH, "g"), + matches: ["name", "equals", "version"], }, - fileContents, + fileContents ); return [ { - name: packageName || directory.split('/').pop()!, + name: packageName || directory.split("/").pop()!, description: packageDescription || undefined, type: CodePackageType.RequirementsTxt, - softwareDevelopmentKits: targets.map((pkg) => ({ - name: pkg.name, - version: pkg.version, + softwareDevelopmentKits: targets.map((package_) => ({ + name: package_.name, + version: package_.version, })), }, ]; diff --git a/src/lib/code-scanning/integrations/swift.ts b/src/lib/code-scanning/integrations/swift.ts index c4a71501..410e75bc 100644 --- a/src/lib/code-scanning/integrations/swift.ts +++ b/src/lib/code-scanning/integrations/swift.ts @@ -1,9 +1,9 @@ -import { readFileSync } from 'fs'; -import { dirname } from 'path'; -import { CodePackageType } from '@transcend-io/privacy-types'; -import { decodeCodec } from '@transcend-io/type-utils'; -import * as t from 'io-ts'; -import { CodeScanningConfig } from '../types'; +import { readFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { CodePackageType } from "@transcend-io/privacy-types"; +import { decodeCodec } from "@transcend-io/type-utils"; +import * as t from "io-ts"; +import { CodeScanningConfig } from "../types"; const SwiftPackage = t.type({ pins: t.array( @@ -15,22 +15,22 @@ const SwiftPackage = t.type({ revision: t.string, version: t.string, }), - }), + }) ), version: t.number, }); export const swift: CodeScanningConfig = { - supportedFiles: ['Package.resolved'], + supportedFiles: ["Package.resolved"], ignoreDirs: [], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, "utf-8"); const parsed = decodeCodec(SwiftPackage, fileContents); return [ { - name: dirname(filePath).split('/').pop() || '', // FIXME pull from Package.swift ->> name if possible + name: dirname(filePath).split("/").pop() || "", // FIXME pull from Package.swift ->> name if possible type: CodePackageType.CocoaPods, // FIXME should be swift softwareDevelopmentKits: parsed.pins.map((target) => ({ name: target.identity, diff --git a/src/lib/consent-manager/buildXdiSyncEndpoint.ts b/src/lib/consent-manager/buildXdiSyncEndpoint.ts index 3a11ecb7..af8dcdaa 100644 --- a/src/lib/consent-manager/buildXdiSyncEndpoint.ts +++ b/src/lib/consent-manager/buildXdiSyncEndpoint.ts @@ -1,11 +1,11 @@ -import colors from 'colors'; -import { difference } from 'lodash-es'; -import { StoredApiKey } from '../../codecs'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; -import { buildTranscendGraphQLClient, fetchConsentManager } from '../graphql'; -import { domainToHost } from './domainToHost'; +import colors from "colors"; +import { difference } from "lodash-es"; +import { StoredApiKey } from "../../codecs"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; +import { buildTranscendGraphQLClient, fetchConsentManager } from "../graphql"; +import { domainToHost } from "./domainToHost"; /** * Sync group configuration mapping @@ -19,11 +19,10 @@ import { domainToHost } from './domainToHost'; * ] * } */ -export type XdiSyncGroups = { [k in string]: string[] }; +export type XdiSyncGroups = Record; /** Regular expression for IP addresses - remove these from sync endpoint */ export const IP_ADDRESS_REGEX = - // eslint-disable-next-line max-len /^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; /** @@ -39,8 +38,8 @@ export async function buildXdiSyncEndpoint( xdiLocation, transcendUrl = DEFAULT_TRANSCEND_API, removeIpAddresses = true, - domainBlockList = ['localhost'], - xdiAllowedCommands = 'ConsentManager:Sync', + domainBlockList = ["localhost"], + xdiAllowedCommands = "ConsentManager:Sync", }: { /** The file location where the XDI file is hosted */ xdiLocation: string; @@ -52,7 +51,7 @@ export async function buildXdiSyncEndpoint( domainBlockList?: string[]; /** Allows XDI commands */ xdiAllowedCommands?: string; - }, + } ): Promise<{ /** Sync group configurations */ syncGroups: XdiSyncGroups; @@ -62,7 +61,7 @@ export async function buildXdiSyncEndpoint( // Convert API keys to list const apiKeysAsList = Array.isArray(apiKeys) ? apiKeys - : [{ apiKey: apiKeys, organizationId: '', organizationName: '' }]; + : [{ apiKey: apiKeys, organizationId: "", organizationName: "" }]; // Fetch configuration for each account const consentManagers = await map( @@ -70,8 +69,8 @@ export async function buildXdiSyncEndpoint( async (apiKey) => { logger.info( colors.magenta( - `Pulling consent metadata for organization - ${apiKey.organizationName}`, - ), + `Pulling consent metadata for organization - ${apiKey.organizationName}` + ) ); // Create a GraphQL client @@ -81,18 +80,18 @@ export async function buildXdiSyncEndpoint( const consentManager = await fetchConsentManager(client); return consentManager; }, - { concurrency: 5 }, + { concurrency: 5 } ); // construct the sync groups const syncGroups: XdiSyncGroups = {}; - consentManagers.forEach((consentManager) => { + for (const consentManager of consentManagers) { // grab the partition key const partitionKey = // take explicit key first consentManager.partition?.partition || // fallback to bundle ID - consentManager.bundleURL.split('/').reverse()[1]; + consentManager.bundleURL.split("/").reverse()[1]; // Ensure that partition exists in the sync groups if (!syncGroups[partitionKey]) { @@ -104,17 +103,17 @@ export async function buildXdiSyncEndpoint( consentManager.configuration.domains .filter( // ignore IP addresses - (domain) => !removeIpAddresses || !IP_ADDRESS_REGEX.test(domain), + (domain) => !removeIpAddresses || !IP_ADDRESS_REGEX.test(domain) ) .map((domain) => domainToHost(domain)), // ignore block list - domainBlockList, + domainBlockList ); // merge existing sync group with hosts for this consent manager syncGroups[partitionKey] = [ ...new Set([...(syncGroups[partitionKey] || []), ...hosts]), ]; - }); + } // Construct the HTML const syncEndpointHtml = ` diff --git a/src/lib/consent-manager/consentManagersToBusinessEntities.ts b/src/lib/consent-manager/consentManagersToBusinessEntities.ts index 1a12e3c4..1007b33e 100644 --- a/src/lib/consent-manager/consentManagersToBusinessEntities.ts +++ b/src/lib/consent-manager/consentManagersToBusinessEntities.ts @@ -1,5 +1,5 @@ -import { BusinessEntityInput, ConsentManagerInput } from '../../codecs'; -import { logger } from '../../logger'; +import { BusinessEntityInput, ConsentManagerInput } from "../../codecs"; +import { logger } from "../../logger"; /** * Combine multiple consent manager configurations into a list of business entity configurations @@ -13,19 +13,19 @@ export function consentManagersToBusinessEntities( name: string; /** Consent manager input */ input?: ConsentManagerInput; - }[], + }[] ): BusinessEntityInput[] { // Construct the business entities YAML definition const businessEntities = inputs.map( ({ name, input }): BusinessEntityInput => ({ // Title of Transcend Instance - title: name.replace('.yml', ''), + title: name.replace(".yml", ""), attributes: [ // Sync domain list ...(input?.domains ? [ { - key: 'Transcend Domain List', + key: "Transcend Domain List", values: [...new Set(input.domains)], }, ] @@ -34,17 +34,17 @@ export function consentManagersToBusinessEntities( ...(input?.bundleUrls ? [ { - key: 'Airgap Production URL', + key: "Airgap Production URL", values: [input.bundleUrls.PRODUCTION], }, { - key: 'Airgap Test URL', + key: "Airgap Test URL", values: [input.bundleUrls.TEST], }, { - key: 'Airgap XDI URL', + key: "Airgap XDI URL", values: [ - input.bundleUrls.PRODUCTION.replace('airgap.js', 'xdi.js'), + input.bundleUrls.PRODUCTION.replace("airgap.js", "xdi.js"), ], }, ] @@ -53,24 +53,24 @@ export function consentManagersToBusinessEntities( ...(input?.partition ? [ { - key: 'Consent Partition Key', + key: "Consent Partition Key", values: [input.partition], }, ] : []), ], - }), + }) ); // Log out info on airgap scripts to host - logger.info('\n\n~~~~~~~~~~~\nAirgap scripts to host:'); - businessEntities.forEach(({ attributes, title }, ind) => { + logger.info("\n\n~~~~~~~~~~~\nAirgap scripts to host:"); + for (const [ind, { attributes, title }] of businessEntities.entries()) { attributes - ?.find((attr) => attr.key === 'Airgap Production URL') + ?.find((attribute) => attribute.key === "Airgap Production URL") ?.values?.forEach((url) => { logger.info(`${ind}) ${title} - ${url}`); }); - }); + } return businessEntities; } diff --git a/src/lib/consent-manager/createConsentToken.ts b/src/lib/consent-manager/createConsentToken.ts index 9469bccc..b628613f 100644 --- a/src/lib/consent-manager/createConsentToken.ts +++ b/src/lib/consent-manager/createConsentToken.ts @@ -1,5 +1,5 @@ -import * as crypto from 'crypto'; -import * as jwt from 'jsonwebtoken'; +import * as crypto from "node:crypto"; +import * as jwt from "jsonwebtoken"; /** * Function to create a consent manager token @@ -13,16 +13,16 @@ import * as jwt from 'jsonwebtoken'; export function createConsentToken( userId: string, base64EncryptionKey: string, - base64SigningKey: string, + base64SigningKey: string ): string { // Read on for where to find these keys - const signingKey = Buffer.from(base64SigningKey, 'base64'); - const encryptionKey = Buffer.from(base64EncryptionKey, 'base64'); + const signingKey = Buffer.from(base64SigningKey, "base64"); + const encryptionKey = Buffer.from(base64EncryptionKey, "base64"); // NIST's AES-KWP implementation { aes 48 } - see https://tools.ietf.org/html/rfc5649 - const encryptionAlgorithm = 'id-aes256-wrap-pad'; + const encryptionAlgorithm = "id-aes256-wrap-pad"; // Initial Value for AES-KWP integrity check - see https://tools.ietf.org/html/rfc5649#section-3 - const iv = Buffer.from('A65959A6', 'hex'); + const iv = Buffer.from("A65959A6", "hex"); // Set up encryption algorithm const cipher = crypto.createCipheriv(encryptionAlgorithm, encryptionKey, iv); @@ -30,7 +30,7 @@ export function createConsentToken( const encryptedIdentifier = Buffer.concat([ cipher.update(userId), cipher.final(), - ]).toString('base64'); + ]).toString("base64"); // Create the JWT content - jwt.sign will add a 'iat' (issued at) field to the payload // If you wanted to add something manually, consider @@ -42,7 +42,7 @@ export function createConsentToken( // Create a JSON web token and HMAC it with SHA-384 const consentToken = jwt.sign(jwtPayload, signingKey, { - algorithm: 'HS384', + algorithm: "HS384", }); return consentToken; diff --git a/src/lib/consent-manager/dataFlowsToDataSilos.ts b/src/lib/consent-manager/dataFlowsToDataSilos.ts index fb83e176..be5a98da 100644 --- a/src/lib/consent-manager/dataFlowsToDataSilos.ts +++ b/src/lib/consent-manager/dataFlowsToDataSilos.ts @@ -1,6 +1,6 @@ -import { union } from 'lodash-es'; -import { DataFlowInput, DataSiloInput } from '../../codecs'; -import { IndexedCatalogs } from '../graphql'; +import { union } from "lodash-es"; +import { DataFlowInput, DataSiloInput } from "../../codecs"; +import { IndexedCatalogs } from "../graphql"; /** * Convert data flow configurations into a set of data silo configurations @@ -12,13 +12,13 @@ import { IndexedCatalogs } from '../graphql'; export function dataFlowsToDataSilos( inputs: DataFlowInput[], { - adTechPurposes = ['SaleOfInfo'], + adTechPurposes = ["SaleOfInfo"], serviceToTitle, serviceToSupportedIntegration, }: IndexedCatalogs & { /** List of purposes that are considered "Ad Tech" */ adTechPurposes?: string[]; - }, + } ): { /** List of data silo configurations for site-tech services */ siteTechDataSilos: DataSiloInput[]; @@ -32,19 +32,19 @@ export function dataFlowsToDataSilos( const adTechIntegrations: string[] = []; // Mapping from service name to list of - const serviceToFoundOnDomain: { [k in string]: string[] } = {}; + const serviceToFoundOnDomain: Record = {}; // iterate over each flow - inputs.forEach((flow) => { + for (const flow of inputs) { // process data flows with services const { service, attributes = [] } = flow; - if (!service || service === 'internalService') { - return; + if (!service || service === "internalService") { + continue; } // create mapping to found on domain const foundOnDomain = attributes.find( - (attr) => attr.key === 'Found on Domain', + (attribute) => attribute.key === "Found on Domain" ); // Create a list of all domains where the data flow was found @@ -52,10 +52,10 @@ export function dataFlowsToDataSilos( if (!serviceToFoundOnDomain[service]) { serviceToFoundOnDomain[service] = []; } - serviceToFoundOnDomain[service]!.push( + serviceToFoundOnDomain[service].push( ...foundOnDomain.values.map((v) => - v.replace('https://', '').replace('http://', ''), - ), + v.replace("https://", "").replace("http://", "") + ) ); serviceToFoundOnDomain[service] = [ ...new Set(serviceToFoundOnDomain[service]), @@ -70,28 +70,28 @@ export function dataFlowsToDataSilos( // remove from site tech list if (siteTechIntegrations.includes(service)) { siteTechIntegrations = siteTechIntegrations.filter( - (s) => s !== service, + (s) => s !== service ); } } else if (!adTechIntegrations.includes(service)) { // add to site tech list siteTechIntegrations.push(service); } - }); + } // create the list of ad tech integrations const adTechDataSilos = [...new Set(adTechIntegrations)].map((service) => ({ title: serviceToTitle[service], ...(serviceToSupportedIntegration[service] ? { integrationName: service } - : { integrationName: 'promptAPerson', 'outer-type': service }), + : { integrationName: "promptAPerson", "outer-type": service }), attributes: [ { - key: 'Tech Type', - values: ['Ad Tech'], + key: "Tech Type", + values: ["Ad Tech"], }, { - key: 'Found On Domain', + key: "Found On Domain", values: serviceToFoundOnDomain[service] || [], }, ], @@ -103,18 +103,18 @@ export function dataFlowsToDataSilos( title: serviceToTitle[service], ...(serviceToSupportedIntegration[service] ? { integrationName: service } - : { integrationName: 'promptAPerson', outerType: service }), + : { integrationName: "promptAPerson", outerType: service }), attributes: [ { - key: 'Tech Type', - values: ['Site Tech'], + key: "Tech Type", + values: ["Site Tech"], }, { - key: 'Found On Domain', + key: "Found On Domain", values: serviceToFoundOnDomain[service] || [], }, ], - }), + }) ); return { diff --git a/src/lib/consent-manager/uploadConsents.ts b/src/lib/consent-manager/uploadConsents.ts index b80063e2..071bd1b8 100644 --- a/src/lib/consent-manager/uploadConsents.ts +++ b/src/lib/consent-manager/uploadConsents.ts @@ -1,20 +1,20 @@ -import { ConsentPreferencesBody } from '@transcend-io/airgap.js-types'; -import { decodeCodec } from '@transcend-io/type-utils'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import * as t from 'io-ts'; -import { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; -import { createTranscendConsentGotInstance } from '../graphql'; -import { createConsentToken } from './createConsentToken'; -import type { ConsentPreferenceUpload } from './types'; +import { ConsentPreferencesBody } from "@transcend-io/airgap.js-types"; +import { decodeCodec } from "@transcend-io/type-utils"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import * as t from "io-ts"; +import { DEFAULT_TRANSCEND_CONSENT_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; +import { createTranscendConsentGotInstance } from "../graphql"; +import { createConsentToken } from "./createConsentToken"; +import type { ConsentPreferenceUpload } from "./types"; export const USP_STRING_REGEX = /^[0-9][Y|N]([Y|N])[Y|N]$/; export const PurposeMap = t.record( t.string, - t.union([t.boolean, t.literal('Auto')]), + t.union([t.boolean, t.literal("Auto")]) ); /** @@ -48,15 +48,15 @@ export async function uploadConsents({ // Ensure usp strings are valid const invalidUspStrings = preferences.filter( - (pref) => pref.usp && !USP_STRING_REGEX.test(pref.usp), + (pref) => pref.usp && !USP_STRING_REGEX.test(pref.usp) ); if (invalidUspStrings.length > 0) { throw new Error( `Received invalid usp strings: ${JSON.stringify( invalidUspStrings, null, - 2, - )}`, + 2 + )}` ); } @@ -79,37 +79,37 @@ export async function uploadConsents({ `Received invalid purpose maps: ${JSON.stringify( invalidPurposeMaps, null, - 2, - )}`, + 2 + )}` ); } // Ensure usp or preferences are provided const invalidInputs = preferences.filter( - (pref) => !pref.usp && !pref.purposes, + (pref) => !pref.usp && !pref.purposes ); if (invalidInputs.length > 0) { throw new Error( `Received invalid inputs, expected either purposes or usp to be defined: ${JSON.stringify( invalidInputs, null, - 2, - )}`, + 2 + )}` ); } logger.info( colors.magenta( - `Uploading ${preferences.length} user preferences to partition ${partition}`, - ), + `Uploading ${preferences.length} user preferences to partition ${partition}` + ) ); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Build a GraphQL client @@ -119,7 +119,7 @@ export async function uploadConsents({ preferences, async ({ userId, - confirmed = 'true', + confirmed = "true", updated, prompted, purposes, @@ -128,7 +128,7 @@ export async function uploadConsents({ const token = createConsentToken( userId, base64EncryptionKey, - base64SigningKey, + base64SigningKey ); // parse usp string @@ -140,14 +140,14 @@ export async function uploadConsents({ token, partition, consent: { - confirmed: confirmed === 'true', + confirmed: confirmed === "true", purposes: purposes ? decodeCodec(PurposeMap, purposes) : consent.usp - ? { SaleOfInfo: saleStatus === 'Y' } - : {}, - ...(updated ? { updated: updated === 'true' } : {}), - ...(prompted ? { prompted: prompted === 'true' } : {}), + ? { SaleOfInfo: saleStatus === "Y" } + : {}, + ...(updated ? { updated: updated === "true" } : {}), + ...(prompted ? { prompted: prompted === "true" } : {}), ...consent, }, } as ConsentPreferencesBody; @@ -155,34 +155,34 @@ export async function uploadConsents({ // Make the request try { await transcendConsentApi - .post('sync', { + .post("sync", { json: input, }) .json(); - } catch (err) { + } catch (error) { try { - const parsed = JSON.parse(err?.response?.body || '{}'); + const parsed = JSON.parse(error?.response?.body || "{}"); if (parsed.error) { logger.error(colors.red(`Error: ${parsed.error}`)); } - } catch (e) { + } catch { // continue } throw new Error( `Received an error from server: ${ - err?.response?.body || err?.message - }`, + error?.response?.body || error?.message + }` ); } total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( @@ -191,7 +191,7 @@ export async function uploadConsents({ preferences.length } user preferences to partition ${partition} in "${ totalTime / 1000 - }" seconds!`, - ), + }" seconds!` + ) ); } diff --git a/src/lib/consent-manager/uploadCookiesFromCsv.ts b/src/lib/consent-manager/uploadCookiesFromCsv.ts index 02d40635..f22487d7 100644 --- a/src/lib/consent-manager/uploadCookiesFromCsv.ts +++ b/src/lib/consent-manager/uploadCookiesFromCsv.ts @@ -1,23 +1,23 @@ -import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { CookieCsvInput, CookieInput } from '../../codecs'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { buildTranscendGraphQLClient, syncCookies } from '../graphql'; -import { splitCsvToList } from '../requests'; -import { readCsv } from '../requests/readCsv'; +import { ConsentTrackerStatus } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { CookieCsvInput, CookieInput } from "../../codecs"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { buildTranscendGraphQLClient, syncCookies } from "../graphql"; +import { splitCsvToList } from "../requests"; +import { readCsv } from "../requests/readCsv"; -const OMIT_COLUMNS = [ - 'ID', - 'Activity', - 'Encounters', - 'Last Seen At', - 'Has Native Do Not Sell/Share Support', - 'IAB USP API Support', - 'Service Description', - 'Website URL', - 'Categories of Recipients', -]; +const OMIT_COLUMNS = new Set([ + "ID", + "Activity", + "Encounters", + "Last Seen At", + "Has Native Do Not Sell/Share Support", + "IAB USP API Support", + "Service Description", + "Website URL", + "Categories of Recipients", +]); /** * Upload a set of cookies from CSV @@ -49,7 +49,7 @@ export async function uploadCookiesFromCsv({ // Convert these inputs into a format that the other function can use const validatedCookieInputs = cookieInputs.map( ({ - 'Is Regex?': isRegex, + "Is Regex?": isRegex, Notes, // TODO: https://transcend.height.app/T-26391 - export in CSV // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -61,8 +61,8 @@ export async function uploadCookiesFromCsv({ Name, ...rest }): CookieInput => ({ - ...(typeof isRegex === 'string' - ? { isRegex: isRegex.toLowerCase() === 'true' } + ...(typeof isRegex === "string" + ? { isRegex: isRegex.toLowerCase() === "true" } : {}), name: Name, description: Notes, @@ -78,12 +78,12 @@ export async function uploadCookiesFromCsv({ attributes: Object.entries(rest) // filter out native columns that are exported from the admin dashboard // but not custom attributes - .filter(([key]) => !OMIT_COLUMNS.includes(key)) + .filter(([key]) => !OMIT_COLUMNS.has(key)) .map(([key, value]) => ({ key, values: splitCsvToList(value), })), - }), + }) ); // Upload the cookies into Transcend dashboard @@ -93,8 +93,8 @@ export async function uploadCookiesFromCsv({ if (!syncedCookies) { logger.error( colors.red( - 'Encountered error(s) syncing cookies from CSV, see logs above for more info. ', - ), + "Encountered error(s) syncing cookies from CSV, see logs above for more info. " + ) ); process.exit(1); } diff --git a/src/lib/consent-manager/uploadDataFlowsFromCsv.ts b/src/lib/consent-manager/uploadDataFlowsFromCsv.ts index c9b1ed87..e01f1c4a 100644 --- a/src/lib/consent-manager/uploadDataFlowsFromCsv.ts +++ b/src/lib/consent-manager/uploadDataFlowsFromCsv.ts @@ -1,23 +1,23 @@ -import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { DataFlowCsvInput, DataFlowInput } from '../../codecs'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { buildTranscendGraphQLClient, syncDataFlows } from '../graphql'; -import { splitCsvToList } from '../requests'; -import { readCsv } from '../requests/readCsv'; +import { ConsentTrackerStatus } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { DataFlowCsvInput, DataFlowInput } from "../../codecs"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { buildTranscendGraphQLClient, syncDataFlows } from "../graphql"; +import { splitCsvToList } from "../requests"; +import { readCsv } from "../requests/readCsv"; -const OMIT_COLUMNS = [ - 'ID', - 'Activity', - 'Encounters', - 'Last Seen At', - 'Has Native Do Not Sell/Share Support', - 'IAB USP API Support', - 'Service Description', - 'Website URL', - 'Categories of Recipients', -]; +const OMIT_COLUMNS = new Set([ + "ID", + "Activity", + "Encounters", + "Last Seen At", + "Has Native Do Not Sell/Share Support", + "IAB USP API Support", + "Service Description", + "Website URL", + "Categories of Recipients", +]); /** * Upload a set of data flows from CSV @@ -61,7 +61,7 @@ export async function uploadDataFlowsFromCsv({ Status, Owners, Teams, - 'Connections Made To': value, + "Connections Made To": value, ...rest }): DataFlowInput => ({ value, @@ -79,27 +79,27 @@ export async function uploadDataFlowsFromCsv({ attributes: Object.entries(rest) // filter out native columns that are exported from the admin dashboard // but not custom attributes - .filter(([key]) => !OMIT_COLUMNS.includes(key)) + .filter(([key]) => !OMIT_COLUMNS.has(key)) .map(([key, value]) => ({ key, values: splitCsvToList(value), })), - }), + }) ); // Upload the data flows into Transcend dashboard const syncedDataFlows = await syncDataFlows( client, validatedDataFlowInputs, - classifyService, + classifyService ); // Log errors if (!syncedDataFlows) { logger.error( colors.red( - 'Encountered error(s) syncing data flows from CSV, see logs above for more info. ', - ), + "Encountered error(s) syncing data flows from CSV, see logs above for more info. " + ) ); process.exit(1); } diff --git a/src/lib/cron/markCronIdentifierCompleted.ts b/src/lib/cron/markCronIdentifierCompleted.ts index d73735ce..ce1ce7d3 100644 --- a/src/lib/cron/markCronIdentifierCompleted.ts +++ b/src/lib/cron/markCronIdentifierCompleted.ts @@ -1,5 +1,5 @@ -import type { Got } from 'got'; -import * as t from 'io-ts'; +import type { Got } from "got"; +import * as t from "io-ts"; /** * Minimal set required to mark as completed @@ -22,13 +22,13 @@ export type CronIdentifierPush = t.TypeOf; */ export async function markCronIdentifierCompleted( sombra: Got, - { nonce, identifier }: CronIdentifierPush, + { nonce, identifier }: CronIdentifierPush ): Promise { try { // Make the GraphQL request - await sombra.put('v1/data-silo', { + await sombra.put("v1/data-silo", { headers: { - 'x-transcend-nonce': nonce, + "x-transcend-nonce": nonce, }, json: { profiles: [ @@ -39,13 +39,15 @@ export async function markCronIdentifierCompleted( }, }); return true; - } catch (err) { + } catch (error) { // handle gracefully - if (err.response?.statusCode === 409) { + if (error.response?.statusCode === 409) { return false; } throw new Error( - `Received an error from server: ${err?.response?.body || err?.message}`, + `Received an error from server: ${ + error?.response?.body || error?.message + }` ); } } diff --git a/src/lib/cron/markRequestDataSiloIdsCompleted.ts b/src/lib/cron/markRequestDataSiloIdsCompleted.ts index a6b6566a..f05f1b53 100644 --- a/src/lib/cron/markRequestDataSiloIdsCompleted.ts +++ b/src/lib/cron/markRequestDataSiloIdsCompleted.ts @@ -1,15 +1,15 @@ -import { RequestDataSiloStatus } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { RequestDataSiloStatus } from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, CHANGE_REQUEST_DATA_SILO_STATUS, fetchRequestDataSilo, makeGraphQLRequest, -} from '../graphql'; +} from "../graphql"; /** * Given a CSV of Request IDs, mark associated RequestDataSilos as completed @@ -42,18 +42,18 @@ export async function markRequestDataSiloIdsCompleted({ const client = buildTranscendGraphQLClient(transcendUrl, auth); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Notify Transcend logger.info( colors.magenta( - `Notifying Transcend for data silo "${dataSiloId}" marking "${requestIds.length}" requests as completed.`, - ), + `Notifying Transcend for data silo "${dataSiloId}" marking "${requestIds.length}" requests as completed.` + ) ); let total = 0; @@ -74,26 +74,26 @@ export async function markRequestDataSiloIdsCompleted({ requestDataSiloId: requestDataSilo.id, status, }); - } catch (err) { - if (!err.message.includes('Client error: Request must be active:')) { - throw err; + } catch (error) { + if (!error.message.includes("Client error: Request must be active:")) { + throw error; } } total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( - `Successfully notified Transcend in "${totalTime / 1000}" seconds!`, - ), + `Successfully notified Transcend in "${totalTime / 1000}" seconds!` + ) ); return requestIds.length; } diff --git a/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts b/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts index a35a0755..8594279c 100644 --- a/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts +++ b/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts @@ -1,25 +1,26 @@ -import { RequestAction } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; +import { RequestAction } from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchRequestDataSiloActiveCount, -} from '../graphql'; +} from "../graphql"; import { CronIdentifier, pullCronPageOfIdentifiers, -} from './pullCronPageOfIdentifiers'; +} from "./pullCronPageOfIdentifiers"; /** * A CSV formatted identifier */ -export type CsvFormattedIdentifier = { - [k in string]: string | null | boolean | number; -}; +export type CsvFormattedIdentifier = Record< + string, + string | null | boolean | number +>; export interface CronIdentifierWithAction extends CronIdentifier { /** The request action that the identifier relates to */ @@ -71,7 +72,7 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ // Validate savePageSize if (savePageSize % apiPageSize !== 0) { throw new Error( - `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`, + `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}` ); } @@ -91,20 +92,20 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ logger.info( colors.magenta( `Pulling ${ - skipRequestCount ? 'all' : totalRequestCount + skipRequestCount ? "all" : totalRequestCount } outstanding request identifiers ` + `for data silo: "${dataSiloId}" for requests of types "${actions.join( - '", "', - )}"`, - ), + '", "' + )}"` + ) ); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); const foundRequestIds = new Set(); @@ -142,14 +143,10 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ const csvFormattedIdentifiers = identifiersWithAction.map( ({ attributes, ...identifier }) => ({ ...identifier, - ...attributes.reduce( - (acc, val) => - Object.assign(acc, { - [val.key]: val.values.join(','), - }), - {}, + ...Object.fromEntries( + attributes.map((value) => [value.key, value.values.join(",")]) ), - }), + }) ); identifiers.push(...identifiersWithAction); @@ -163,14 +160,14 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ shouldContinue = pageIdentifiers.length === apiPageSize; offset += apiPageSize; - if (!skipRequestCount) { - progressBar.update(foundRequestIds.size); - } else { + if (skipRequestCount) { logger.info( colors.magenta( - `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`, - ), + `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests` + ) ); + } else { + progressBar.update(foundRequestIds.size); } } }); @@ -183,15 +180,15 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ if (!skipRequestCount) { progressBar.stop(); } - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully pulled ${identifiers.length} outstanding identifiers from ${ foundRequestIds.size - } requests in "${totalTime / 1000}" seconds!`, - ), + } requests in "${totalTime / 1000}" seconds!` + ) ); return { identifiers }; diff --git a/src/lib/cron/pullCronPageOfIdentifiers.ts b/src/lib/cron/pullCronPageOfIdentifiers.ts index ade1f39c..9688fd62 100644 --- a/src/lib/cron/pullCronPageOfIdentifiers.ts +++ b/src/lib/cron/pullCronPageOfIdentifiers.ts @@ -1,7 +1,7 @@ -import { RequestAction } from '@transcend-io/privacy-types'; -import { decodeCodec } from '@transcend-io/type-utils'; -import type { Got } from 'got'; -import * as t from 'io-ts'; +import { RequestAction } from "@transcend-io/privacy-types"; +import { decodeCodec } from "@transcend-io/type-utils"; +import type { Got } from "got"; +import * as t from "io-ts"; export const CronIdentifier = t.type({ /** The identifier value */ @@ -25,7 +25,7 @@ export const CronIdentifier = t.type({ t.type({ key: t.string, values: t.array(t.string), - }), + }) ), }); @@ -56,7 +56,7 @@ export async function pullCronPageOfIdentifiers( limit?: number; /** Page to pull in */ offset?: number; - }, + } ): Promise { try { // Make the GraphQL request @@ -73,12 +73,14 @@ export async function pullCronPageOfIdentifiers( t.type({ items: t.array(CronIdentifier), }), - response, + response ); return items; - } catch (err) { + } catch (error) { throw new Error( - `Received an error from server: ${err?.response?.body || err?.message}`, + `Received an error from server: ${ + error?.response?.body || error?.message + }` ); } } diff --git a/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts b/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts index a35a0755..8594279c 100644 --- a/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts +++ b/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts @@ -1,25 +1,26 @@ -import { RequestAction } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; +import { RequestAction } from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchRequestDataSiloActiveCount, -} from '../graphql'; +} from "../graphql"; import { CronIdentifier, pullCronPageOfIdentifiers, -} from './pullCronPageOfIdentifiers'; +} from "./pullCronPageOfIdentifiers"; /** * A CSV formatted identifier */ -export type CsvFormattedIdentifier = { - [k in string]: string | null | boolean | number; -}; +export type CsvFormattedIdentifier = Record< + string, + string | null | boolean | number +>; export interface CronIdentifierWithAction extends CronIdentifier { /** The request action that the identifier relates to */ @@ -71,7 +72,7 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ // Validate savePageSize if (savePageSize % apiPageSize !== 0) { throw new Error( - `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`, + `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}` ); } @@ -91,20 +92,20 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ logger.info( colors.magenta( `Pulling ${ - skipRequestCount ? 'all' : totalRequestCount + skipRequestCount ? "all" : totalRequestCount } outstanding request identifiers ` + `for data silo: "${dataSiloId}" for requests of types "${actions.join( - '", "', - )}"`, - ), + '", "' + )}"` + ) ); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); const foundRequestIds = new Set(); @@ -142,14 +143,10 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ const csvFormattedIdentifiers = identifiersWithAction.map( ({ attributes, ...identifier }) => ({ ...identifier, - ...attributes.reduce( - (acc, val) => - Object.assign(acc, { - [val.key]: val.values.join(','), - }), - {}, + ...Object.fromEntries( + attributes.map((value) => [value.key, value.values.join(",")]) ), - }), + }) ); identifiers.push(...identifiersWithAction); @@ -163,14 +160,14 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ shouldContinue = pageIdentifiers.length === apiPageSize; offset += apiPageSize; - if (!skipRequestCount) { - progressBar.update(foundRequestIds.size); - } else { + if (skipRequestCount) { logger.info( colors.magenta( - `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`, - ), + `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests` + ) ); + } else { + progressBar.update(foundRequestIds.size); } } }); @@ -183,15 +180,15 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ if (!skipRequestCount) { progressBar.stop(); } - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully pulled ${identifiers.length} outstanding identifiers from ${ foundRequestIds.size - } requests in "${totalTime / 1000}" seconds!`, - ), + } requests in "${totalTime / 1000}" seconds!` + ) ); return { identifiers }; diff --git a/src/lib/cron/pushCronIdentifiersFromCsv.ts b/src/lib/cron/pushCronIdentifiersFromCsv.ts index 790e653f..ca4cab47 100644 --- a/src/lib/cron/pushCronIdentifiersFromCsv.ts +++ b/src/lib/cron/pushCronIdentifiersFromCsv.ts @@ -1,15 +1,15 @@ -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { chunk } from 'lodash-es'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map, mapSeries } from '../bluebird-replace'; -import { createSombraGotInstance } from '../graphql'; -import { readCsv } from '../requests'; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { chunk } from "lodash-es"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map, mapSeries } from "../bluebird-replace"; +import { createSombraGotInstance } from "../graphql"; +import { readCsv } from "../requests"; import { CronIdentifierPush, markCronIdentifierCompleted, -} from './markCronIdentifierCompleted'; +} from "./markCronIdentifierCompleted"; /** * Given a CSV of cron job outputs, mark all requests as completed in Transcend @@ -51,16 +51,16 @@ export async function pushCronIdentifiersFromCsv({ // Notify Transcend logger.info( colors.magenta( - `Notifying Transcend for data silo "${dataSiloId}" marking "${activeResults.length}" identifiers as completed.`, - ), + `Notifying Transcend for data silo "${dataSiloId}" marking "${activeResults.length}" identifiers as completed.` + ) ); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); let successCount = 0; @@ -73,14 +73,14 @@ export async function pushCronIdentifiersFromCsv({ const totalChunks = chunks.length; const processChunk = async ( items: CronIdentifierPush[], - chunkIndex: number, + chunkIndex: number ): Promise => { logger.info( colors.blue( `Processing chunk ${chunkIndex + 1}/${totalChunks} (${ chunk.length - } items)`, - ), + } items)` + ) ); // Process the items of the chunk concurrently @@ -92,11 +92,11 @@ export async function pushCronIdentifiersFromCsv({ } else { failureCount += 1; } - } catch (e) { + } catch (error) { logger.error( colors.red( - `Error notifying Transcend for identifier "${identifier.identifier}" - ${e?.message}`, - ), + `Error notifying Transcend for identifier "${identifier.identifier}" - ${error?.message}` + ) ); errorCount += 1; } @@ -106,7 +106,7 @@ export async function pushCronIdentifiersFromCsv({ // Sleep between chunks (except for the last chunk) if (sleepSeconds > 0 && chunkIndex < totalChunks - 1) { logger.info( - colors.yellow(`Sleeping for ${sleepSeconds}s before next chunk...`), + colors.yellow(`Sleeping for ${sleepSeconds}s before next chunk...`) ); await new Promise((resolve) => { @@ -119,31 +119,31 @@ export async function pushCronIdentifiersFromCsv({ await mapSeries(chunks, processChunk); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully notified Transcend for ${successCount} identifiers in "${ totalTime / 1000 - }" seconds!`, - ), + }" seconds!` + ) ); if (failureCount) { logger.info( colors.magenta( `There were ${failureCount} identifiers that were not in a state to be updated.` + - 'They likely have already been resolved.', - ), + "They likely have already been resolved." + ) ); } if (errorCount) { logger.error( colors.red( - `There were ${errorCount} identifiers that failed to be updated. Please review the logs for more information.`, - ), + `There were ${errorCount} identifiers that failed to be updated. Please review the logs for more information.` + ) ); - throw new Error('Failed to update all identifiers'); + throw new Error("Failed to update all identifiers"); } return activeResults.length; } diff --git a/src/lib/cron/writeCsv.ts b/src/lib/cron/writeCsv.ts index 677c67f1..c4e19451 100644 --- a/src/lib/cron/writeCsv.ts +++ b/src/lib/cron/writeCsv.ts @@ -1,6 +1,6 @@ -import { appendFileSync, createWriteStream, writeFileSync } from 'fs'; -import { ObjByString } from '@transcend-io/type-utils'; -import * as fastcsv from 'fast-csv'; +import { appendFileSync, createWriteStream, writeFileSync } from "node:fs"; +import { ObjByString } from "@transcend-io/type-utils"; +import * as fastcsv from "fast-csv"; /** * Escape a CSV value @@ -9,8 +9,8 @@ import * as fastcsv from 'fast-csv'; * @returns Escaped value */ function escapeCsvValue(value: string): string { - if (value.includes('"') || value.includes(',') || value.includes('\n')) { - return `"${value.replace(/"/g, '""')}"`; + if (value.includes('"') || value.includes(",") || value.includes("\n")) { + return `"${value.replaceAll('"', '""')}"`; } return value; } @@ -24,7 +24,7 @@ function escapeCsvValue(value: string): string { export function writeCsvSync( filePath: string, data: ObjByString[], - headers: string[], + headers: string[] ): void { const rows: string[][] = []; @@ -33,8 +33,8 @@ export function writeCsvSync( // Build CSV content with proper escaping const csvContent = rows - .map((row) => row.map(escapeCsvValue).join(',')) - .join('\n'); + .map((row) => row.map(escapeCsvValue).join(",")) + .join("\n"); // Write to file, overwriting existing content writeFileSync(filePath, csvContent); @@ -53,8 +53,8 @@ export function appendCsvSync(filePath: string, data: ObjByString[]): void { // Build CSV content with proper escaping const csvContent = rows - .map((row) => row.map(escapeCsvValue).join(',')) - .join('\n'); + .map((row) => row.map(escapeCsvValue).join(",")) + .join("\n"); // Append to file with leading newline appendFileSync(filePath, `\n${csvContent}`); @@ -70,7 +70,7 @@ export function appendCsvSync(filePath: string, data: ObjByString[]): void { export async function writeCsv( filePath: string, data: ObjByString[], - headers: boolean | string[] = true, + headers: boolean | string[] = true ): Promise { const ws = createWriteStream(filePath); await new Promise((resolve, reject) => { @@ -78,10 +78,12 @@ export async function writeCsv( fastcsv .write(data, { headers, objectMode: true }) .pipe(ws) - .on('error', reject) - .on('end', () => resolve(true)); - } catch (err) { - reject(err); + .on("error", reject) + .on("end", () => { + resolve(true); + }); + } catch (error) { + reject(error); } }); } @@ -98,11 +100,14 @@ export function parseFilePath(filePath: string): { /** Extension of the file */ extension: string; } { - const lastDotIndex = filePath.lastIndexOf('.'); + const lastDotIndex = filePath.lastIndexOf("."); return { baseName: - lastDotIndex !== -1 ? filePath.substring(0, lastDotIndex) : filePath, - extension: lastDotIndex !== -1 ? filePath.substring(lastDotIndex) : '.csv', + lastDotIndex === -1 + ? filePath + : filePath.slice(0, Math.max(0, lastDotIndex)), + extension: + lastDotIndex === -1 ? ".csv" : filePath.slice(Math.max(0, lastDotIndex)), }; } @@ -119,7 +124,7 @@ export async function writeLargeCsv( filePath: string, data: ObjByString[], headers: boolean | string[] = true, - chunkSize = 100000, + chunkSize = 100_000 ): Promise { if (data.length <= chunkSize) { // If data is small enough, write to single file @@ -132,13 +137,16 @@ export async function writeLargeCsv( const totalChunks = Math.ceil(data.length / chunkSize); const { baseName, extension } = parseFilePath(filePath); - for (let i = 0; i < totalChunks; i += 1) { - const start = i * chunkSize; + for (let index = 0; index < totalChunks; index += 1) { + const start = index * chunkSize; const end = Math.min(start + chunkSize, data.length); const chunk = data.slice(start, end); // Create filename with chunk number and zero-padding - const chunkNumber = String(i + 1).padStart(String(totalChunks).length, '0'); + const chunkNumber = String(index + 1).padStart( + String(totalChunks).length, + "0" + ); const chunkFilePath = `${baseName}_part${chunkNumber}_of_${totalChunks}${extension}`; await writeCsv(chunkFilePath, chunk, headers); diff --git a/src/lib/data-inventory/pullAllDatapoints.ts b/src/lib/data-inventory/pullAllDatapoints.ts index b955a988..ca56995b 100644 --- a/src/lib/data-inventory/pullAllDatapoints.ts +++ b/src/lib/data-inventory/pullAllDatapoints.ts @@ -1,23 +1,22 @@ -/* eslint-disable max-lines */ import { SubDataPointDataSubCategoryGuessStatus, type DataCategoryType, -} from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { gql } from 'graphql-request'; -import type { GraphQLClient } from 'graphql-request'; -import { chunk, keyBy, sortBy, uniq } from 'lodash-es'; -import type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; +} from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { gql } from "graphql-request"; +import type { GraphQLClient } from "graphql-request"; +import { chunk, keyBy, sortBy, uniq } from "lodash-es"; +import type { DataCategoryInput, ProcessingPurposeInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; import { DATA_SILO_EXPORT, DATAPOINT_EXPORT, makeGraphQLRequest, SUB_DATA_POINTS_COUNT, type DataSiloAttributeValue, -} from '../graphql'; +} from "../graphql"; export interface DataSiloCsvPreview { /** ID of dataSilo */ @@ -100,17 +99,17 @@ async function pullSubDatapoints( }: DatapointFilterOptions & { /** Page size to pull in */ pageSize?: number; - } = {}, + } = {} ): Promise { const subDataPoints: SubDataPointCsvPreview[] = []; // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Filters @@ -139,7 +138,7 @@ async function pullSubDatapoints( filterBy, }); - logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints')); + logger.info(colors.magenta("[Step 1/3] Pulling in all subdatapoints")); progressBar.start(totalCount, 0); let total = 0; @@ -194,7 +193,7 @@ async function pullSubDatapoints( status classifierVersion }` - : '' + : "" } ${ includeAttributes @@ -204,7 +203,7 @@ async function pullSubDatapoints( } name }` - : '' + : "" } } } @@ -218,37 +217,37 @@ async function pullSubDatapoints( // TODO: https://transcend.height.app/T-40484 - add cursor support // ...(cursor ? { cursor: { id: cursor } } : {}), }, - }, + } ); - cursor = nodes[nodes.length - 1]?.id as string; + cursor = nodes.at(-1)?.id; subDataPoints.push(...nodes); shouldContinue = nodes.length === pageSize; total += nodes.length; offset += nodes.length; progressBar.update(total); - } catch (err) { + } catch (error) { logger.error( colors.red( - `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`, - ), + `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}` + ) ); - throw err; + throw error; } } while (shouldContinue); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; - const sorted = sortBy(subDataPoints, 'name'); + const sorted = sortBy(subDataPoints, "name"); logger.info( colors.green( `Successfully pulled in ${sorted.length} subdatapoints in ${ totalTime / 1000 - } seconds!`, - ), + } seconds!` + ) ); return sorted; } @@ -270,23 +269,23 @@ async function pullDatapoints( dataPointIds: string[]; /** Page size to pull in */ pageSize?: number; - }, + } ): Promise { const dataPoints: DataPointCsvPreview[] = []; // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); logger.info( colors.magenta( - `[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`, - ), + `[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints` + ) ); // Group by 100 @@ -314,28 +313,28 @@ async function pullDatapoints( dataPoints.push(...nodes); total += dataPointIdsGroup.length; progressBar.update(total); - } catch (err) { + } catch (error) { logger.error( colors.red( `An error fetching subdatapoints for IDs ${dataPointIdsGroup.join( - ', ', - )}`, - ), + ", " + )}` + ) ); - throw err; + throw error; } }); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully pulled in ${dataPoints.length} dataPoints in ${ totalTime / 1000 - } seconds!`, - ), + } seconds!` + ) ); return dataPoints; } @@ -357,23 +356,23 @@ async function pullDataSilos( dataSiloIds: string[]; /** Page size to pull in */ pageSize?: number; - }, + } ): Promise { const dataSilos: DataSiloCsvPreview[] = []; // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); logger.info( colors.magenta( - `[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`, - ), + `[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos` + ) ); // Group by 100 @@ -401,26 +400,26 @@ async function pullDataSilos( dataSilos.push(...nodes); total += dataSiloIdsGroup.length; progressBar.update(total); - } catch (err) { + } catch (error) { logger.error( colors.red( - `An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`, - ), + `An error fetching data silos for IDs ${dataSiloIdsGroup.join(", ")}` + ) ); - throw err; + throw error; } }); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully pulled in ${dataSilos.length} data silos in ${ totalTime / 1000 - } seconds!`, - ), + } seconds!` + ) ); return dataSilos; } @@ -444,7 +443,7 @@ export async function pullAllDatapoints( }: DatapointFilterOptions & { /** Page size to pull in */ pageSize?: number; - } = {}, + } = {} ): Promise< (SubDataPointCsvPreview & { /** Data point information */ @@ -468,14 +467,14 @@ export async function pullAllDatapoints( const dataPoints = await pullDatapoints(client, { dataPointIds, }); - const dataPointById = keyBy(dataPoints, 'id'); + const dataPointById = keyBy(dataPoints, "id"); // The data silo IDs to grab const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId)); const dataSilos = await pullDataSilos(client, { dataSiloIds: allDataSiloIds, }); - const dataSiloById = keyBy(dataSilos, 'id'); + const dataSiloById = keyBy(dataSilos, "id"); return subDatapoints.map((subDataPoint) => ({ ...subDataPoint, @@ -483,4 +482,3 @@ export async function pullAllDatapoints( dataSilo: dataSiloById[subDataPoint.dataSiloId], })); } -/* eslint-enable max-lines */ diff --git a/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts b/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts index 5c08cf25..f595d356 100644 --- a/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts +++ b/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts @@ -1,11 +1,11 @@ -import type { UnstructuredSubDataPointRecommendationStatus } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { gql, type GraphQLClient } from 'graphql-request'; -import { sortBy } from 'lodash-es'; -import type { DataCategoryInput } from '../../codecs'; -import { logger } from '../../logger'; -import { ENTRY_COUNT, makeGraphQLRequest } from '../graphql'; +import type { UnstructuredSubDataPointRecommendationStatus } from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { gql, type GraphQLClient } from "graphql-request"; +import { sortBy } from "lodash-es"; +import type { DataCategoryInput } from "../../codecs"; +import { logger } from "../../logger"; +import { ENTRY_COUNT, makeGraphQLRequest } from "../graphql"; interface UnstructuredSubDataPointRecommendationCsvPreview { /** ID of subDatapoint */ @@ -68,18 +68,18 @@ export async function pullUnstructuredSubDataPointRecommendations( }: EntryFilterOptions & { /** Page size to pull in */ pageSize?: number; - } = {}, + } = {} ): Promise { const unstructuredSubDataPointRecommendations: UnstructuredSubDataPointRecommendationCsvPreview[] = []; // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Filters @@ -102,7 +102,7 @@ export async function pullUnstructuredSubDataPointRecommendations( filterBy, }); - logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints')); + logger.info(colors.magenta("[Step 1/3] Pulling in all subdatapoints")); progressBar.start(totalCount, 0); let total = 0; @@ -138,8 +138,8 @@ export async function pullUnstructuredSubDataPointRecommendations( dataSiloId scannedObjectPathId scannedObjectId - ${includeEncryptedSnippets ? 'name' : ''} - ${includeEncryptedSnippets ? 'contextSnippet' : ''} + ${includeEncryptedSnippets ? "name" : ""} + ${includeEncryptedSnippets ? "contextSnippet" : ""} dataSubCategory { name category @@ -158,37 +158,37 @@ export async function pullUnstructuredSubDataPointRecommendations( filterBy: { ...filterBy, }, - }, + } ); - cursor = nodes[nodes.length - 1]?.id as string; + cursor = nodes.at(-1)?.id; unstructuredSubDataPointRecommendations.push(...nodes); shouldContinue = nodes.length === pageSize; total += nodes.length; offset += nodes.length; progressBar.update(total); - } catch (err) { + } catch (error) { logger.error( colors.red( - `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`, - ), + `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}` + ) ); - throw err; + throw error; } } while (shouldContinue); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; - const sorted = sortBy(unstructuredSubDataPointRecommendations, 'name'); + const sorted = sortBy(unstructuredSubDataPointRecommendations, "name"); logger.info( colors.green( `Successfully pulled in ${sorted.length} subdatapoints in ${ totalTime / 1000 - } seconds!`, - ), + } seconds!` + ) ); return sorted; } diff --git a/src/lib/graphql/fetchAllAssessments.ts b/src/lib/graphql/fetchAllAssessments.ts index f5b9af12..4400472d 100644 --- a/src/lib/graphql/fetchAllAssessments.ts +++ b/src/lib/graphql/fetchAllAssessments.ts @@ -1,4 +1,3 @@ -/* eslint-disable max-lines */ import { AssessmentFormStatus, AssessmentQuestionSubType, @@ -10,10 +9,10 @@ import { ProcessingPurpose, RetentionScheduleOperation, RetentionScheduleType, -} from '@transcend-io/privacy-types'; -import { GraphQLClient } from 'graphql-request'; -import { ASSESSMENTS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "@transcend-io/privacy-types"; +import { GraphQLClient } from "graphql-request"; +import { ASSESSMENTS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Represents an assessment with various properties and metadata. @@ -341,7 +340,7 @@ const PAGE_SIZE = 20; * @returns All assessments in the organization */ export async function fetchAllAssessments( - client: GraphQLClient, + client: GraphQLClient ): Promise { const assessments: Assessment[] = []; let offset = 0; @@ -367,4 +366,3 @@ export async function fetchAllAssessments( return assessments.sort((a, b) => a.title.localeCompare(b.title)); } -/* eslint-enable max-lines */ diff --git a/src/lib/graphql/fetchAllRequestIdentifiers.ts b/src/lib/graphql/fetchAllRequestIdentifiers.ts index a1b07547..52646f7a 100644 --- a/src/lib/graphql/fetchAllRequestIdentifiers.ts +++ b/src/lib/graphql/fetchAllRequestIdentifiers.ts @@ -1,13 +1,13 @@ -import { IdentifierType } from '@transcend-io/privacy-types'; -import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; -import type { Got } from 'got'; -import { GraphQLClient } from 'graphql-request'; -import * as t from 'io-ts'; -import semver from 'semver'; -import { SOMBRA_VERSION } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { IdentifierType } from "@transcend-io/privacy-types"; +import { decodeCodec, valuesOf } from "@transcend-io/type-utils"; +import type { Got } from "got"; +import { GraphQLClient } from "graphql-request"; +import * as t from "io-ts"; +import semver from "semver"; +import { SOMBRA_VERSION } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; -const MIN_SOMBRA_VERSION_TO_DECRYPT = '7.180'; +const MIN_SOMBRA_VERSION_TO_DECRYPT = "7.180"; const RequestIdentifier = t.type({ /** ID of request */ @@ -45,7 +45,7 @@ export async function fetchAllRequestIdentifiers( }: { /** ID of request to filter on */ requestId: string; - }, + } ): Promise { const requestIdentifiers: RequestIdentifier[] = []; let offset = 0; @@ -65,22 +65,22 @@ export async function fetchAllRequestIdentifiers( version: string; }; }; - }>(client!, SOMBRA_VERSION); + }>(client, SOMBRA_VERSION); if (version && semver.lt(version, MIN_SOMBRA_VERSION_TO_DECRYPT)) { throw new Error( - `Please upgrade Sombra to ${MIN_SOMBRA_VERSION_TO_DECRYPT} or greater to use this command.`, + `Please upgrade Sombra to ${MIN_SOMBRA_VERSION_TO_DECRYPT} or greater to use this command.` ); } do { let response: unknown; try { - response = await sombra! + response = await sombra .post<{ /** Decrypted identifiers */ identifiers: RequestIdentifier[]; - }>('v1/request-identifiers', { + }>("v1/request-identifiers", { json: { first: PAGE_SIZE, offset, @@ -88,17 +88,17 @@ export async function fetchAllRequestIdentifiers( }, }) .json(); - } catch (err) { + } catch (error) { throw new Error( `Failed to fetch request identifiers: ${ - err?.response?.body || err?.message - }`, + error?.response?.body || error?.message + }` ); } const { identifiers: nodes } = decodeCodec( RequestIdentifiersResponse, - response, + response ); requestIdentifiers.push(...nodes); diff --git a/src/lib/graphql/fetchAllRequests.ts b/src/lib/graphql/fetchAllRequests.ts index 4a5284e0..3d231e8b 100644 --- a/src/lib/graphql/fetchAllRequests.ts +++ b/src/lib/graphql/fetchAllRequests.ts @@ -1,19 +1,19 @@ -import { LanguageKey } from '@transcend-io/internationalization'; +import { LanguageKey } from "@transcend-io/internationalization"; import { IsoCountryCode, IsoCountrySubdivisionCode, RequestAction, RequestOrigin, RequestStatus, -} from '@transcend-io/privacy-types'; -import { valuesOf } from '@transcend-io/type-utils'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import * as t from 'io-ts'; -import { logger } from '../../logger'; -import { REQUESTS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "@transcend-io/privacy-types"; +import { valuesOf } from "@transcend-io/type-utils"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import * as t from "io-ts"; +import { logger } from "../../logger"; +import { REQUESTS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; export const PrivacyRequest = t.intersection([ t.type({ @@ -53,7 +53,7 @@ export const PrivacyRequest = t.intersection([ id: t.string, attributeKey: t.type({ name: t.string, id: t.string }), name: t.string, - }), + }) ), }), t.partial({ @@ -111,15 +111,15 @@ export async function fetchAllRequests( * at runtime while other filters are applied at the GraphQL level. */ requestIds?: string[]; - } = {}, + } = {} ): Promise { - logger.info(colors.magenta('Fetching requests...')); + logger.info(colors.magenta("Fetching requests...")); // create a new progress bar instance and use shades_classic theme - const t0 = new Date().getTime(); + const t0 = Date.now(); const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // read in requests @@ -170,7 +170,7 @@ export async function fetchAllRequests( } while (shouldContinue); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; // Log completion time @@ -178,20 +178,20 @@ export async function fetchAllRequests( colors.green( `Completed fetching of ${requests.length} request in "${ totalTime / 1000 - }" seconds.`, - ), + }" seconds.` + ) ); // Filter down requests by request ID let allRequests = requests; if (requestIds && requestIds.length > 0) { allRequests = allRequests.filter((request) => - requestIds.includes(request.id), + requestIds.includes(request.id) ); logger.info( colors.green( - `Filtered down to ${allRequests.length} requests based on ${requestIds.length} provided IDs.`, - ), + `Filtered down to ${allRequests.length} requests based on ${requestIds.length} provided IDs.` + ) ); } diff --git a/src/lib/graphql/fetchApiKeys.ts b/src/lib/graphql/fetchApiKeys.ts index d11aeedd..cd7a0bec 100644 --- a/src/lib/graphql/fetchApiKeys.ts +++ b/src/lib/graphql/fetchApiKeys.ts @@ -1,10 +1,10 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { difference, keyBy, uniq } from 'lodash-es'; -import { TranscendInput } from '../../codecs'; -import { logger } from '../../logger'; -import { API_KEYS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { difference, keyBy, uniq } from "lodash-es"; +import { TranscendInput } from "../../codecs"; +import { logger } from "../../logger"; +import { API_KEYS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; export interface ApiKey { /** ID of API key */ @@ -15,7 +15,7 @@ export interface ApiKey { const PAGE_SIZE = 20; -const ADMIN_LINK = 'https://app.transcend.io/infrastructure/api-keys'; +const ADMIN_LINK = "https://app.transcend.io/infrastructure/api-keys"; /** * Fetch all API keys in an organization @@ -26,7 +26,7 @@ const ADMIN_LINK = 'https://app.transcend.io/infrastructure/api-keys'; */ export async function fetchAllApiKeys( client: GraphQLClient, - titles?: string[], + titles?: string[] ): Promise { const apiKeys: ApiKey[] = []; let offset = 0; @@ -65,36 +65,36 @@ export async function fetchAllApiKeys( */ export async function fetchApiKeys( { - 'api-keys': apiKeyInputs = [], - 'data-silos': dataSilos = [], + "api-keys": apiKeyInputs = [], + "data-silos": dataSilos = [], }: TranscendInput, client: GraphQLClient, - fetchAll = false, -): Promise<{ [k in string]: ApiKey }> { + fetchAll = false +): Promise> { logger.info( colors.magenta( - `Fetching ${fetchAll ? 'all' : apiKeyInputs.length} API keys...`, - ), + `Fetching ${fetchAll ? "all" : apiKeyInputs.length} API keys...` + ) ); const titles = apiKeyInputs.map(({ title }) => title); const expectedApiKeyTitles = uniq( dataSilos - .map((silo) => silo['api-key-title']) - .filter((x): x is string => !!x), + .map((silo) => silo["api-key-title"]) + .filter((x): x is string => !!x) ); const allTitlesExpected = [...expectedApiKeyTitles, ...titles]; const apiKeys = await fetchAllApiKeys( client, - fetchAll ? undefined : [...expectedApiKeyTitles, ...titles], + fetchAll ? undefined : [...expectedApiKeyTitles, ...titles] ); // Create a map - const apiKeysByTitle = keyBy(apiKeys, 'title'); + const apiKeysByTitle = keyBy(apiKeys, "title"); // Determine expected set of apiKeys expected const missingApiKeys = difference( allTitlesExpected, - apiKeys.map(({ title }) => title), + apiKeys.map(({ title }) => title) ); // If there are missing apiKeys, throw an error @@ -102,9 +102,9 @@ export async function fetchApiKeys( logger.info( colors.red( `Failed to find API keys "${missingApiKeys.join( - '", "', - )}"! Make sure these API keys are created at: ${ADMIN_LINK}`, - ), + '", "' + )}"! Make sure these API keys are created at: ${ADMIN_LINK}` + ) ); process.exit(1); } diff --git a/src/lib/graphql/fetchCatalogs.ts b/src/lib/graphql/fetchCatalogs.ts index d8a9ff06..d8992af2 100644 --- a/src/lib/graphql/fetchCatalogs.ts +++ b/src/lib/graphql/fetchCatalogs.ts @@ -1,6 +1,6 @@ -import { GraphQLClient } from 'graphql-request'; -import { CATALOGS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { GraphQLClient } from "graphql-request"; +import { CATALOGS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; export interface Catalog { /** Integration name */ @@ -20,7 +20,7 @@ const PAGE_SIZE = 100; * @returns Integration catalogs */ export async function fetchAllCatalogs( - client: GraphQLClient, + client: GraphQLClient ): Promise { const catalogs: Catalog[] = []; let offset = 0; @@ -45,15 +45,15 @@ export async function fetchAllCatalogs( shouldContinue = nodes.length === PAGE_SIZE; } while (shouldContinue); return catalogs.sort((a, b) => - a.integrationName.localeCompare(b.integrationName), + a.integrationName.localeCompare(b.integrationName) ); } export interface IndexedCatalogs { /** Mapping from service name to service title */ - serviceToTitle: { [k in string]: string }; + serviceToTitle: Record; /** Mapping from service name to boolean indicate if service has API integration support */ - serviceToSupportedIntegration: { [k in string]: boolean }; + serviceToSupportedIntegration: Record; } /** @@ -72,19 +72,19 @@ export async function fetchAndIndexCatalogs(client: GraphQLClient): Promise< const catalogs = await fetchAllCatalogs(client); // Create mapping from service name to service title - const serviceToTitle = catalogs.reduce( - (acc, catalog) => - Object.assign(acc, { [catalog.integrationName]: catalog.title }), - {} as { [k in string]: string }, + const serviceToTitle = Object.fromEntries( + catalogs.map>((catalog) => [ + catalog.integrationName, + catalog.title, + ]) ); // Create mapping from service name to boolean indicate if service has API integration support - const serviceToSupportedIntegration = catalogs.reduce( - (acc, catalog) => - Object.assign(acc, { - [catalog.integrationName]: catalog.hasApiFunctionality, - }), - {} as { [k in string]: boolean }, + const serviceToSupportedIntegration = Object.fromEntries( + catalogs.map>((catalog) => [ + catalog.integrationName, + catalog.hasApiFunctionality, + ]) ); return { diff --git a/src/lib/graphql/fetchDataSubjects.ts b/src/lib/graphql/fetchDataSubjects.ts index b6a40b66..3d36bc9d 100644 --- a/src/lib/graphql/fetchDataSubjects.ts +++ b/src/lib/graphql/fetchDataSubjects.ts @@ -1,12 +1,12 @@ -import { RequestActionObjectResolver } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { difference, flatten, keyBy, uniq } from 'lodash-es'; -import { TranscendInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { CREATE_DATA_SUBJECT, DATA_SUBJECTS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { RequestActionObjectResolver } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { difference, flatten, keyBy, uniq } from "lodash-es"; +import { TranscendInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; +import { CREATE_DATA_SUBJECT, DATA_SUBJECTS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; export interface DataSubject { /** ID of data subject */ @@ -36,7 +36,7 @@ export interface DataSubject { * @returns List of data subject configurations */ export async function fetchAllDataSubjects( - client: GraphQLClient, + client: GraphQLClient ): Promise { // Fetch all data subjects in the organization const { internalSubjects } = await makeGraphQLRequest<{ @@ -56,18 +56,18 @@ export async function fetchAllDataSubjects( */ export async function ensureAllDataSubjectsExist( { - 'data-silos': dataSilos = [], - 'data-subjects': dataSubjects = [], + "data-silos": dataSilos = [], + "data-subjects": dataSubjects = [], enrichers = [], }: TranscendInput, client: GraphQLClient, - fetchAll = false, -): Promise<{ [type in string]: DataSubject }> { + fetchAll = false +): Promise> { // Only need to fetch data subjects if specified in config const expectedDataSubjects = uniq([ - ...flatten(dataSilos.map((silo) => silo['data-subjects'] || []) || []), + ...flatten(dataSilos.map((silo) => silo["data-subjects"] || []) || []), ...flatten( - enrichers.map((enricher) => enricher['data-subjects'] || []) || [], + enrichers.map((enricher) => enricher["data-subjects"] || []) || [] ), ...dataSubjects.map((subject) => subject.type), ]); @@ -77,20 +77,20 @@ export async function ensureAllDataSubjectsExist( // Fetch all data subjects in the organization const internalSubjects = await fetchAllDataSubjects(client); - const dataSubjectByName = keyBy(internalSubjects, 'type'); + const dataSubjectByName = keyBy(internalSubjects, "type"); // Determine expected set of data subjects to create const missingDataSubjects = difference( expectedDataSubjects, - internalSubjects.map(({ type }) => type), + internalSubjects.map(({ type }) => type) ); // If there are missing data subjects, create new ones if (missingDataSubjects.length > 0) { logger.info( colors.magenta( - `Creating ${missingDataSubjects.length} new data subjects...`, - ), + `Creating ${missingDataSubjects.length} new data subjects...` + ) ); await mapSeries(missingDataSubjects, async (dataSubject) => { logger.info(colors.magenta(`Creating data subject ${dataSubject}...`)); @@ -122,13 +122,13 @@ export async function ensureAllDataSubjectsExist( */ export function convertToDataSubjectBlockList( dataSubjectTypes: string[], - allDataSubjects: { [type in string]: DataSubject }, + allDataSubjects: Record ): string[] { - dataSubjectTypes.forEach((type) => { + for (const type of dataSubjectTypes) { if (!allDataSubjects[type]) { throw new Error(`Expected to find data subject definition: ${type}`); } - }); + } return Object.values(allDataSubjects) .filter((silo) => !dataSubjectTypes.includes(silo.type)) @@ -144,13 +144,13 @@ export function convertToDataSubjectBlockList( */ export function convertToDataSubjectAllowlist( dataSubjectTypes: string[], - allDataSubjects: { [type in string]: DataSubject }, + allDataSubjects: Record ): string[] { - dataSubjectTypes.forEach((type) => { + for (const type of dataSubjectTypes) { if (!allDataSubjects[type]) { throw new Error(`Expected to find data subject definition: ${type}`); } - }); + } return Object.values(allDataSubjects) .filter((silo) => !dataSubjectTypes.includes(silo.type)) diff --git a/src/lib/graphql/fetchIdentifiers.ts b/src/lib/graphql/fetchIdentifiers.ts index 4333bf74..d17e0913 100644 --- a/src/lib/graphql/fetchIdentifiers.ts +++ b/src/lib/graphql/fetchIdentifiers.ts @@ -1,12 +1,12 @@ -import { IdentifierType, RequestAction } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { difference, flatten, keyBy, uniq } from 'lodash-es'; -import { TranscendInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { CREATE_IDENTIFIER, IDENTIFIERS, NEW_IDENTIFIER_TYPES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { IdentifierType, RequestAction } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { difference, flatten, keyBy, uniq } from "lodash-es"; +import { TranscendInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; +import { CREATE_IDENTIFIER, IDENTIFIERS, NEW_IDENTIFIER_TYPES } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; export interface Identifier { /** ID of identifier */ @@ -53,7 +53,7 @@ const PAGE_SIZE = 20; * @returns All identifiers in the organization */ export async function fetchAllIdentifiers( - client: GraphQLClient, + client: GraphQLClient ): Promise { const identifiers: Identifier[] = []; let offset = 0; @@ -93,40 +93,38 @@ export async function fetchAllIdentifiers( export async function fetchIdentifiersAndCreateMissing( { enrichers = [], - 'data-silos': dataSilos = [], + "data-silos": dataSilos = [], identifiers = [], }: TranscendInput, client: GraphQLClient, - skipPublish = false, -): Promise<{ [k in string]: Identifier }> { + skipPublish = false +): Promise> { // Grab all existing identifiers const allIdentifiers = await fetchAllIdentifiers(client); // Create a map - const identifiersByName = keyBy(allIdentifiers, 'name'); + const identifiersByName = keyBy(allIdentifiers, "name"); // Determine expected set of identifiers const expectedIdentifiers = uniq([ ...flatten( enrichers.map((enricher) => [ - enricher['input-identifier'], - ...enricher['output-identifiers'], - ]), + enricher["input-identifier"], + ...enricher["output-identifiers"], + ]) ), - ...flatten(dataSilos.map((dataSilo) => dataSilo['identity-keys'])), + ...flatten(dataSilos.map((dataSilo) => dataSilo["identity-keys"])), ...identifiers.map(({ name }) => name), ]).filter((x) => !!x); const missingIdentifiers = difference( expectedIdentifiers, - allIdentifiers.map(({ name }) => name), + allIdentifiers.map(({ name }) => name) ); // If there are missing identifiers, create new ones if (missingIdentifiers.length > 0) { logger.info( - colors.magenta( - `Creating ${missingIdentifiers.length} new identifiers...`, - ), + colors.magenta(`Creating ${missingIdentifiers.length} new identifiers...`) ); const { newIdentifierTypes } = await makeGraphQLRequest<{ /** Query response */ @@ -135,7 +133,9 @@ export async function fetchIdentifiersAndCreateMissing( name: string; }[]; }>(client, NEW_IDENTIFIER_TYPES); - const nativeTypesRemaining = newIdentifierTypes.map(({ name }) => name); + const nativeTypesRemaining = new Set( + newIdentifierTypes.map(({ name }) => name) + ); await mapSeries(missingIdentifiers, async (identifier) => { logger.info(colors.magenta(`Creating identifier ${identifier}...`)); const { createIdentifier } = await makeGraphQLRequest<{ @@ -147,9 +147,7 @@ export async function fetchIdentifiersAndCreateMissing( }>(client, CREATE_IDENTIFIER, { input: { name: identifier, - type: nativeTypesRemaining.includes(identifier!) - ? identifier - : 'custom', + type: nativeTypesRemaining.has(identifier!) ? identifier : "custom", skipPublish, }, }); diff --git a/src/lib/graphql/fetchPrompts.ts b/src/lib/graphql/fetchPrompts.ts index ceee7254..6d26c475 100644 --- a/src/lib/graphql/fetchPrompts.ts +++ b/src/lib/graphql/fetchPrompts.ts @@ -1,10 +1,10 @@ import { PromptResponseFormat, PromptStatus, -} from '@transcend-io/privacy-types'; -import { GraphQLClient } from 'graphql-request'; -import { PROMPTS, PROMPTS_WITH_VARIABLES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "@transcend-io/privacy-types"; +import { GraphQLClient } from "graphql-request"; +import { PROMPTS, PROMPTS_WITH_VARIABLES } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; export interface Prompt { /** ID of prompt */ @@ -45,7 +45,7 @@ export async function fetchAllPrompts( ids?: string[]; /** Filter by titles */ titles?: string[]; - } = {}, + } = {} ): Promise { const prompts: Prompt[] = []; let offset = 0; @@ -81,7 +81,7 @@ export async function fetchAllPrompts( /** * The basic metadata needed to use a prompt at runtime */ -export type TranscendPromptTemplated = { +export interface TranscendPromptTemplated { /** ID of prompt */ id: string; /** Title of prompt */ @@ -98,12 +98,12 @@ export type TranscendPromptTemplated = { maxTokensToSample?: number; /** Response format */ responseFormat?: PromptResponseFormat; -}; +} /** * The basic metadata needed to use a prompt partial at runtime */ -export type TranscendPromptPartialTemplated = { +export interface TranscendPromptPartialTemplated { /** ID of prompt */ id: string; /** Title of prompt */ @@ -112,32 +112,32 @@ export type TranscendPromptPartialTemplated = { slug: string; /** Content of prompt */ content: string; -}; +} /** * Calculated variables */ -export type PromptCalculatedVariable = { +export interface PromptCalculatedVariable { /** JSON stringified data to template */ data: string | null; /** Name of variable */ name: string; -}; +} /** * Runtime variables */ -export type PromptRuntimeVariable = { +export interface PromptRuntimeVariable { /** Type of variable */ type: string; /** Name of variable */ name: string; -}; +} /** * Metadata useful for filling variables within a prompt */ -export type TranscendPromptsAndVariables = { +export interface TranscendPromptsAndVariables { /** Prompts ready to be templated */ prompts: TranscendPromptTemplated[]; /** Prompt partials */ @@ -146,7 +146,7 @@ export type TranscendPromptsAndVariables = { calculatedVariables: PromptCalculatedVariable[]; /** Runtime variables to be templated */ runtimeVariables: PromptRuntimeVariable[]; -}; +} /** * Fetch prompts with templated variables @@ -165,7 +165,7 @@ export async function fetchPromptsWithVariables( promptIds?: string[]; /** Filter by prompt titles */ promptTitles?: string[]; - } = {}, + } = {} ): Promise { const { promptsWithVariables } = await makeGraphQLRequest<{ /** Prompts */ diff --git a/src/lib/graphql/fetchRequestDataSilo.ts b/src/lib/graphql/fetchRequestDataSilo.ts index a01a87e7..bd1c3c31 100644 --- a/src/lib/graphql/fetchRequestDataSilo.ts +++ b/src/lib/graphql/fetchRequestDataSilo.ts @@ -1,13 +1,13 @@ import { RequestDataSiloStatus, RequestStatus, -} from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { logger } from '../../logger'; -import { REQUEST_DATA_SILOS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { logger } from "../../logger"; +import { REQUEST_DATA_SILOS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; export interface RequestDataSilo { /** ID of RequestDataSilo */ @@ -44,13 +44,13 @@ export async function fetchRequestDataSilos( requestStatuses?: RequestStatus[]; /** When true, skip logging */ skipLog?: boolean; - }, + } ): Promise { // create a new progress bar instance and use shades_classic theme - const t0 = new Date().getTime(); + const t0 = Date.now(); const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); const requestDataSilos: RequestDataSilo[] = []; @@ -92,7 +92,7 @@ export async function fetchRequestDataSilos( } while (shouldContinue); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; // Log completion time @@ -101,8 +101,8 @@ export async function fetchRequestDataSilos( colors.green( `Completed fetching of ${ requestDataSilos.length - } request data silos in "${totalTime / 1000}" seconds.`, - ), + } request data silos in "${totalTime / 1000}" seconds.` + ) ); } @@ -126,7 +126,7 @@ export async function fetchRequestDataSilo( requestId: string; /** Data silo ID */ dataSiloId: string; - }, + } ): Promise { const nodes = await fetchRequestDataSilos(client, { requestId, @@ -135,7 +135,7 @@ export async function fetchRequestDataSilo( }); if (nodes.length !== 1) { throw new Error( - `Failed to find RequestDataSilo with requestId:${requestId},dataSiloId:${dataSiloId}`, + `Failed to find RequestDataSilo with requestId:${requestId},dataSiloId:${dataSiloId}` ); } diff --git a/src/lib/graphql/formatAttributeValues.ts b/src/lib/graphql/formatAttributeValues.ts index ffc85a13..f3e0a96f 100644 --- a/src/lib/graphql/formatAttributeValues.ts +++ b/src/lib/graphql/formatAttributeValues.ts @@ -1,4 +1,4 @@ -import type { DataSiloAttributeValue } from './syncDataSilos'; +import type { DataSiloAttributeValue } from "./syncDataSilos"; export interface FormattedAttribute { /** Attribute key */ @@ -14,21 +14,23 @@ export interface FormattedAttribute { * @returns formatted attributes */ export function formatAttributeValues( - vals: DataSiloAttributeValue[], + vals: DataSiloAttributeValue[] ): FormattedAttribute[] { const attributes: FormattedAttribute[] = []; - vals.map((val) => { - let foundKey = attributes.find((att) => att.key === val.attributeKey.name); + vals.map((value) => { + let foundKey = attributes.find( + (att) => att.key === value.attributeKey.name + ); if (foundKey === undefined) { foundKey = { - key: val.attributeKey.name, - values: [val.name], + key: value.attributeKey.name, + values: [value.name], }; attributes.push(foundKey); } else { - foundKey.values.push(val.name); + foundKey.values.push(value.name); } return attributes; }); diff --git a/src/lib/graphql/gqls/consentManager.ts b/src/lib/graphql/gqls/consentManager.ts index de4cca45..da2af09f 100644 --- a/src/lib/graphql/gqls/consentManager.ts +++ b/src/lib/graphql/gqls/consentManager.ts @@ -1,5 +1,4 @@ -/* eslint-disable max-lines */ -import { gql } from 'graphql-request'; +import { gql } from "graphql-request"; // TODO: https://transcend.height.app/T-27909 - order by createdAt // TODO: https://transcend.height.app/T-27909 - enable optimizations @@ -409,4 +408,3 @@ export const CREATE_CONSENT_PARTITION = gql` } } `; -/* eslint-enable max-lines */ diff --git a/src/lib/graphql/loginUser.ts b/src/lib/graphql/loginUser.ts index 4603dba6..1bd6afb9 100644 --- a/src/lib/graphql/loginUser.ts +++ b/src/lib/graphql/loginUser.ts @@ -1,6 +1,6 @@ -import { GraphQLClient } from 'graphql-request'; -import { ASSUME_ROLE, DETERMINE_LOGIN_METHOD, LOGIN } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { GraphQLClient } from "graphql-request"; +import { ASSUME_ROLE, DETERMINE_LOGIN_METHOD, LOGIN } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; export interface OrganizationPreview { /** Name of organization */ @@ -37,7 +37,7 @@ export async function loginUser( email: string; /** Password of user */ password: string; - }, + } ): Promise<{ /** Cookie to be used to make subsequent requests */ loginCookie: string; @@ -80,9 +80,9 @@ export async function loginUser( } = res.data; // Get login cookie from response - const loginCookie = res.headers.get('set-cookie'); - if (!loginCookie || !loginCookie.includes('laravel')) { - throw new Error('Failed to get login cookie in response'); + const loginCookie = res.headers.get("set-cookie"); + if (!loginCookie?.includes("laravel")) { + throw new Error("Failed to get login cookie in response"); } return { @@ -107,7 +107,7 @@ export async function assumeRole( email: string; /** Role of user assuming into */ roleId: string; - }, + } ): Promise { const { determineLoginMethod: { loginMethod }, diff --git a/src/lib/graphql/makeGraphQLRequest.ts b/src/lib/graphql/makeGraphQLRequest.ts index cd89f82c..a4bfe985 100644 --- a/src/lib/graphql/makeGraphQLRequest.ts +++ b/src/lib/graphql/makeGraphQLRequest.ts @@ -1,10 +1,10 @@ -import colors from 'colors'; +import colors from "colors"; import type { GraphQLClient, RequestDocument, Variables, -} from 'graphql-request'; -import { logger } from '../../logger'; +} from "graphql-request"; +import { logger } from "../../logger"; const MAX_RETRIES = 4; @@ -16,16 +16,18 @@ const MAX_RETRIES = 4; */ function sleepPromise(sleepTime: number): Promise { return new Promise((resolve) => { - setTimeout(() => resolve(sleepTime), sleepTime); + setTimeout(() => { + resolve(sleepTime); + }, sleepTime); }); } const KNOWN_ERRORS = [ - 'syntax error', - 'got invalid value', - 'Client error', - 'cannot affect row a second time', - 'GRAPHQL_VALIDATION_FAILED', + "syntax error", + "got invalid value", + "Client error", + "cannot affect row a second time", + "GRAPHQL_VALIDATION_FAILED", ]; /** @@ -43,53 +45,54 @@ export async function makeGraphQLRequest( document: RequestDocument, variables?: V, requestHeaders?: Record | string[][] | Headers, - maxRequests = MAX_RETRIES, + maxRequests = MAX_RETRIES ): Promise { let retryCount = 0; - // eslint-disable-next-line no-constant-condition + while (true) { try { const result = await client.request(document, variables, requestHeaders); return result as T; - } catch (err) { - if (err.message.includes('API key is invalid')) { + } catch (error) { + if (error.message.includes("API key is invalid")) { logger.error( colors.red( - 'API key is invalid. ' + - 'Please ensure that the key provided to `transcendAuth` has the proper scope and is not expired, ' + - 'and that `transcendUrl` corresponds to the correct backend for your organization.', - ), + "API key is invalid. " + + "Please ensure that the key provided to `transcendAuth` has the proper scope and is not expired, " + + "and that `transcendUrl` corresponds to the correct backend for your organization." + ) ); process.exit(1); } - if (KNOWN_ERRORS.some((msg) => err.message.includes(msg))) { - throw err; + if (KNOWN_ERRORS.some((message) => error.message.includes(message))) { + throw error; } // wait for rate limit to resolve - if (err.message.startsWith('Client error: Too many requests')) { - const rateLimitResetAt = err.response.headers?.get('x-ratelimit-reset'); + if (error.message.startsWith("Client error: Too many requests")) { + const rateLimitResetAt = + error.response.headers?.get("x-ratelimit-reset"); const sleepTime = rateLimitResetAt - ? new Date(rateLimitResetAt).getTime() - new Date().getTime() + 100 + ? new Date(rateLimitResetAt).getTime() - Date.now() + 100 : 1000 * 10; logger.log( colors.yellow( - `DETECTED RATE LIMIT: ${err.message}. Sleeping for ${sleepTime}ms`, - ), + `DETECTED RATE LIMIT: ${error.message}. Sleeping for ${sleepTime}ms` + ) ); await sleepPromise(sleepTime); } if (retryCount >= maxRequests) { - throw err; + throw error; } retryCount += 1; logger.log( colors.yellow( - `REQUEST FAILED: ${err.message}. Retrying ${retryCount}/${maxRequests}...`, - ), + `REQUEST FAILED: ${error.message}. Retrying ${retryCount}/${maxRequests}...` + ) ); } } diff --git a/src/lib/graphql/pullTranscendConfiguration.ts b/src/lib/graphql/pullTranscendConfiguration.ts index 67a5041b..4d9ffdc1 100644 --- a/src/lib/graphql/pullTranscendConfiguration.ts +++ b/src/lib/graphql/pullTranscendConfiguration.ts @@ -1,13 +1,12 @@ -/* eslint-disable max-lines */ -import { LanguageKey } from '@transcend-io/internationalization'; +import { LanguageKey } from "@transcend-io/internationalization"; import { ActionItemCode, ConsentTrackerStatus, RequestAction, -} from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { flatten, keyBy, mapValues } from 'lodash-es'; +} from "@transcend-io/privacy-types"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { flatten, keyBy, mapValues } from "lodash-es"; import { ActionInput, ActionItemCollectionInput, @@ -42,53 +41,53 @@ import { TeamInput, TranscendInput, VendorInput, -} from '../../codecs'; -import { TranscendPullResource } from '../../enums'; -import { logger } from '../../logger'; -import { fetchAllActionItemCollections } from './fetchAllActionItemCollections'; -import { fetchAllActionItems } from './fetchAllActionItems'; -import { fetchAllActions } from './fetchAllActions'; -import { fetchAllAgentFiles } from './fetchAllAgentFiles'; -import { fetchAllAgentFunctions } from './fetchAllAgentFunctions'; -import { fetchAllAgents } from './fetchAllAgents'; -import { fetchAllAssessments } from './fetchAllAssessments'; -import { fetchAllAssessmentTemplates } from './fetchAllAssessmentTemplates'; -import { fetchAllAttributes } from './fetchAllAttributes'; -import { fetchAllBusinessEntities } from './fetchAllBusinessEntities'; -import { fetchAllCookies } from './fetchAllCookies'; -import { fetchAllDataCategories } from './fetchAllDataCategories'; -import { fetchAllDataFlows } from './fetchAllDataFlows'; -import { fetchAllMessages } from './fetchAllMessages'; -import { fetchAllPolicies } from './fetchAllPolicies'; -import { fetchAllPrivacyCenters } from './fetchAllPrivacyCenters'; -import { fetchAllProcessingPurposes } from './fetchAllProcessingPurposes'; -import { fetchAllPurposesAndPreferences } from './fetchAllPurposesAndPreferences'; -import { fetchAllTeams } from './fetchAllTeams'; -import { fetchAllVendors } from './fetchAllVendors'; -import { fetchApiKeys } from './fetchApiKeys'; +} from "../../codecs"; +import { TranscendPullResource } from "../../enums"; +import { logger } from "../../logger"; +import { fetchAllActionItemCollections } from "./fetchAllActionItemCollections"; +import { fetchAllActionItems } from "./fetchAllActionItems"; +import { fetchAllActions } from "./fetchAllActions"; +import { fetchAllAgentFiles } from "./fetchAllAgentFiles"; +import { fetchAllAgentFunctions } from "./fetchAllAgentFunctions"; +import { fetchAllAgents } from "./fetchAllAgents"; +import { fetchAllAssessments } from "./fetchAllAssessments"; +import { fetchAllAssessmentTemplates } from "./fetchAllAssessmentTemplates"; +import { fetchAllAttributes } from "./fetchAllAttributes"; +import { fetchAllBusinessEntities } from "./fetchAllBusinessEntities"; +import { fetchAllCookies } from "./fetchAllCookies"; +import { fetchAllDataCategories } from "./fetchAllDataCategories"; +import { fetchAllDataFlows } from "./fetchAllDataFlows"; +import { fetchAllMessages } from "./fetchAllMessages"; +import { fetchAllPolicies } from "./fetchAllPolicies"; +import { fetchAllPrivacyCenters } from "./fetchAllPrivacyCenters"; +import { fetchAllProcessingPurposes } from "./fetchAllProcessingPurposes"; +import { fetchAllPurposesAndPreferences } from "./fetchAllPurposesAndPreferences"; +import { fetchAllTeams } from "./fetchAllTeams"; +import { fetchAllVendors } from "./fetchAllVendors"; +import { fetchApiKeys } from "./fetchApiKeys"; import { fetchConsentManager, fetchConsentManagerExperiences, fetchConsentManagerTheme, -} from './fetchConsentManagerId'; +} from "./fetchConsentManagerId"; import { convertToDataSubjectAllowlist, fetchAllDataSubjects, -} from './fetchDataSubjects'; -import { fetchAllIdentifiers } from './fetchIdentifiers'; -import { fetchAllPromptGroups } from './fetchPromptGroups'; -import { fetchAllPromptPartials } from './fetchPromptPartials'; -import { fetchAllPrompts } from './fetchPrompts'; -import { formatAttributeValues } from './formatAttributeValues'; +} from "./fetchDataSubjects"; +import { fetchAllIdentifiers } from "./fetchIdentifiers"; +import { fetchAllPromptGroups } from "./fetchPromptGroups"; +import { fetchAllPromptPartials } from "./fetchPromptPartials"; +import { fetchAllPrompts } from "./fetchPrompts"; +import { formatAttributeValues } from "./formatAttributeValues"; import { AssessmentNestedRule, parseAssessmentDisplayLogic, -} from './parseAssessmentDisplayLogic'; -import { parseAssessmentRiskLogic } from './parseAssessmentRiskLogic'; -import { fetchEnrichedDataSilos } from './syncDataSilos'; -import { fetchAllEnrichers } from './syncEnrichers'; -import { fetchPartitions } from './syncPartitions'; -import { fetchAllTemplates } from './syncTemplates'; +} from "./parseAssessmentDisplayLogic"; +import { parseAssessmentRiskLogic } from "./parseAssessmentRiskLogic"; +import { fetchEnrichedDataSilos } from "./syncDataSilos"; +import { fetchAllEnrichers } from "./syncEnrichers"; +import { fetchPartitions } from "./syncPartitions"; +import { fetchAllTemplates } from "./syncTemplates"; export const DEFAULT_TRANSCEND_PULL_RESOURCES = [ TranscendPullResource.DataSilos, @@ -137,11 +136,11 @@ export async function pullTranscendConfiguration( includeGuessedCategories, skipSubDatapoints, trackerStatuses = Object.values(ConsentTrackerStatus), - }: TranscendPullConfigurationInput, + }: TranscendPullConfigurationInput ): Promise { if (dataSiloIds.length > 0 && integrationNames.length > 0) { throw new Error( - 'Only 1 of integrationNames OR dataSiloIds can be provided', + "Only 1 of integrationNames OR dataSiloIds can be provided" ); } @@ -344,21 +343,21 @@ export async function pullTranscendConfiguration( // Save API keys const apiKeyTitles = flatten( - dataSilos.map(([{ apiKeys }]) => apiKeys.map(({ title }) => title)), + dataSilos.map(([{ apiKeys }]) => apiKeys.map(({ title }) => title)) ); const relevantApiKeys = Object.values(apiKeyTitleMap).filter(({ title }) => resources.includes(TranscendPullResource.ApiKeys) ? true - : apiKeyTitles.includes(title), + : apiKeyTitles.includes(title) ); if ( relevantApiKeys.length > 0 && resources.includes(TranscendPullResource.ApiKeys) ) { - result['api-keys'] = relevantApiKeys.map( + result["api-keys"] = relevantApiKeys.map( ({ title }): ApiKeyInput => ({ title, - }), + }) ); } @@ -378,7 +377,7 @@ export async function pullTranscendConfiguration( consentManager && resources.includes(TranscendPullResource.ConsentManager) ) { - result['consent-manager'] = { + result["consent-manager"] = { bundleUrls: { TEST: consentManager.testBundleURL, PRODUCTION: consentManager.bundleURL, @@ -399,14 +398,14 @@ export async function pullTranscendConfiguration( uspapi: consentManager.configuration.uspapi || undefined, // TODO: https://transcend.height.app/T-23919 - reconsider simpler yml shape syncGroups: consentManager.configuration.syncGroups || undefined, - theme: !consentManagerTheme - ? undefined - : { + theme: consentManagerTheme + ? { primaryColor: consentManagerTheme.primaryColor || undefined, fontColor: consentManagerTheme.fontColor || undefined, privacyPolicy: consentManagerTheme.privacyPolicy || undefined, prompt: consentManagerTheme.prompt, - }, + } + : undefined, experiences: consentManagerExperiences.map((experience) => ({ name: experience.name, displayName: experience.displayName || undefined, @@ -504,30 +503,30 @@ export async function pullTranscendConfiguration( return { title, type, - 'sub-type': subType, + "sub-type": subType, placeholder, description, - 'is-required': isRequired, - 'reference-id': referenceId, - 'display-logic': + "is-required": isRequired, + "reference-id": referenceId, + "display-logic": displayLogicParsed && Object.keys(displayLogicParsed).length > 0 ? { action: displayLogicParsed.action!, rule: displayLogicParsed.rule ? { - 'depends-on-question-reference-id': + "depends-on-question-reference-id": displayLogicParsed.rule .dependsOnQuestionReferenceId, - 'comparison-operator': + "comparison-operator": displayLogicParsed.rule.comparisonOperator, - 'comparison-operands': + "comparison-operands": displayLogicParsed.rule.comparisonOperands, } : undefined, - 'nested-rule': displayLogicParsed.nestedRule + "nested-rule": displayLogicParsed.nestedRule ? { - 'logic-operator': + "logic-operator": displayLogicParsed.nestedRule.logicOperator, rules: /* eslint-disable @typescript-eslint/no-explicit-any */ @@ -537,11 +536,11 @@ export async function pullTranscendConfiguration( .nestedRule as AssessmentNestedRule ).rules || [] ).map((rule) => ({ - 'depends-on-question-reference-id': + "depends-on-question-reference-id": rule.dependsOnQuestionReferenceId, - 'comparison-operator': + "comparison-operator": rule.comparisonOperator, - 'comparison-operands': + "comparison-operands": rule.comparisonOperands, })), /* eslint-enable @typescript-eslint/no-explicit-any */ @@ -549,55 +548,55 @@ export async function pullTranscendConfiguration( : undefined, } : undefined, - 'risk-logic': riskLogic.map((logic): RiskLogicInput => { + "risk-logic": riskLogic.map((logic): RiskLogicInput => { const parsed = parseAssessmentRiskLogic(logic); return { - 'risk-level': parsed.riskAssignment?.riskLevelId, - 'comparison-operands': parsed.comparisonOperands, - 'comparison-operator': parsed.comparisonOperator, + "risk-level": parsed.riskAssignment?.riskLevelId, + "comparison-operands": parsed.comparisonOperands, + "comparison-operator": parsed.comparisonOperator, }; }), - 'risk-categories': riskCategories.map(({ title }) => title), - 'risk-framework': riskFramework?.title, - 'answer-options': answerOptions.map(({ value }) => ({ + "risk-categories": riskCategories.map(({ title }) => title), + "risk-framework": riskFramework?.title, + "answer-options": answerOptions.map(({ value }) => ({ value, })), - 'selected-answers': selectedAnswers.map(({ value }) => value), - 'allowed-mime-types': allowedMimeTypes, - 'allow-select-other': allowSelectOther, - 'sync-model': syncModel || undefined, - 'sync-column': syncColumn || undefined, - 'attribute-key': attributeKey?.name, - 'require-risk-evaluation': requireRiskEvaluation, - 'require-risk-matrix-evaluation': requireRiskMatrixEvaluation, + "selected-answers": selectedAnswers.map(({ value }) => value), + "allowed-mime-types": allowedMimeTypes, + "allow-select-other": allowSelectOther, + "sync-model": syncModel || undefined, + "sync-column": syncColumn || undefined, + "attribute-key": attributeKey?.name, + "require-risk-evaluation": requireRiskEvaluation, + "require-risk-matrix-evaluation": requireRiskMatrixEvaluation, }; - }, + } ), assignees: assignees.map(({ email }) => email), - 'external-assignees': externalAssignees.map(({ email }) => email), - 'is-reviewed': isReviewed, - }), + "external-assignees": externalAssignees.map(({ email }) => email), + "is-reviewed": isReviewed, + }) ), creator: creator?.email, description, status, assignees: assignees.map(({ email }) => email), - 'external-assignees': externalAssignees.map(({ email }) => email), + "external-assignees": externalAssignees.map(({ email }) => email), reviewers: reviewers.map(({ email }) => email), locked: isLocked, archived: isArchived, external: isExternallyCreated, - 'title-is-internal': titleIsInternal, - 'due-date': dueDate || undefined, - 'created-at': createdAt || undefined, - 'assigned-at': assignedAt || undefined, - 'submitted-at': submittedAt || undefined, - 'approved-at': approvedAt || undefined, - 'rejected-at': rejectedAt || undefined, - 'retention-schedule': retentionSchedule + "title-is-internal": titleIsInternal, + "due-date": dueDate || undefined, + "created-at": createdAt || undefined, + "assigned-at": assignedAt || undefined, + "submitted-at": submittedAt || undefined, + "approved-at": approvedAt || undefined, + "rejected-at": rejectedAt || undefined, + "retention-schedule": retentionSchedule ? { type: retentionSchedule.type, - 'duration-days': retentionSchedule.durationDays, + "duration-days": retentionSchedule.durationDays, operand: retentionSchedule.operation, } : undefined, @@ -611,9 +610,9 @@ export async function pullTranscendConfiguration( title: category ? `${category} - ${name}` : purpose - ? `${purpose} - ${name}` - : title || name || type || '', - }), + ? `${purpose} - ${name}` + : title || name || type || "", + }) ), rows: syncedRows.map( ({ resourceType, title, name, category, type, purpose }) => ({ @@ -621,11 +620,11 @@ export async function pullTranscendConfiguration( title: category ? `${category} - ${name}` : purpose - ? `${purpose} - ${name}` - : title || name || type || '', - }), + ? `${purpose} - ${name}` + : title || name || type || "", + }) ), - }), + }) ); } @@ -634,7 +633,7 @@ export async function pullTranscendConfiguration( assessmentTemplates.length > 0 && resources.includes(TranscendPullResource.AssessmentTemplates) ) { - result['assessment-templates'] = assessmentTemplates.map( + result["assessment-templates"] = assessmentTemplates.map( ({ title, description, @@ -680,30 +679,30 @@ export async function pullTranscendConfiguration( return { title, type, - 'sub-type': subType, + "sub-type": subType, placeholder, description, - 'is-required': isRequired, - 'reference-id': referenceId, - 'display-logic': + "is-required": isRequired, + "reference-id": referenceId, + "display-logic": displayLogicParsed && Object.keys(displayLogicParsed).length > 0 ? { action: displayLogicParsed.action!, rule: displayLogicParsed.rule ? { - 'depends-on-question-reference-id': + "depends-on-question-reference-id": displayLogicParsed.rule .dependsOnQuestionReferenceId, - 'comparison-operator': + "comparison-operator": displayLogicParsed.rule.comparisonOperator, - 'comparison-operands': + "comparison-operands": displayLogicParsed.rule.comparisonOperands, } : undefined, - 'nested-rule': displayLogicParsed.nestedRule + "nested-rule": displayLogicParsed.nestedRule ? { - 'logic-operator': + "logic-operator": displayLogicParsed.nestedRule.logicOperator, rules: /* eslint-disable @typescript-eslint/no-explicit-any */ @@ -713,11 +712,11 @@ export async function pullTranscendConfiguration( .nestedRule as AssessmentNestedRule ).rules || [] ).map((rule) => ({ - 'depends-on-question-reference-id': + "depends-on-question-reference-id": rule.dependsOnQuestionReferenceId, - 'comparison-operator': + "comparison-operator": rule.comparisonOperator, - 'comparison-operands': + "comparison-operands": rule.comparisonOperands, })), /* eslint-enable @typescript-eslint/no-explicit-any */ @@ -725,48 +724,48 @@ export async function pullTranscendConfiguration( : undefined, } : undefined, - 'risk-logic': riskLogic.map((logic): RiskLogicInput => { + "risk-logic": riskLogic.map((logic): RiskLogicInput => { const parsed = parseAssessmentRiskLogic(logic); return { - 'risk-level': parsed.riskAssignment?.riskLevelId, - 'risk-matrix-row': parsed.riskAssignment?.riskMatrixRowId, - 'risk-matrix-column': + "risk-level": parsed.riskAssignment?.riskLevelId, + "risk-matrix-row": parsed.riskAssignment?.riskMatrixRowId, + "risk-matrix-column": parsed.riskAssignment?.riskMatrixColumnId, - 'comparison-operands': parsed.comparisonOperands, - 'comparison-operator': parsed.comparisonOperator, + "comparison-operands": parsed.comparisonOperands, + "comparison-operator": parsed.comparisonOperator, }; }), - 'risk-categories': riskCategories.map(({ title }) => title), - 'risk-framework': riskFramework?.title, - 'answer-options': answerOptions.map(({ value }) => ({ + "risk-categories": riskCategories.map(({ title }) => title), + "risk-framework": riskFramework?.title, + "answer-options": answerOptions.map(({ value }) => ({ value, })), - 'allowed-mime-types': allowedMimeTypes, - 'allow-select-other': allowSelectOther, - 'sync-model': syncModel || undefined, - 'sync-column': syncColumn || undefined, - 'attribute-key': attributeKey?.name, - 'require-risk-evaluation': requireRiskEvaluation, - 'require-risk-matrix-evaluation': requireRiskMatrixEvaluation, + "allowed-mime-types": allowedMimeTypes, + "allow-select-other": allowSelectOther, + "sync-model": syncModel || undefined, + "sync-column": syncColumn || undefined, + "attribute-key": attributeKey?.name, + "require-risk-evaluation": requireRiskEvaluation, + "require-risk-matrix-evaluation": requireRiskMatrixEvaluation, }; - }, + } ), - }), + }) ), status, source, creator: creator?.email, locked: isLocked, archived: isArchived, - 'created-at': createdAt || undefined, - 'retention-schedule': retentionSchedule + "created-at": createdAt || undefined, + "retention-schedule": retentionSchedule ? { type: retentionSchedule.type, - 'duration-days': retentionSchedule.durationDays, + "duration-days": retentionSchedule.durationDays, operand: retentionSchedule.operation, } : undefined, - }), + }) ); } @@ -776,7 +775,7 @@ export async function pullTranscendConfiguration( ({ title, content }): PromptInput => ({ title, content, - }), + }) ); } @@ -785,11 +784,11 @@ export async function pullTranscendConfiguration( promptPartials.length > 0 && resources.includes(TranscendPullResource.PromptPartials) ) { - result['prompt-partials'] = promptPartials.map( + result["prompt-partials"] = promptPartials.map( ({ title, content }): PromptPartialInput => ({ title, content, - }), + }) ); } @@ -798,12 +797,12 @@ export async function pullTranscendConfiguration( promptGroups.length > 0 && resources.includes(TranscendPullResource.PromptGroups) ) { - result['prompt-groups'] = promptGroups.map( + result["prompt-groups"] = promptGroups.map( ({ title, description, prompts }): PromptGroupInput => ({ title, description, prompts: prompts.map(({ title }) => title), - }), + }) ); } @@ -821,12 +820,12 @@ export async function pullTranscendConfiguration( }): TeamInput => ({ name, description, - 'sso-department': ssoDepartment || undefined, - 'sso-group': ssoGroup || undefined, - 'sso-title': ssoTitle || undefined, + "sso-department": ssoDepartment || undefined, + "sso-group": ssoGroup || undefined, + "sso-title": ssoTitle || undefined, users: users.map(({ email }) => email), scopes: scopes.map(({ name }) => name), - }), + }) ); } @@ -835,7 +834,7 @@ export async function pullTranscendConfiguration( dataSubjects.length > 0 && resources.includes(TranscendPullResource.DataSubjects) ) { - result['data-subjects'] = dataSubjects.map( + result["data-subjects"] = dataSubjects.map( ({ type, title, @@ -848,7 +847,7 @@ export async function pullTranscendConfiguration( active, adminDashboardDefaultSilentMode, actions: actions.map(({ type }) => type), - }), + }) ); } @@ -859,7 +858,7 @@ export async function pullTranscendConfiguration( title: title?.defaultMessage, content: versions?.[0]?.content?.defaultMessage, disabledLocales, - }), + }) ); } @@ -876,17 +875,18 @@ export async function pullTranscendConfiguration( defaultMessage, targetReactIntlId: targetReactIntlId || undefined, translations: translations.reduce( - (acc, { locale, value }) => Object.assign(acc, { [locale]: value }), - {} as Record, + (accumulator, { locale, value }) => + Object.assign(accumulator, { [locale]: value }), + {} as Record ), - }), + }) ); } // Save privacy center if (privacyCenters.length > 0) { const privacyCenter = privacyCenters[0]; - result['privacy-center'] = { + result["privacy-center"] = { isDisabled: privacyCenter.isDisabled, showPrivacyRequestButton: privacyCenter.showPrivacyRequestButton, showPolicies: privacyCenter.showPolicies, @@ -914,7 +914,7 @@ export async function pullTranscendConfiguration( businessEntities.length > 0 && resources.includes(TranscendPullResource.BusinessEntities) ) { - result['business-entities'] = businessEntities.map( + result["business-entities"] = businessEntities.map( ({ title, description, @@ -936,7 +936,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }), + }) ); } @@ -963,7 +963,7 @@ export async function pullTranscendConfiguration( waitingPeriod, regionDetectionMethod, regionList: regionList.length > 0 ? regionList : undefined, - }), + }) ); } @@ -1003,7 +1003,7 @@ export async function pullTranscendConfiguration( displayTitle: displayTitle?.defaultMessage, displayDescription: displayDescription?.defaultMessage, displayOrder, - }), + }) ); } @@ -1031,7 +1031,7 @@ export async function pullTranscendConfiguration( codeInterpreterEnabled, retrievalEnabled, prompt: prompt?.title, - 'large-language-model': { + "large-language-model": { name: largeLanguageModel.name, client: largeLanguageModel.client, }, @@ -1041,15 +1041,15 @@ export async function pullTranscendConfiguration( owners && owners.length > 0 ? owners.map(({ email }) => email) : undefined, - 'agent-functions': + "agent-functions": agentFunctions && agentFunctions.length > 0 ? agentFunctions.map(({ name }) => name) : undefined, - 'agent-files': + "agent-files": agentFiles && agentFiles.length > 0 ? agentFiles.map(({ name }) => name) : undefined, - }), + }) ); } @@ -1058,7 +1058,7 @@ export async function pullTranscendConfiguration( actionItems.length > 0 && resources.includes(TranscendPullResource.ActionItems) ) { - result['action-items'] = actionItems.map( + result["action-items"] = actionItems.map( ({ teams, users, @@ -1088,7 +1088,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }), + }) ); } @@ -1097,7 +1097,7 @@ export async function pullTranscendConfiguration( actionItemCollections.length > 0 && resources.includes(TranscendPullResource.ActionItemCollections) ) { - result['action-item-collections'] = actionItemCollections.map( + result["action-item-collections"] = actionItemCollections.map( ({ title, description, @@ -1108,7 +1108,7 @@ export async function pullTranscendConfiguration( description: description || undefined, hidden, productLine, - }), + }) ); } @@ -1117,12 +1117,12 @@ export async function pullTranscendConfiguration( agentFunctions.length > 0 && resources.includes(TranscendPullResource.AgentFunctions) ) { - result['agent-functions'] = agentFunctions.map( + result["agent-functions"] = agentFunctions.map( ({ name, description, parameters }): AgentFunctionInput => ({ name, description, parameters: JSON.stringify(parameters), - }), + }) ); } @@ -1131,14 +1131,14 @@ export async function pullTranscendConfiguration( agentFiles.length > 0 && resources.includes(TranscendPullResource.AgentFiles) ) { - result['agent-files'] = agentFiles.map( + result["agent-files"] = agentFiles.map( ({ name, description, fileId, size, purpose }): AgentFileInput => ({ name, description, fileId, size, purpose, - }), + }) ); } @@ -1180,7 +1180,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }), + }) ); } @@ -1189,7 +1189,7 @@ export async function pullTranscendConfiguration( dataCategories.length > 0 && resources.includes(TranscendPullResource.DataCategories) ) { - result['data-categories'] = dataCategories.map( + result["data-categories"] = dataCategories.map( ({ name, category, @@ -1213,7 +1213,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }), + }) ); } @@ -1222,7 +1222,7 @@ export async function pullTranscendConfiguration( processingPurposes.length > 0 && resources.includes(TranscendPullResource.ProcessingPurposes) ) { - result['processing-purposes'] = processingPurposes.map( + result["processing-purposes"] = processingPurposes.map( ({ name, purpose, @@ -1244,7 +1244,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }), + }) ); } @@ -1253,7 +1253,7 @@ export async function pullTranscendConfiguration( dataFlows.length > 0 && resources.includes(TranscendPullResource.DataFlows) ) { - result['data-flows'] = dataFlows.map( + result["data-flows"] = dataFlows.map( ({ value, type, @@ -1277,7 +1277,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }), + }) ); } @@ -1307,7 +1307,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }), + }) ); } @@ -1333,7 +1333,7 @@ export async function pullTranscendConfiguration( color: color || undefined, description, })), - }), + }) ); } @@ -1359,15 +1359,15 @@ export async function pullTranscendConfiguration( title, description: description || undefined, trackingType, - 'default-consent': defaultConsent, + "default-consent": defaultConsent, configurable, - 'show-in-consent-manager': showInConsentManager, - 'show-in-privacy-center': showInPrivacyCenter, - 'is-active': isActive, - 'display-order': displayOrder, - 'opt-out-signals': optOutSignals.length > 0 ? optOutSignals : undefined, - 'auth-level': authLevel || undefined, - 'preference-topics': topics.map( + "show-in-consent-manager": showInConsentManager, + "show-in-privacy-center": showInPrivacyCenter, + "is-active": isActive, + "display-order": displayOrder, + "opt-out-signals": optOutSignals.length > 0 ? optOutSignals : undefined, + "auth-level": authLevel || undefined, + "preference-topics": topics.map( ({ title, type, @@ -1379,8 +1379,8 @@ export async function pullTranscendConfiguration( title: title.defaultMessage, type, description: displayDescription.defaultMessage, - 'default-configuration': defaultConfiguration, - 'show-in-privacy-center': showInPrivacyCenter, + "default-configuration": defaultConfiguration, + "show-in-privacy-center": showInPrivacyCenter, ...(preferenceOptionValues.length > 0 ? { options: preferenceOptionValues.map(({ title, slug }) => ({ @@ -1389,9 +1389,9 @@ export async function pullTranscendConfiguration( })), } : {}), - }), + }) ), - }), + }) ); } @@ -1428,22 +1428,22 @@ export async function pullTranscendConfiguration( title, url: url || undefined, type, - 'input-identifier': inputIdentifier?.name, - 'output-identifiers': identifiers.map(({ name }) => name), - 'privacy-actions': + "input-identifier": inputIdentifier?.name, + "output-identifiers": identifiers.map(({ name }) => name), + "privacy-actions": Object.values(RequestAction).length === actions.length ? undefined : actions, testRegex: testRegex || undefined, lookerQueryTitle: lookerQueryTitle || undefined, - expirationDuration: parseInt(expirationDuration, 10), + expirationDuration: Number.parseInt(expirationDuration, 10), transitionRequestStatus: transitionRequestStatus || undefined, phoneNumbers: phoneNumbers && phoneNumbers.length > 0 ? phoneNumbers : undefined, regionList: regionList && regionList.length > 0 ? regionList : undefined, - 'data-subjects': dataSubjects.map(({ type }) => type), - }), + "data-subjects": dataSubjects.map(({ type }) => type), + }) ); } @@ -1452,8 +1452,8 @@ export async function pullTranscendConfiguration( dataSilos.length > 0 && resources.includes(TranscendPullResource.DataSilos) ) { - const indexedDataSubjects = keyBy(dataSubjects, 'type'); - result['data-silos'] = dataSilos.map( + const indexedDataSubjects = keyBy(dataSubjects, "type"); + result["data-silos"] = dataSilos.map( ([ { title, @@ -1486,16 +1486,16 @@ export async function pullTranscendConfiguration( title, description, integrationName: type, - 'outer-type': outerType || undefined, + "outer-type": outerType || undefined, url: url || undefined, - 'api-key-title': apiKeys[0]?.title, - 'identity-keys': identifiers + "api-key-title": apiKeys[0]?.title, + "identity-keys": identifiers .filter(({ isConnected }) => isConnected) .map(({ name }) => name), ...(dependentDataSilos.length > 0 ? { - 'deletion-dependencies': dependentDataSilos.map( - ({ title }) => title, + "deletion-dependencies": dependentDataSilos.map( + ({ title }) => title ), } : {}), @@ -1514,23 +1514,23 @@ export async function pullTranscendConfiguration( country: country || undefined, countrySubDivision: countrySubDivision || undefined, disabled: !isLive, - 'data-subjects': + "data-subjects": subjectBlocklist.length > 0 ? convertToDataSubjectAllowlist( subjectBlocklist.map(({ type }) => type), - indexedDataSubjects, + indexedDataSubjects ) : undefined, ...(catalog.hasAvcFunctionality ? { - 'email-settings': { - 'notify-email-address': notifyEmailAddress || undefined, - 'send-frequency': promptAVendorEmailSendFrequency, - 'send-type': promptAVendorEmailSendType, - 'include-identifiers-attachment': + "email-settings": { + "notify-email-address": notifyEmailAddress || undefined, + "send-frequency": promptAVendorEmailSendFrequency, + "send-type": promptAVendorEmailSendType, + "include-identifiers-attachment": promptAVendorEmailIncludeIdentifiersAttachment, - 'completion-link-type': promptAVendorEmailCompletionLinkType, - 'manual-work-retry-frequency': manualWorkRetryFrequency, + "completion-link-type": promptAVendorEmailCompletionLinkType, + "manual-work-retry-frequency": manualWorkRetryFrequency, }, } : {}), @@ -1550,18 +1550,18 @@ export async function pullTranscendConfiguration( ...(dataPoint.path.length > 0 ? { path: dataPoint.path } : {}), ...(dataPoint.dataCollection?.title ? { - 'data-collection-tag': + "data-collection-tag": dataPoint.dataCollection.title.defaultMessage, } : {}), ...(dataPoint.dbIntegrationQueries.length > 0 ? { - 'privacy-action-queries': mapValues( - keyBy(dataPoint.dbIntegrationQueries, 'requestType'), + "privacy-action-queries": mapValues( + keyBy(dataPoint.dbIntegrationQueries, "requestType"), (databaseIntegrationQuery) => databaseIntegrationQuery.suggestedQuery || databaseIntegrationQuery.query || - undefined, + undefined ), } : {}), @@ -1577,10 +1577,10 @@ export async function pullTranscendConfiguration( ...(includeGuessedCategories && field.pendingCategoryGuesses ? { - 'guessed-categories': + "guessed-categories": field.pendingCategoryGuesses .filter( - (guess) => guess.status === 'PENDING', + (guess) => guess.status === "PENDING" ) .map((guess) => ({ category: { @@ -1594,33 +1594,32 @@ export async function pullTranscendConfiguration( })), } : {}), - 'access-request-visibility-enabled': + "access-request-visibility-enabled": field.accessRequestVisibilityEnabled, - 'erasure-request-redaction-enabled': + "erasure-request-redaction-enabled": field.erasureRequestRedactionEnabled, attributes: field.attributeValues !== undefined && field.attributeValues.length > 0 ? formatAttributeValues(field.attributeValues) : undefined, - }), + }) ) .sort((a, b) => a.key.localeCompare(b.key)), } : {}), - 'privacy-actions': dataPoint.actionSettings + "privacy-actions": dataPoint.actionSettings .filter(({ active }) => active) .map(({ type }) => type), - }), + }) ) .sort((a, b) => [...(a.path ?? []), a.key] - .join('.') - .localeCompare([...(b.path ?? []), b.key].join('.')), + .join(".") + .localeCompare([...(b.path ?? []), b.key].join(".")) ), - }), + }) ); } return result; } -/* eslint-enable max-lines */ diff --git a/src/lib/graphql/syncActionItemCollections.ts b/src/lib/graphql/syncActionItemCollections.ts index 4c3f257b..5e31b829 100644 --- a/src/lib/graphql/syncActionItemCollections.ts +++ b/src/lib/graphql/syncActionItemCollections.ts @@ -1,18 +1,18 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy } from 'lodash-es'; -import { ActionItemCollectionInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { keyBy } from "lodash-es"; +import { ActionItemCollectionInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; import { ActionItemCollection, fetchAllActionItemCollections, -} from './fetchAllActionItemCollections'; +} from "./fetchAllActionItemCollections"; import { CREATE_ACTION_ITEM_COLLECTION, UPDATE_ACTION_ITEM_COLLECTION, -} from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Input to create a new action item collection @@ -23,11 +23,11 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; */ export async function createActionItemCollection( client: GraphQLClient, - actionItemCollection: ActionItemCollectionInput, -): Promise> { + actionItemCollection: ActionItemCollectionInput +): Promise> { const input = { title: actionItemCollection.title, - description: actionItemCollection.description || '', + description: actionItemCollection.description || "", hidden: actionItemCollection.hidden || false, productLine: actionItemCollection.productLine, }; @@ -54,7 +54,7 @@ export async function createActionItemCollection( export async function updateActionItemCollection( client: GraphQLClient, input: ActionItemCollectionInput, - actionItemCollectionId: string, + actionItemCollectionId: string ): Promise { await makeGraphQLRequest(client, UPDATE_ACTION_ITEM_COLLECTION, { input: { @@ -76,27 +76,28 @@ export async function updateActionItemCollection( */ export async function syncActionItemCollections( client: GraphQLClient, - inputs: ActionItemCollectionInput[], + inputs: ActionItemCollectionInput[] ): Promise { let encounteredError = false; // Fetch existing logger.info( - colors.magenta(`Syncing "${inputs.length}" action item collections...`), + colors.magenta(`Syncing "${inputs.length}" action item collections...`) ); // Fetch existing - const existingActionItemCollections = - await fetchAllActionItemCollections(client); + const existingActionItemCollections = await fetchAllActionItemCollections( + client + ); // Look up by title - const collectionByTitle: { [k in string]: ActionItemCollection } = keyBy( + const collectionByTitle: Record = keyBy( existingActionItemCollections, - 'title', + "title" ); // Create new actionItems const newCollections = inputs.filter( - (input) => !collectionByTitle[input.title], + (input) => !collectionByTitle[input.title] ); // Create new actionItem collections @@ -105,15 +106,15 @@ export async function syncActionItemCollections( await createActionItemCollection(client, input); logger.info( colors.green( - `Successfully created action item collection "${input.title}"!`, - ), + `Successfully created action item collection "${input.title}"!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to create action item collection "${input.title}"! - ${err.message}`, - ), + `Failed to create action item collection "${input.title}"! - ${error.message}` + ) ); } }); @@ -127,15 +128,15 @@ export async function syncActionItemCollections( await updateActionItemCollection(client, input, actionItemId); logger.info( colors.green( - `Successfully synced action item collection "${input.title}"!`, - ), + `Successfully synced action item collection "${input.title}"!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync action item collection "${input.title}"! - ${err.message}`, - ), + `Failed to sync action item collection "${input.title}"! - ${error.message}` + ) ); } }); diff --git a/src/lib/graphql/syncActionItems.ts b/src/lib/graphql/syncActionItems.ts index 4c4060c4..c013a2c1 100644 --- a/src/lib/graphql/syncActionItems.ts +++ b/src/lib/graphql/syncActionItems.ts @@ -1,17 +1,17 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { chunk, keyBy, uniq } from 'lodash-es'; -import { ActionItemInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { chunk, keyBy, uniq } from "lodash-es"; +import { ActionItemInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; import { ActionItemCollection, fetchAllActionItemCollections, -} from './fetchAllActionItemCollections'; -import { ActionItem, fetchAllActionItems } from './fetchAllActionItems'; -import { Attribute, fetchAllAttributes } from './fetchAllAttributes'; -import { CREATE_ACTION_ITEMS, UPDATE_ACTION_ITEMS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "./fetchAllActionItemCollections"; +import { ActionItem, fetchAllActionItems } from "./fetchAllActionItems"; +import { Attribute, fetchAllAttributes } from "./fetchAllAttributes"; +import { CREATE_ACTION_ITEMS, UPDATE_ACTION_ITEMS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Input to create a new actionItem @@ -24,10 +24,10 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; export async function createActionItems( client: GraphQLClient, actionItems: ActionItemInput[], - actionItemCollectionByTitle: { [k in string]: ActionItemCollection }, + actionItemCollectionByTitle: Record, // TODO: https://transcend.height.app/T-38961 - insert attributes // eslint-disable-next-line @typescript-eslint/no-unused-vars - attributeKeysByName: { [k in string]: Attribute } = {}, + attributeKeysByName: Record = {} ): Promise { // TODO: https://transcend.height.app/T-38961 - insert attributes // const getAttribute = (key: string): string => { @@ -62,7 +62,7 @@ export async function createActionItems( } : {}), collectionIds: actionItem.collections.map( - (collectionTitle) => actionItemCollectionByTitle[collectionTitle].id, + (collectionTitle) => actionItemCollectionByTitle[collectionTitle].id ), })), }); @@ -81,9 +81,7 @@ export async function updateActionItem( client: GraphQLClient, input: ActionItemInput, actionItemId: string, - attributeKeysByName: { - [k in string]: Attribute; - } = {}, + attributeKeysByName: Record = {} ): Promise { const getAttribute = (key: string): string => { const existing = attributeKeysByName[key]; @@ -125,11 +123,11 @@ export async function updateActionItem( function actionItemToUniqueCode({ title, collections, -}: Pick): string { +}: Pick): string { return `${title}-${collections .map((c) => c.title) .sort() - .join('-')}`; + .join("-")}`; } /** @@ -141,8 +139,8 @@ function actionItemToUniqueCode({ function actionItemInputToUniqueCode({ title, collections, -}: Pick): string { - return `${title}-${collections.sort().join('-')}`; +}: Pick): string { + return `${title}-${collections.sort().join("-")}`; } /** @@ -154,7 +152,7 @@ function actionItemInputToUniqueCode({ */ export async function syncActionItems( client: GraphQLClient, - inputs: ActionItemInput[], + inputs: ActionItemInput[] ): Promise { let encounteredError = false; // Fetch existing @@ -162,7 +160,7 @@ export async function syncActionItems( // Determine if attributes are syncing const hasAttributes = inputs.some( - (input) => input.attributes && input.attributes.length > 0, + (input) => input.attributes && input.attributes.length > 0 ); // Fetch existing @@ -174,29 +172,29 @@ export async function syncActionItems( ]); // Look up by title - const actionItemCollectionByTitle: { [k in string]: ActionItemCollection } = - keyBy(existingActionItemCollections, 'title'); - const actionItemByTitle: { [k in string]: ActionItem } = keyBy( + const actionItemCollectionByTitle: Record = + keyBy(existingActionItemCollections, "title"); + const actionItemByTitle: Record = keyBy( existingActionItems, - actionItemToUniqueCode, + actionItemToUniqueCode ); - const attributeKeysByName = keyBy(attributeKeys, 'name'); - const actionItemByCxId: { [k in string]: ActionItem } = keyBy( + const attributeKeysByName = keyBy(attributeKeys, "name"); + const actionItemByCxId: Record = keyBy( existingActionItems.filter((x) => !!x.customerExperienceActionItemIds), - ({ customerExperienceActionItemIds }) => customerExperienceActionItemIds[0], + ({ customerExperienceActionItemIds }) => customerExperienceActionItemIds[0] ); // Ensure all collections exist const missingCollections = uniq( - inputs.map((input) => input.collections).flat(), + inputs.flatMap((input) => input.collections) ).filter((collectionTitle) => !actionItemCollectionByTitle[collectionTitle]); if (missingCollections.length > 0) { logger.info( colors.red( `Missing action item collections: "${missingCollections.join( - '", "', - )}" - please create them first!`, - ), + '", "' + )}" - please create them first!` + ) ); return false; } @@ -205,30 +203,30 @@ export async function syncActionItems( const newActionItems = inputs.filter( (input) => !actionItemByTitle[actionItemInputToUniqueCode(input)] && - !actionItemByCxId[input.customerExperienceActionItemId!], + !actionItemByCxId[input.customerExperienceActionItemId!] ); // Create new actionItems if (newActionItems.length > 0) { try { logger.info( - colors.magenta(`Creating "${newActionItems.length}" actionItems...`), + colors.magenta(`Creating "${newActionItems.length}" actionItems...`) ); await createActionItems( client, newActionItems, actionItemCollectionByTitle, - attributeKeysByName, + attributeKeysByName ); logger.info( colors.green( - `Successfully created "${newActionItems.length}" actionItems!`, - ), + `Successfully created "${newActionItems.length}" actionItems!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create action items! - ${err.message}`), + colors.red(`Failed to create action items! - ${error.message}`) ); } } @@ -245,14 +243,14 @@ export async function syncActionItems( try { await updateActionItem(client, input, actionItemId, attributeKeysByName); logger.info( - colors.green(`Successfully synced action item "${input.title}"!`), + colors.green(`Successfully synced action item "${input.title}"!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync action item "${input.title}"! - ${err.message}`, - ), + `Failed to sync action item "${input.title}"! - ${error.message}` + ) ); } }); diff --git a/src/lib/graphql/syncAgentFiles.ts b/src/lib/graphql/syncAgentFiles.ts index 9ed5a59d..c2bd90fa 100644 --- a/src/lib/graphql/syncAgentFiles.ts +++ b/src/lib/graphql/syncAgentFiles.ts @@ -1,12 +1,12 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy } from 'lodash-es'; -import { AgentFileInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { AgentFile, fetchAllAgentFiles } from './fetchAllAgentFiles'; -import { CREATE_AGENT_FILE, UPDATE_AGENT_FILES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { keyBy } from "lodash-es"; +import { AgentFileInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; +import { AgentFile, fetchAllAgentFiles } from "./fetchAllAgentFiles"; +import { CREATE_AGENT_FILE, UPDATE_AGENT_FILES } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Input to create a new agent file @@ -17,8 +17,8 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; */ export async function createAgentFile( client: GraphQLClient, - agentFile: AgentFileInput, -): Promise> { + agentFile: AgentFileInput +): Promise> { const input = { name: agentFile.name, description: agentFile.description, @@ -50,7 +50,7 @@ export async function createAgentFile( */ export async function updateAgentFiles( client: GraphQLClient, - agentFileIdPairs: [AgentFileInput, string][], + agentFileIdPairs: [AgentFileInput, string][] ): Promise { await makeGraphQLRequest(client, UPDATE_AGENT_FILES, { input: { @@ -75,7 +75,7 @@ export async function updateAgentFiles( */ export async function syncAgentFiles( client: GraphQLClient, - inputs: AgentFileInput[], + inputs: AgentFileInput[] ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" agent files...`)); @@ -86,9 +86,10 @@ export async function syncAgentFiles( const existingAgentFiles = await fetchAllAgentFiles(client); // Look up by name - const agentFileByName: { - [k in string]: Pick; - } = keyBy(existingAgentFiles, 'name'); + const agentFileByName: Record< + string, + Pick + > = keyBy(existingAgentFiles, "name"); // Create new agent files const newAgentFiles = inputs.filter((input) => !agentFileByName[input.name]); @@ -99,14 +100,14 @@ export async function syncAgentFiles( const newAgentFile = await createAgentFile(client, agentFile); agentFileByName[newAgentFile.name] = newAgentFile; logger.info( - colors.green(`Successfully synced agent file "${agentFile.name}"!`), + colors.green(`Successfully synced agent file "${agentFile.name}"!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync agent file "${agentFile.name}"! - ${err.message}`, - ), + `Failed to sync agent file "${agentFile.name}"! - ${error.message}` + ) ); } }); @@ -116,17 +117,17 @@ export async function syncAgentFiles( logger.info(colors.magenta(`Updating "${inputs.length}" agent files!`)); await updateAgentFiles( client, - inputs.map((input) => [input, agentFileByName[input.name].id]), + inputs.map((input) => [input, agentFileByName[input.name].id]) ); logger.info( - colors.green(`Successfully synced "${inputs.length}" agent files!`), + colors.green(`Successfully synced "${inputs.length}" agent files!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" agent files! - ${err.message}`, - ), + `Failed to sync "${inputs.length}" agent files! - ${error.message}` + ) ); } diff --git a/src/lib/graphql/syncAgentFunctions.ts b/src/lib/graphql/syncAgentFunctions.ts index 2fea50fd..53cc09aa 100644 --- a/src/lib/graphql/syncAgentFunctions.ts +++ b/src/lib/graphql/syncAgentFunctions.ts @@ -1,15 +1,15 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy } from 'lodash-es'; -import { AgentFunctionInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { keyBy } from "lodash-es"; +import { AgentFunctionInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; import { AgentFunction, fetchAllAgentFunctions, -} from './fetchAllAgentFunctions'; -import { CREATE_AGENT_FUNCTION, UPDATE_AGENT_FUNCTIONS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "./fetchAllAgentFunctions"; +import { CREATE_AGENT_FUNCTION, UPDATE_AGENT_FUNCTIONS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Input to create a new agent function @@ -20,8 +20,8 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; */ export async function createAgentFunction( client: GraphQLClient, - agentFunction: AgentFunctionInput, -): Promise> { + agentFunction: AgentFunctionInput +): Promise> { const input = { name: agentFunction.name, description: agentFunction.description, @@ -50,7 +50,7 @@ export async function createAgentFunction( */ export async function updateAgentFunctions( client: GraphQLClient, - agentFunctionIdPairs: [AgentFunctionInput, string][], + agentFunctionIdPairs: [AgentFunctionInput, string][] ): Promise { await makeGraphQLRequest(client, UPDATE_AGENT_FUNCTIONS, { input: { @@ -73,7 +73,7 @@ export async function updateAgentFunctions( */ export async function syncAgentFunctions( client: GraphQLClient, - inputs: AgentFunctionInput[], + inputs: AgentFunctionInput[] ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" agent functions...`)); @@ -84,13 +84,14 @@ export async function syncAgentFunctions( const existingAgentFunctions = await fetchAllAgentFunctions(client); // Look up by name - const agentFunctionByName: { - [k in string]: Pick; - } = keyBy(existingAgentFunctions, 'name'); + const agentFunctionByName: Record< + string, + Pick + > = keyBy(existingAgentFunctions, "name"); // Create new agent functions const newAgentFunctions = inputs.filter( - (input) => !agentFunctionByName[input.name], + (input) => !agentFunctionByName[input.name] ); // Create new agent functions @@ -100,15 +101,15 @@ export async function syncAgentFunctions( agentFunctionByName[newAgentFunction.name] = newAgentFunction; logger.info( colors.green( - `Successfully synced agent function "${agentFunction.name}"!`, - ), + `Successfully synced agent function "${agentFunction.name}"!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync agent function "${agentFunction.name}"! - ${err.message}`, - ), + `Failed to sync agent function "${agentFunction.name}"! - ${error.message}` + ) ); } }); @@ -118,17 +119,17 @@ export async function syncAgentFunctions( logger.info(colors.magenta(`Updating "${inputs.length}" agent functions!`)); await updateAgentFunctions( client, - inputs.map((input) => [input, agentFunctionByName[input.name].id]), + inputs.map((input) => [input, agentFunctionByName[input.name].id]) ); logger.info( - colors.green(`Successfully synced "${inputs.length}" agent functions!`), + colors.green(`Successfully synced "${inputs.length}" agent functions!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" agent functions! - ${err.message}`, - ), + `Failed to sync "${inputs.length}" agent functions! - ${error.message}` + ) ); } diff --git a/src/lib/graphql/syncAgents.ts b/src/lib/graphql/syncAgents.ts index 31e731b9..974f39d0 100644 --- a/src/lib/graphql/syncAgents.ts +++ b/src/lib/graphql/syncAgents.ts @@ -1,12 +1,12 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy } from 'lodash-es'; -import { AgentInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { Agent, fetchAllAgents } from './fetchAllAgents'; -import { CREATE_AGENT, UPDATE_AGENTS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { keyBy } from "lodash-es"; +import { AgentInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; +import { Agent, fetchAllAgents } from "./fetchAllAgents"; +import { CREATE_AGENT, UPDATE_AGENTS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Input to create a new agent @@ -17,16 +17,16 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; */ export async function createAgent( client: GraphQLClient, - agent: AgentInput, -): Promise> { + agent: AgentInput +): Promise> { const input = { name: agent.name, description: agent.description, codeInterpreterEnabled: agent.codeInterpreterEnabled, retrievalEnabled: agent.retrievalEnabled, promptTitle: agent.prompt, - largeLanguageModelName: agent['large-language-model'].name, - largeLanguageModelClient: agent['large-language-model'].client, + largeLanguageModelName: agent["large-language-model"].name, + largeLanguageModelClient: agent["large-language-model"].client, // TODO: https://transcend.height.app/T-32760 - agentFunction, agentFile // TODO: https://transcend.height.app/T-31994 - owners and teams }; @@ -51,7 +51,7 @@ export async function createAgent( */ export async function updateAgents( client: GraphQLClient, - agentIdParis: [AgentInput, string][], + agentIdParis: [AgentInput, string][] ): Promise { await makeGraphQLRequest(client, UPDATE_AGENTS, { input: { @@ -76,7 +76,7 @@ export async function updateAgents( */ export async function syncAgents( client: GraphQLClient, - inputs: AgentInput[], + inputs: AgentInput[] ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" agents...`)); @@ -87,9 +87,10 @@ export async function syncAgents( const existingAgents = await fetchAllAgents(client); // Look up by name - const agentByName: { - [k in string]: Pick; - } = keyBy(existingAgents, 'name'); + const agentByName: Record< + string, + Pick + > = keyBy(existingAgents, "name"); // Create new agents const newAgents = inputs.filter((input) => !agentByName[input.name]); @@ -100,10 +101,10 @@ export async function syncAgents( const newAgent = await createAgent(client, agent); agentByName[newAgent.name] = newAgent; logger.info(colors.green(`Successfully synced agent "${agent.name}"!`)); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync agent "${agent.name}"! - ${err.message}`), + colors.red(`Failed to sync agent "${agent.name}"! - ${error.message}`) ); } }); @@ -113,13 +114,15 @@ export async function syncAgents( logger.info(colors.magenta(`Updating "${inputs.length}" agents!`)); await updateAgents( client, - inputs.map((input) => [input, agentByName[input.name].id]), + inputs.map((input) => [input, agentByName[input.name].id]) ); logger.info(colors.green(`Successfully synced "${inputs.length}" agents!`)); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync "${inputs.length}" agents ! - ${err.message}`), + colors.red( + `Failed to sync "${inputs.length}" agents ! - ${error.message}` + ) ); } diff --git a/src/lib/graphql/syncAttribute.ts b/src/lib/graphql/syncAttribute.ts index 6ccfa73f..609da1bd 100644 --- a/src/lib/graphql/syncAttribute.ts +++ b/src/lib/graphql/syncAttribute.ts @@ -1,18 +1,18 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { difference, groupBy, keyBy } from 'lodash-es'; -import { AttributeInput } from '../../codecs'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; -import { Attribute } from './fetchAllAttributes'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { difference, groupBy, keyBy } from "lodash-es"; +import { AttributeInput } from "../../codecs"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; +import { Attribute } from "./fetchAllAttributes"; import { CREATE_ATTRIBUTE, CREATE_ATTRIBUTE_VALUES, DELETE_ATTRIBUTE_VALUE, UPDATE_ATTRIBUTE, UPDATE_ATTRIBUTE_VALUES, -} from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Sync attribute @@ -32,7 +32,7 @@ export async function syncAttribute( existingAttribute?: Attribute; /** When true, delete extra attributes not specified in the list of values */ deleteExtraAttributeValues?: boolean; - }, + } ): Promise { // attribute key input const input = { @@ -42,7 +42,16 @@ export async function syncAttribute( // create or update attribute key let attributeKeyId: string; - if (!existingAttribute) { + if (existingAttribute) { + await makeGraphQLRequest(client, UPDATE_ATTRIBUTE, { + attributeKeyId: existingAttribute.id, + description: existingAttribute.isCustom + ? attribute.description + : undefined, + ...input, + }); + attributeKeyId = existingAttribute.id; + } else { const { createAttributeKey: { attributeKey }, } = await makeGraphQLRequest<{ @@ -60,27 +69,18 @@ export async function syncAttribute( ...input, }); attributeKeyId = attributeKey.id; - } else { - await makeGraphQLRequest(client, UPDATE_ATTRIBUTE, { - attributeKeyId: existingAttribute.id, - description: existingAttribute.isCustom - ? attribute.description - : undefined, - ...input, - }); - attributeKeyId = existingAttribute.id; } // upsert attribute values - const existingAttributeMap = keyBy(existingAttribute?.values || [], 'name'); + const existingAttributeMap = keyBy(existingAttribute?.values || [], "name"); const { existingValues = [], newValues = [] } = groupBy( attribute.values || [], (field) => - existingAttributeMap[field.name] ? 'existingValues' : 'newValues', + existingAttributeMap[field.name] ? "existingValues" : "newValues" ); const removedValues = difference( (existingAttribute?.values || []).map(({ name }) => name), - (attribute.values || []).map(({ name }) => name), + (attribute.values || []).map(({ name }) => name) ); // Create new attribute values @@ -108,7 +108,7 @@ export async function syncAttribute( })), }); logger.info( - colors.green(`Updated ${existingValues.length} attribute values`), + colors.green(`Updated ${existingValues.length} attribute values`) ); } @@ -123,10 +123,10 @@ export async function syncAttribute( }, { concurrency: 10, - }, + } ); logger.info( - colors.green(`Deleted ${removedValues.length} attribute values`), + colors.green(`Deleted ${removedValues.length} attribute values`) ); } } diff --git a/src/lib/graphql/syncBusinessEntities.ts b/src/lib/graphql/syncBusinessEntities.ts index 8fec0a27..1fa10623 100644 --- a/src/lib/graphql/syncBusinessEntities.ts +++ b/src/lib/graphql/syncBusinessEntities.ts @@ -1,15 +1,15 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { chunk, keyBy } from 'lodash-es'; -import { BusinessEntityInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { chunk, keyBy } from "lodash-es"; +import { BusinessEntityInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; import { BusinessEntity, fetchAllBusinessEntities, -} from './fetchAllBusinessEntities'; -import { CREATE_BUSINESS_ENTITY, UPDATE_BUSINESS_ENTITIES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "./fetchAllBusinessEntities"; +import { CREATE_BUSINESS_ENTITY, UPDATE_BUSINESS_ENTITIES } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Input to create a new business entity @@ -20,7 +20,7 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; */ export async function createBusinessEntity( client: GraphQLClient, - businessEntity: BusinessEntityInput, + businessEntity: BusinessEntityInput ): Promise { const input = { title: businessEntity.title, @@ -55,7 +55,7 @@ export async function createBusinessEntity( */ export async function updateBusinessEntities( client: GraphQLClient, - businessEntityIdParis: [BusinessEntityInput, string][], + businessEntityIdParis: [BusinessEntityInput, string][] ): Promise { const chunkedUpdates = chunk(businessEntityIdParis, 100); await mapSeries(chunkedUpdates, async (chunked) => { @@ -86,11 +86,11 @@ export async function updateBusinessEntities( */ export async function syncBusinessEntities( client: GraphQLClient, - inputs: BusinessEntityInput[], + inputs: BusinessEntityInput[] ): Promise { // Fetch existing logger.info( - colors.magenta(`Syncing "${inputs.length}" business entities...`), + colors.magenta(`Syncing "${inputs.length}" business entities...`) ); let encounteredError = false; @@ -99,11 +99,11 @@ export async function syncBusinessEntities( const existingBusinessEntities = await fetchAllBusinessEntities(client); // Look up by title - const businessEntityByTitle = keyBy(existingBusinessEntities, 'title'); + const businessEntityByTitle = keyBy(existingBusinessEntities, "title"); // Create new business entities const newBusinessEntities = inputs.filter( - (input) => !businessEntityByTitle[input.title], + (input) => !businessEntityByTitle[input.title] ); // Create new business entities @@ -111,20 +111,20 @@ export async function syncBusinessEntities( try { const newBusinessEntity = await createBusinessEntity( client, - businessEntity, + businessEntity ); businessEntityByTitle[newBusinessEntity.title] = newBusinessEntity; logger.info( colors.green( - `Successfully synced business entity "${businessEntity.title}"!`, - ), + `Successfully synced business entity "${businessEntity.title}"!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync business entity "${businessEntity.title}"! - ${err.message}`, - ), + `Failed to sync business entity "${businessEntity.title}"! - ${error.message}` + ) ); } }); @@ -132,21 +132,21 @@ export async function syncBusinessEntities( // Update all business entities try { logger.info( - colors.magenta(`Updating "${inputs.length}" business entities!`), + colors.magenta(`Updating "${inputs.length}" business entities!`) ); await updateBusinessEntities( client, - inputs.map((input) => [input, businessEntityByTitle[input.title].id]), + inputs.map((input) => [input, businessEntityByTitle[input.title].id]) ); logger.info( - colors.green(`Successfully synced "${inputs.length}" business entities!`), + colors.green(`Successfully synced "${inputs.length}" business entities!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" business entities ! - ${err.message}`, - ), + `Failed to sync "${inputs.length}" business entities ! - ${error.message}` + ) ); } diff --git a/src/lib/graphql/syncCodePackages.ts b/src/lib/graphql/syncCodePackages.ts index 700a23ac..3e29b388 100644 --- a/src/lib/graphql/syncCodePackages.ts +++ b/src/lib/graphql/syncCodePackages.ts @@ -1,19 +1,19 @@ -import { CodePackageType } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { chunk, keyBy, uniq, uniqBy } from 'lodash-es'; -import { CodePackageInput, RepositoryInput } from '../../codecs'; -import { logger } from '../../logger'; -import { map, mapSeries } from '../bluebird-replace'; -import { CodePackage, fetchAllCodePackages } from './fetchAllCodePackages'; -import { CREATE_CODE_PACKAGE, UPDATE_CODE_PACKAGES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { syncRepositories } from './syncRepositories'; -import { syncSoftwareDevelopmentKits } from './syncSoftwareDevelopmentKits'; +import { CodePackageType } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { chunk, keyBy, uniq, uniqBy } from "lodash-es"; +import { CodePackageInput, RepositoryInput } from "../../codecs"; +import { logger } from "../../logger"; +import { map, mapSeries } from "../bluebird-replace"; +import { CodePackage, fetchAllCodePackages } from "./fetchAllCodePackages"; +import { CREATE_CODE_PACKAGE, UPDATE_CODE_PACKAGES } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { syncRepositories } from "./syncRepositories"; +import { syncSoftwareDevelopmentKits } from "./syncSoftwareDevelopmentKits"; const CHUNK_SIZE = 100; -const LOOKUP_SPLIT_KEY = '%%%%'; +const LOOKUP_SPLIT_KEY = "%%%%"; /** * Create a new code package @@ -47,7 +47,7 @@ export async function createCodePackage( teamIds?: string[]; /** Names of teams */ teamNames?: string[]; - }, + } ): Promise { const { createCodePackage: { codePackage }, @@ -61,7 +61,7 @@ export async function createCodePackage( input, }); logger.info( - colors.green(`Successfully created code package "${input.name}"!`), + colors.green(`Successfully created code package "${input.name}"!`) ); return codePackage; } @@ -100,7 +100,7 @@ export async function updateCodePackages( teamIds?: string[]; /** Names of teams */ teamNames?: string[]; - }[], + }[] ): Promise { const { updateCodePackages: { codePackages }, @@ -116,7 +116,7 @@ export async function updateCodePackages( }, }); logger.info( - colors.green(`Successfully updated ${inputs.length} code packages!`), + colors.green(`Successfully updated ${inputs.length} code packages!`) ); return codePackages; } @@ -132,7 +132,7 @@ export async function updateCodePackages( export async function syncCodePackages( client: GraphQLClient, codePackages: CodePackageInput[], - concurrency = 20, + concurrency = 20 ): Promise { let encounteredError = false; const [ @@ -145,40 +145,38 @@ export async function syncCodePackages( syncSoftwareDevelopmentKits( client, uniqBy( - codePackages - .map(({ type, softwareDevelopmentKits = [] }) => - softwareDevelopmentKits.map(({ name }) => ({ - name, - codePackageType: type, - })), - ) - .flat(), + codePackages.flatMap(({ type, softwareDevelopmentKits = [] }) => + softwareDevelopmentKits.map(({ name }) => ({ + name, + codePackageType: type, + })) + ), ({ name, codePackageType }) => - `${name}${LOOKUP_SPLIT_KEY}${codePackageType}`, + `${name}${LOOKUP_SPLIT_KEY}${codePackageType}` ), - concurrency, + concurrency ), // make sure all Repositories exist syncRepositories( client, - uniqBy(codePackages, 'repositoryName').map( + uniqBy(codePackages, "repositoryName").map( ({ repositoryName }) => ({ name: repositoryName, url: `https://github.com/${repositoryName}`, - }) as RepositoryInput, - ), + } as RepositoryInput) + ) ), ]); const softwareDevelopmentKitLookup = keyBy( existingSoftwareDevelopmentKits, ({ name, codePackageType }) => - `${name}${LOOKUP_SPLIT_KEY}${codePackageType}`, + `${name}${LOOKUP_SPLIT_KEY}${codePackageType}` ); const codePackagesLookup = keyBy( existingCodePackages, - ({ name, type }) => `${name}${LOOKUP_SPLIT_KEY}${type}`, + ({ name, type }) => `${name}${LOOKUP_SPLIT_KEY}${type}` ); // Determine which codePackages are new vs existing @@ -196,8 +194,8 @@ export async function syncCodePackages( try { logger.info( colors.magenta( - `Creating "${newCodePackages.length}" new code packages...`, - ), + `Creating "${newCodePackages.length}" new code packages...` + ) ); await map( newCodePackages, @@ -214,11 +212,11 @@ export async function syncCodePackages( ]; if (!sdk) { throw new Error( - `Failed to find SDK with name: "${name}"`, + `Failed to find SDK with name: "${name}"` ); } return sdk.id; - }), + }) ), } : {}), @@ -226,26 +224,28 @@ export async function syncCodePackages( }, { concurrency, - }, + } ); logger.info( colors.green( - `Successfully synced ${newCodePackages.length} code packages!`, - ), + `Successfully synced ${newCodePackages.length} code packages!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; - logger.info(colors.red(`Failed to create code packages! - ${err.message}`)); + logger.info( + colors.red(`Failed to create code packages! - ${error.message}`) + ); } // Update existing codePackages const existingCodePackageInputs = mapCodePackagesToExisting.filter( - (x): x is [CodePackageInput, string] => !!x[1], + (x): x is [CodePackageInput, string] => !!x[1] ); logger.info( colors.magenta( - `Updating "${existingCodePackageInputs.length}" code packages...`, - ), + `Updating "${existingCodePackageInputs.length}" code packages...` + ) ); const chunks = chunk(existingCodePackageInputs, CHUNK_SIZE); @@ -267,25 +267,25 @@ export async function syncCodePackages( ]; if (!sdk) { throw new Error( - `Failed to find SDK with name: "${name}"`, + `Failed to find SDK with name: "${name}"` ); } return sdk.id; - }), + }) ), } : {}), id, - }), - ), + }) + ) ); logger.info( - colors.green(`Successfully updated "${chunk.length}" code packages!`), + colors.green(`Successfully updated "${chunk.length}" code packages!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to update code packages! - ${err.message}`), + colors.red(`Failed to update code packages! - ${error.message}`) ); } }); diff --git a/src/lib/graphql/syncConfigurationToTranscend.ts b/src/lib/graphql/syncConfigurationToTranscend.ts index e08eb574..e5ac1eac 100644 --- a/src/lib/graphql/syncConfigurationToTranscend.ts +++ b/src/lib/graphql/syncConfigurationToTranscend.ts @@ -1,47 +1,46 @@ -/* eslint-disable max-lines */ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { TranscendInput } from '../../codecs'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; -import { fetchAllActions } from './fetchAllActions'; -import { fetchAllAttributes } from './fetchAllAttributes'; -import { fetchApiKeys } from './fetchApiKeys'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { TranscendInput } from "../../codecs"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; +import { fetchAllActions } from "./fetchAllActions"; +import { fetchAllAttributes } from "./fetchAllAttributes"; +import { fetchApiKeys } from "./fetchApiKeys"; import { ensureAllDataSubjectsExist, fetchAllDataSubjects, -} from './fetchDataSubjects'; +} from "./fetchDataSubjects"; import { fetchIdentifiersAndCreateMissing, Identifier, -} from './fetchIdentifiers'; -import { syncAction } from './syncAction'; -import { syncActionItemCollections } from './syncActionItemCollections'; -import { syncActionItems } from './syncActionItems'; -import { syncAgentFiles } from './syncAgentFiles'; -import { syncAgentFunctions } from './syncAgentFunctions'; -import { syncAgents } from './syncAgents'; -import { syncAttribute } from './syncAttribute'; -import { syncBusinessEntities } from './syncBusinessEntities'; -import { syncConsentManager } from './syncConsentManager'; -import { syncCookies } from './syncCookies'; -import { syncDataCategories } from './syncDataCategories'; -import { syncDataFlows } from './syncDataFlows'; -import { syncDataSiloDependencies, syncDataSilos } from './syncDataSilos'; -import { syncDataSubject } from './syncDataSubject'; -import { syncEnricher } from './syncEnrichers'; -import { syncIdentifier } from './syncIdentifier'; -import { syncIntlMessages } from './syncIntlMessages'; -import { syncPartitions } from './syncPartitions'; -import { syncPolicies } from './syncPolicies'; -import { syncPrivacyCenter } from './syncPrivacyCenter'; -import { syncProcessingPurposes } from './syncProcessingPurposes'; -import { syncPromptGroups } from './syncPromptGroups'; -import { syncPromptPartials } from './syncPromptPartials'; -import { syncPrompts } from './syncPrompts'; -import { syncTeams } from './syncTeams'; -import { syncTemplate } from './syncTemplates'; -import { syncVendors } from './syncVendors'; +} from "./fetchIdentifiers"; +import { syncAction } from "./syncAction"; +import { syncActionItemCollections } from "./syncActionItemCollections"; +import { syncActionItems } from "./syncActionItems"; +import { syncAgentFiles } from "./syncAgentFiles"; +import { syncAgentFunctions } from "./syncAgentFunctions"; +import { syncAgents } from "./syncAgents"; +import { syncAttribute } from "./syncAttribute"; +import { syncBusinessEntities } from "./syncBusinessEntities"; +import { syncConsentManager } from "./syncConsentManager"; +import { syncCookies } from "./syncCookies"; +import { syncDataCategories } from "./syncDataCategories"; +import { syncDataFlows } from "./syncDataFlows"; +import { syncDataSiloDependencies, syncDataSilos } from "./syncDataSilos"; +import { syncDataSubject } from "./syncDataSubject"; +import { syncEnricher } from "./syncEnrichers"; +import { syncIdentifier } from "./syncIdentifier"; +import { syncIntlMessages } from "./syncIntlMessages"; +import { syncPartitions } from "./syncPartitions"; +import { syncPolicies } from "./syncPolicies"; +import { syncPrivacyCenter } from "./syncPrivacyCenter"; +import { syncProcessingPurposes } from "./syncProcessingPurposes"; +import { syncPromptGroups } from "./syncPromptGroups"; +import { syncPromptPartials } from "./syncPromptPartials"; +import { syncPrompts } from "./syncPrompts"; +import { syncTeams } from "./syncTeams"; +import { syncTemplate } from "./syncTemplates"; +import { syncVendors } from "./syncVendors"; const CONCURRENCY = 10; @@ -71,7 +70,7 @@ export async function syncConfigurationToTranscend( deleteExtraAttributeValues?: boolean; /** classify data flow service if missing */ classifyService?: boolean; - }, + } ): Promise { let encounteredError = false; @@ -82,26 +81,26 @@ export async function syncConfigurationToTranscend( attributes, actions, identifiers, - 'data-subjects': dataSubjects, - 'business-entities': businessEntities, + "data-subjects": dataSubjects, + "business-entities": businessEntities, enrichers, cookies, - 'consent-manager': consentManager, - 'data-silos': dataSilos, - 'data-flows': dataFlows, + "consent-manager": consentManager, + "data-silos": dataSilos, + "data-flows": dataFlows, prompts, - 'prompt-groups': promptGroups, - 'prompt-partials': promptPartials, + "prompt-groups": promptGroups, + "prompt-partials": promptPartials, agents, - 'agent-functions': agentFunctions, - 'agent-files': agentFiles, + "agent-functions": agentFunctions, + "agent-files": agentFiles, vendors, - 'data-categories': dataCategories, - 'processing-purposes': processingPurposes, - 'action-items': actionItems, - 'action-item-collections': actionItemCollections, + "data-categories": dataCategories, + "processing-purposes": processingPurposes, + "action-items": actionItems, + "action-item-collections": actionItemCollections, teams, - 'privacy-center': privacyCenter, + "privacy-center": privacyCenter, messages, policies, partitions, @@ -114,9 +113,9 @@ export async function syncConfigurationToTranscend( ? fetchIdentifiersAndCreateMissing( input, client, - !publishToPrivacyCenter, + !publishToPrivacyCenter ) - : ({} as { [k in string]: Identifier }), + : ({} as Record), // Grab all data subjects in the organization dataSilos || dataSubjects || enrichers ? ensureAllDataSubjectsExist(input, client) @@ -124,22 +123,22 @@ export async function syncConfigurationToTranscend( // Grab API keys dataSilos && dataSilos - .map((dataSilo) => dataSilo['api-key-title'] || []) - .reduce((acc, lst) => acc + lst.length, 0) > 0 + .map((dataSilo) => dataSilo["api-key-title"] || []) + .reduce((accumulator, lst) => accumulator + lst.length, 0) > 0 ? fetchApiKeys(input, client) : {}, ]); // Sync consent manager if (consentManager) { - logger.info(colors.magenta('Syncing consent manager...')); + logger.info(colors.magenta("Syncing consent manager...")); try { await syncConsentManager(client, consentManager); - logger.info(colors.green('Successfully synced consent manager!')); - } catch (err) { + logger.info(colors.green("Successfully synced consent manager!")); + } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync consent manager! - ${err.message}`), + colors.red(`Failed to sync consent manager! - ${error.message}`) ); } } @@ -166,7 +165,7 @@ export async function syncConfigurationToTranscend( // Sync email templates if (templates) { logger.info( - colors.magenta(`Syncing "${templates.length}" email templates...`), + colors.magenta(`Syncing "${templates.length}" email templates...`) ); await map( templates, @@ -175,20 +174,20 @@ export async function syncConfigurationToTranscend( try { await syncTemplate(template, client); logger.info( - colors.green(`Successfully synced template "${template.title}"!`), + colors.green(`Successfully synced template "${template.title}"!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync template "${template.title}"! - ${err.message}`, - ), + `Failed to sync template "${template.title}"! - ${error.message}` + ) ); } }, { concurrency: CONCURRENCY, - }, + } ); logger.info(colors.green(`Synced "${templates.length}" email templates!`)); } @@ -197,7 +196,7 @@ export async function syncConfigurationToTranscend( if (businessEntities) { const businessEntitySuccess = await syncBusinessEntities( client, - businessEntities, + businessEntities ); encounteredError = encounteredError || !businessEntitySuccess; } @@ -212,7 +211,7 @@ export async function syncConfigurationToTranscend( if (dataCategories) { const dataCategoriesSuccess = await syncDataCategories( client, - dataCategories, + dataCategories ); encounteredError = encounteredError || !dataCategoriesSuccess; } @@ -221,7 +220,7 @@ export async function syncConfigurationToTranscend( if (processingPurposes) { const processingPurposesSuccess = await syncProcessingPurposes( client, - processingPurposes, + processingPurposes ); encounteredError = encounteredError || !processingPurposesSuccess; } @@ -242,7 +241,7 @@ export async function syncConfigurationToTranscend( if (agentFunctions) { const agentFunctionsSuccess = await syncAgentFunctions( client, - agentFunctions, + agentFunctions ); encounteredError = encounteredError || !agentFunctionsSuccess; } @@ -263,7 +262,7 @@ export async function syncConfigurationToTranscend( if (actionItemCollections) { const actionItemCollectionsSuccess = await syncActionItemCollections( client, - actionItemCollections, + actionItemCollections ); encounteredError = encounteredError || !actionItemCollectionsSuccess; } @@ -277,7 +276,7 @@ export async function syncConfigurationToTranscend( attributes, async (attribute) => { const existing = existingAttributes.find( - (attr) => attr.name === attribute.name, + (attribute_) => attribute_.name === attribute.name ); logger.info(colors.magenta(`Syncing attribute "${attribute.name}"...`)); @@ -287,20 +286,20 @@ export async function syncConfigurationToTranscend( deleteExtraAttributeValues, }); logger.info( - colors.green(`Successfully synced attribute "${attribute.name}"!`), + colors.green(`Successfully synced attribute "${attribute.name}"!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync attribute "${attribute.name}"! - ${err.message}`, - ), + `Failed to sync attribute "${attribute.name}"! - ${error.message}` + ) ); } }, { concurrency: CONCURRENCY, - }, + } ); logger.info(colors.green(`Synced "${attributes.length}" attributes!`)); } @@ -325,20 +324,20 @@ export async function syncConfigurationToTranscend( dataSubjectsByName, }); logger.info( - colors.green(`Successfully synced enricher "${enricher.title}"!`), + colors.green(`Successfully synced enricher "${enricher.title}"!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync enricher "${enricher.title}"! - ${err.message}`, - ), + `Failed to sync enricher "${enricher.title}"! - ${error.message}` + ) ); } }, { concurrency: CONCURRENCY, - }, + } ); logger.info(colors.green(`Synced "${enrichers.length}" enrichers!`)); } @@ -347,7 +346,7 @@ export async function syncConfigurationToTranscend( if (identifiers) { // Fetch existing logger.info( - colors.magenta(`Syncing "${identifiers.length}" identifiers...`), + colors.magenta(`Syncing "${identifiers.length}" identifiers...`) ); await map( identifiers, @@ -355,12 +354,12 @@ export async function syncConfigurationToTranscend( const existing = identifierByName[identifier.name]; if (!existing) { throw new Error( - `Failed to find identifier with name: ${identifier.type}. Should have been auto-created by cli.`, + `Failed to find identifier with name: ${identifier.type}. Should have been auto-created by cli.` ); } logger.info( - colors.magenta(`Syncing identifier "${identifier.type}"...`), + colors.magenta(`Syncing identifier "${identifier.type}"...`) ); try { await syncIdentifier(client, { @@ -370,22 +369,20 @@ export async function syncConfigurationToTranscend( skipPublish: !publishToPrivacyCenter, }); logger.info( - colors.green( - `Successfully synced identifier "${identifier.type}"!`, - ), + colors.green(`Successfully synced identifier "${identifier.type}"!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync identifier "${identifier.type}"! - ${err.message}`, - ), + `Failed to sync identifier "${identifier.type}"! - ${error.message}` + ) ); } }, { concurrency: CONCURRENCY, - }, + } ); logger.info(colors.green(`Synced "${identifiers.length}" identifiers!`)); } @@ -399,11 +396,11 @@ export async function syncConfigurationToTranscend( actions, async (action) => { const existing = existingActions.find( - (act) => act.type === action.type, + (act) => act.type === action.type ); if (!existing) { throw new Error( - `Failed to find action with type: ${action.type}. Should have already existing in the organization.`, + `Failed to find action with type: ${action.type}. Should have already existing in the organization.` ); } @@ -415,20 +412,20 @@ export async function syncConfigurationToTranscend( skipPublish: !publishToPrivacyCenter, }); logger.info( - colors.green(`Successfully synced action "${action.type}"!`), + colors.green(`Successfully synced action "${action.type}"!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync action "${action.type}"! - ${err.message}`, - ), + `Failed to sync action "${action.type}"! - ${error.message}` + ) ); } }, { concurrency: CONCURRENCY, - }, + } ); logger.info(colors.green(`Synced "${actions.length}" actions!`)); } @@ -437,23 +434,23 @@ export async function syncConfigurationToTranscend( if (dataSubjects) { // Fetch existing logger.info( - colors.magenta(`Syncing "${dataSubjects.length}" data subjects...`), + colors.magenta(`Syncing "${dataSubjects.length}" data subjects...`) ); const existingDataSubjects = await fetchAllDataSubjects(client); await map( dataSubjects, async (dataSubject) => { const existing = existingDataSubjects.find( - (subj) => subj.type === dataSubject.type, + (subj) => subj.type === dataSubject.type ); if (!existing) { throw new Error( - `Failed to find data subject with type: ${dataSubject.type}. Should have already existing in the organization.`, + `Failed to find data subject with type: ${dataSubject.type}. Should have already existing in the organization.` ); } logger.info( - colors.magenta(`Syncing data subject "${dataSubject.type}"...`), + colors.magenta(`Syncing data subject "${dataSubject.type}"...`) ); try { await syncDataSubject(client, { @@ -463,21 +460,21 @@ export async function syncConfigurationToTranscend( }); logger.info( colors.green( - `Successfully synced data subject "${dataSubject.type}"!`, - ), + `Successfully synced data subject "${dataSubject.type}"!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync data subject "${dataSubject.type}"! - ${err.message}`, - ), + `Failed to sync data subject "${dataSubject.type}"! - ${error.message}` + ) ); } }, { concurrency: CONCURRENCY, - }, + } ); logger.info(colors.green(`Synced "${dataSubjects.length}" data subjects!`)); } @@ -487,7 +484,7 @@ export async function syncConfigurationToTranscend( const syncedDataFlows = await syncDataFlows( client, dataFlows, - classifyService, + classifyService ); encounteredError = encounteredError || !syncedDataFlows; } @@ -521,17 +518,18 @@ export async function syncConfigurationToTranscend( dataSubjectsByName, apiKeysByTitle: apiKeyTitleMap, pageSize, - }, + } ); - dataSilos?.forEach((dataSilo) => { - // Queue up dependency update - if (dataSilo['deletion-dependencies']) { - dependencyUpdates.push([ - dataSiloTitleToId[dataSilo.title], - dataSilo['deletion-dependencies'], - ]); + if (dataSilos) + for (const dataSilo of dataSilos) { + // Queue up dependency update + if (dataSilo["deletion-dependencies"]) { + dependencyUpdates.push([ + dataSiloTitleToId[dataSilo.title], + dataSilo["deletion-dependencies"], + ]); + } } - }); encounteredError = encounteredError || !success; } @@ -546,4 +544,3 @@ export async function syncConfigurationToTranscend( return encounteredError; } -/* eslint-enable max-lines */ diff --git a/src/lib/graphql/syncConsentManager.ts b/src/lib/graphql/syncConsentManager.ts index 818451ec..2f45a885 100644 --- a/src/lib/graphql/syncConsentManager.ts +++ b/src/lib/graphql/syncConsentManager.ts @@ -1,22 +1,22 @@ import { InitialViewState, OnConsentExpiry, -} from '@transcend-io/airgap.js-types'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy } from 'lodash-es'; +} from "@transcend-io/airgap.js-types"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { keyBy } from "lodash-es"; import { ConsentManageExperienceInput, ConsentManagerInput, -} from '../../codecs'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; -import { fetchAllPurposes } from './fetchAllPurposes'; +} from "../../codecs"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; +import { fetchAllPurposes } from "./fetchAllPurposes"; import { fetchConsentManagerExperiences, fetchConsentManagerId, -} from './fetchConsentManagerId'; -import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; +} from "./fetchConsentManagerId"; +import { fetchPrivacyCenterId } from "./fetchPrivacyCenterId"; import { CREATE_CONSENT_EXPERIENCE, CREATE_CONSENT_MANAGER, @@ -30,12 +30,12 @@ import { UPDATE_CONSENT_MANAGER_THEME, UPDATE_CONSENT_MANAGER_VERSION, UPDATE_TOGGLE_USP_API, -} from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; -import { fetchPartitions } from './syncPartitions'; +} from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { fetchPartitions } from "./syncPartitions"; const PURPOSES_LINK = - 'https://app.transcend.io/consent-manager/regional-experiences/purposes'; + "https://app.transcend.io/consent-manager/regional-experiences/purposes"; /** * Sync consent manager experiences up to Transcend @@ -45,15 +45,15 @@ const PURPOSES_LINK = */ export async function syncConsentManagerExperiences( client: GraphQLClient, - experiences: ConsentManageExperienceInput[], + experiences: ConsentManageExperienceInput[] ): Promise { // Fetch existing experiences and const existingExperiences = await fetchConsentManagerExperiences(client); - const experienceLookup = keyBy(existingExperiences, 'name'); + const experienceLookup = keyBy(existingExperiences, "name"); // Fetch existing purposes const purposes = await fetchAllPurposes(client); - const purposeLookup = keyBy(purposes, 'trackingType'); + const purposeLookup = keyBy(purposes, "trackingType"); // Bulk update or create experiences await map( @@ -65,7 +65,7 @@ export async function syncConsentManagerExperiences( if (!existingPurpose) { throw new Error( `Invalid purpose trackingType provided at consentManager.experiences[${ind}].purposes[${ind2}]: ` + - `${purpose.trackingType}. See list of valid purposes ${PURPOSES_LINK}`, + `${purpose.trackingType}. See list of valid purposes ${PURPOSES_LINK}` ); } return existingPurpose.id; @@ -75,7 +75,7 @@ export async function syncConsentManagerExperiences( if (!existingPurpose) { throw new Error( `Invalid purpose trackingType provided at consentManager.experiences[${ind}].optedOutPurposes[${ind2}]: ` + - `${purpose.trackingType}. See list of valid purposes ${PURPOSES_LINK}`, + `${purpose.trackingType}. See list of valid purposes ${PURPOSES_LINK}` ); } return existingPurpose.id; @@ -93,9 +93,9 @@ export async function syncConsentManagerExperiences( onConsentExpiry: exp.onConsentExpiry, consentExpiry: exp.consentExpiry, displayPriority: - exp.displayPriority !== existingExperience.displayPriority - ? exp.displayPriority - : undefined, + exp.displayPriority === existingExperience.displayPriority + ? undefined + : exp.displayPriority, viewState: exp.viewState, purposes: purposeIds, optedOutPurposes: optedOutPurposeIds, @@ -104,7 +104,7 @@ export async function syncConsentManagerExperiences( }, }); logger.info( - colors.green(`Successfully synced consent experience "${exp.name}"!`), + colors.green(`Successfully synced consent experience "${exp.name}"!`) ); } else { // create new experience @@ -125,15 +125,13 @@ export async function syncConsentManagerExperiences( }, }); logger.info( - colors.green( - `Successfully created consent experience "${exp.name}"!`, - ), + colors.green(`Successfully created consent experience "${exp.name}"!`) ); } }, { concurrency: 10, - }, + } ); } @@ -145,16 +143,16 @@ export async function syncConsentManagerExperiences( */ export async function syncConsentManager( client: GraphQLClient, - consentManager: ConsentManagerInput, + consentManager: ConsentManagerInput ): Promise { let airgapBundleId: string; // ensure the consent manager is created and deployed try { airgapBundleId = await fetchConsentManagerId(client, 1); - } catch (err) { + } catch (error) { // TODO: https://transcend.height.app/T-23778 - if (err.message.includes('AirgapBundle not found')) { + if (error.message.includes("AirgapBundle not found")) { const privacyCenterId = await fetchPrivacyCenterId(client); const { createConsentManager } = await makeGraphQLRequest<{ @@ -172,7 +170,7 @@ export async function syncConsentManager( }); airgapBundleId = createConsentManager.consentManager.id; } else { - throw err; + throw error; } } @@ -188,11 +186,11 @@ export async function syncConsentManager( if (consentManager.partition) { const partitions = await fetchPartitions(client); const partitionToUpdate = partitions.find( - (part) => part.name === consentManager.partition, + (part) => part.name === consentManager.partition ); if (!partitionToUpdate) { throw new Error( - `Partition "${consentManager.partition}" not found. Please create the partition first.`, + `Partition "${consentManager.partition}" not found. Please create the partition first.` ); } await makeGraphQLRequest(client, UPDATE_CONSENT_MANAGER_PARTITION, { diff --git a/src/lib/graphql/syncCookies.ts b/src/lib/graphql/syncCookies.ts index 0faa8ace..2c0dd7a4 100644 --- a/src/lib/graphql/syncCookies.ts +++ b/src/lib/graphql/syncCookies.ts @@ -1,13 +1,13 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { chunk } from 'lodash-es'; -import { CookieInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { fetchConsentManagerId } from './fetchConsentManagerId'; -import { UPDATE_OR_CREATE_COOKIES } from './gqls'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { chunk } from "lodash-es"; +import { CookieInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; +import { fetchConsentManagerId } from "./fetchConsentManagerId"; +import { UPDATE_OR_CREATE_COOKIES } from "./gqls"; // import { keyBy } from 'lodash-es'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; const MAX_PAGE_SIZE = 100; @@ -19,7 +19,7 @@ const MAX_PAGE_SIZE = 100; */ export async function updateOrCreateCookies( client: GraphQLClient, - cookieInputs: CookieInput[], + cookieInputs: CookieInput[] ): Promise { const airgapBundleId = await fetchConsentManagerId(client); @@ -64,7 +64,7 @@ export async function updateOrCreateCookies( */ export async function syncCookies( client: GraphQLClient, - cookies: CookieInput[], + cookies: CookieInput[] ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${cookies.length}" cookies...`)); @@ -73,14 +73,14 @@ export async function syncCookies( const notUnique = cookies.filter( (cookie) => cookies.filter( - (cook) => cookie.name === cook.name && cookie.isRegex === cook.isRegex, - ).length > 1, + (cook) => cookie.name === cook.name && cookie.isRegex === cook.isRegex + ).length > 1 ); if (notUnique.length > 0) { throw new Error( `Failed to upload cookies as there were non-unique entries found: ${notUnique .map(({ name }) => name) - .join(',')}`, + .join(",")}` ); } @@ -88,9 +88,9 @@ export async function syncCookies( logger.info(colors.magenta(`Upserting "${cookies.length}" new cookies...`)); await updateOrCreateCookies(client, cookies); logger.info(colors.green(`Successfully synced ${cookies.length} cookies!`)); - } catch (err) { + } catch (error) { encounteredError = true; - logger.info(colors.red(`Failed to create cookies! - ${err.message}`)); + logger.info(colors.red(`Failed to create cookies! - ${error.message}`)); } return !encounteredError; diff --git a/src/lib/graphql/syncDataCategories.ts b/src/lib/graphql/syncDataCategories.ts index c00912d2..0b37f92c 100644 --- a/src/lib/graphql/syncDataCategories.ts +++ b/src/lib/graphql/syncDataCategories.ts @@ -1,15 +1,15 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy } from 'lodash-es'; -import { DataCategoryInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { keyBy } from "lodash-es"; +import { DataCategoryInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; import { DataSubCategory, fetchAllDataCategories, -} from './fetchAllDataCategories'; -import { CREATE_DATA_SUB_CATEGORY, UPDATE_DATA_SUB_CATEGORIES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "./fetchAllDataCategories"; +import { CREATE_DATA_SUB_CATEGORY, UPDATE_DATA_SUB_CATEGORIES } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Input to create a new data category @@ -20,8 +20,8 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; */ export async function createDataCategory( client: GraphQLClient, - dataCategory: DataCategoryInput, -): Promise> { + dataCategory: DataCategoryInput +): Promise> { const input = { name: dataCategory.name, category: dataCategory.category, @@ -49,7 +49,7 @@ export async function createDataCategory( */ export async function updateDataCategories( client: GraphQLClient, - dataCategoryIdPairs: [DataCategoryInput, string][], + dataCategoryIdPairs: [DataCategoryInput, string][] ): Promise { await makeGraphQLRequest(client, UPDATE_DATA_SUB_CATEGORIES, { input: { @@ -72,7 +72,7 @@ export async function updateDataCategories( */ export async function syncDataCategories( client: GraphQLClient, - inputs: DataCategoryInput[], + inputs: DataCategoryInput[] ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" data categories...`)); @@ -83,16 +83,17 @@ export async function syncDataCategories( const existingDataCategories = await fetchAllDataCategories(client); // Look up by name - const dataCategoryByName: { - [k in string]: Pick; - } = keyBy( + const dataCategoryByName: Record< + string, + Pick + > = keyBy( existingDataCategories, - ({ name, category }) => `${name}:${category}`, + ({ name, category }) => `${name}:${category}` ); // Create new data categories const newDataCategories = inputs.filter( - (input) => !dataCategoryByName[`${input.name}:${input.category}`], + (input) => !dataCategoryByName[`${input.name}:${input.category}`] ); // Create new data categories @@ -104,15 +105,15 @@ export async function syncDataCategories( ] = newDataCategory; logger.info( colors.green( - `Successfully synced data category "${dataCategory.name}"!`, - ), + `Successfully synced data category "${dataCategory.name}"!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync data category "${dataCategory.name}"! - ${err.message}`, - ), + `Failed to sync data category "${dataCategory.name}"! - ${error.message}` + ) ); } }); @@ -125,17 +126,17 @@ export async function syncDataCategories( inputs.map((input) => [ input, dataCategoryByName[`${input.name}:${input.category}`].id, - ]), + ]) ); logger.info( - colors.green(`Successfully synced "${inputs.length}" data categories!`), + colors.green(`Successfully synced "${inputs.length}" data categories!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" data categories ! - ${err.message}`, - ), + `Failed to sync "${inputs.length}" data categories ! - ${error.message}` + ) ); } diff --git a/src/lib/graphql/syncDataFlows.ts b/src/lib/graphql/syncDataFlows.ts index 0c119837..b781f6a7 100644 --- a/src/lib/graphql/syncDataFlows.ts +++ b/src/lib/graphql/syncDataFlows.ts @@ -1,14 +1,14 @@ -import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { chunk } from 'lodash-es'; -import { DataFlowInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { fetchAllDataFlows } from './fetchAllDataFlows'; -import { fetchConsentManagerId } from './fetchConsentManagerId'; -import { CREATE_DATA_FLOWS, UPDATE_DATA_FLOWS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { ConsentTrackerStatus } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { chunk } from "lodash-es"; +import { DataFlowInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; +import { fetchAllDataFlows } from "./fetchAllDataFlows"; +import { fetchConsentManagerId } from "./fetchConsentManagerId"; +import { CREATE_DATA_FLOWS, UPDATE_DATA_FLOWS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; const MAX_PAGE_SIZE = 100; @@ -22,7 +22,7 @@ const MAX_PAGE_SIZE = 100; export async function updateDataFlows( client: GraphQLClient, dataFlowInputs: [DataFlowInput, string][], - classifyService = false, + classifyService = false ): Promise { const airgapBundleId = await fetchConsentManagerId(client); @@ -70,7 +70,7 @@ export async function updateDataFlows( export async function createDataFlows( client: GraphQLClient, dataFlowInputs: DataFlowInput[], - classifyService = false, + classifyService = false ): Promise { const airgapBundleId = await fetchConsentManagerId(client); // TODO: https://transcend.height.app/T-19841 - add with custom purposes @@ -117,7 +117,7 @@ export async function createDataFlows( export async function syncDataFlows( client: GraphQLClient, dataFlows: DataFlowInput[], - classifyService: boolean, + classifyService: boolean ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${dataFlows.length}" data flows...`)); @@ -127,8 +127,8 @@ export async function syncDataFlows( const notUnique = dataFlows.filter( (dataFlow) => dataFlows.filter( - (flow) => dataFlow.value === flow.value && dataFlow.type === flow.type, - ).length > 1, + (flow) => dataFlow.value === flow.value && dataFlow.type === flow.type + ).length > 1 ); // Throw error to prompt user to de-dupe before uploading @@ -136,13 +136,13 @@ export async function syncDataFlows( throw new Error( `Failed to upload data flows as there were non-unique entries found: ${notUnique .map(({ value }) => value) - .join(',')}`, + .join(",")}` ); } // Fetch existing data flows to determine whether we are creating a new data flow // or updating an existing data flow - logger.info(colors.magenta('Fetching data flows...')); + logger.info(colors.magenta("Fetching data flows...")); const [existingLiveDataFlows, existingInReviewDataFlows] = await Promise.all([ fetchAllDataFlows(client, ConsentTrackerStatus.Live), fetchAllDataFlows(client, ConsentTrackerStatus.NeedsReview), @@ -153,7 +153,7 @@ export async function syncDataFlows( const mapDataFlowsToExisting = dataFlows.map((dataFlow) => [ dataFlow, allDataFlows.find( - (flow) => dataFlow.value === flow.value && dataFlow.type === flow.type, + (flow) => dataFlow.value === flow.value && dataFlow.type === flow.type )?.id, ]); @@ -163,34 +163,34 @@ export async function syncDataFlows( .map(([flow]) => flow as DataFlowInput); try { logger.info( - colors.magenta(`Creating "${newDataFlows.length}" new data flows...`), + colors.magenta(`Creating "${newDataFlows.length}" new data flows...`) ); await createDataFlows(client, newDataFlows, classifyService); logger.info( - colors.green(`Successfully synced ${newDataFlows.length} data flows!`), + colors.green(`Successfully synced ${newDataFlows.length} data flows!`) ); - } catch (err) { + } catch (error) { encounteredError = true; - logger.info(colors.red(`Failed to create data flows! - ${err.message}`)); + logger.info(colors.red(`Failed to create data flows! - ${error.message}`)); } // Update existing data flows const existingDataFlows = mapDataFlowsToExisting.filter( - (x): x is [DataFlowInput, string] => !!x[1], + (x): x is [DataFlowInput, string] => !!x[1] ); try { logger.info( - colors.magenta(`Updating "${existingDataFlows.length}" data flows...`), + colors.magenta(`Updating "${existingDataFlows.length}" data flows...`) ); await updateDataFlows(client, existingDataFlows, classifyService); logger.info( colors.green( - `Successfully updated "${existingDataFlows.length}" data flows!`, - ), + `Successfully updated "${existingDataFlows.length}" data flows!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; - logger.info(colors.red(`Failed to create data flows! - ${err.message}`)); + logger.info(colors.red(`Failed to create data flows! - ${error.message}`)); } logger.info(colors.green(`Synced "${dataFlows.length}" data flows!`)); diff --git a/src/lib/graphql/syncDataSilos.ts b/src/lib/graphql/syncDataSilos.ts index 3e38c3fc..2c47f458 100644 --- a/src/lib/graphql/syncDataSilos.ts +++ b/src/lib/graphql/syncDataSilos.ts @@ -1,4 +1,3 @@ -/* eslint-disable max-lines */ import { ConfidenceLabel, IsoCountryCode, @@ -7,24 +6,24 @@ import { PromptAVendorEmailSendType, RequestActionObjectResolver, SubDataPointDataSubCategoryGuessStatus, -} from '@transcend-io/privacy-types'; -import { apply } from '@transcend-io/type-utils'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { chunk, keyBy, sortBy } from 'lodash-es'; +} from "@transcend-io/privacy-types"; +import { apply } from "@transcend-io/type-utils"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { chunk, keyBy, sortBy } from "lodash-es"; import { DataCategoryInput, DataSiloInput, ProcessingPurposeInput, -} from '../../codecs'; -import { logger } from '../../logger'; -import { map, mapSeries } from '../bluebird-replace'; -import { ApiKey } from './fetchApiKeys'; +} from "../../codecs"; +import { logger } from "../../logger"; +import { map, mapSeries } from "../bluebird-replace"; +import { ApiKey } from "./fetchApiKeys"; import { convertToDataSubjectBlockList, DataSubject, -} from './fetchDataSubjects'; +} from "./fetchDataSubjects"; import { CREATE_DATA_SILOS, DATA_POINTS, @@ -34,8 +33,8 @@ import { SUB_DATA_POINTS_WITH_GUESSES, UPDATE_DATA_SILOS, UPDATE_OR_CREATE_DATA_POINT, -} from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; export interface DataSiloAttributeValue { /** Key associated to value */ @@ -94,12 +93,12 @@ export async function fetchAllDataSilos( integrationNames?: string[]; /** GQL query for data silos */ gql?: string; - }, + } ): Promise { logger.info( colors.magenta( - `Fetching ${ids.length === 0 ? 'all' : ids.length} Data Silos...`, - ), + `Fetching ${ids.length === 0 ? "all" : ids.length} Data Silos...` + ) ); const dataSilos: TDataSilo[] = []; @@ -132,13 +131,13 @@ export async function fetchAllDataSilos( logger.info( colors.green( `Found a total of ${dataSilos.length} data silo${ - ids.length > 0 ? ` matching IDs ${ids.join(',')}` : '' + ids.length > 0 ? ` matching IDs ${ids.join(",")}` : "" }s${ integrationNames.length > 0 - ? ` matching integrationNames ${integrationNames.join(',')}` - : '' - }`, - ), + ? ` matching integrationNames ${integrationNames.join(",")}` + : "" + }` + ) ); return dataSilos.sort((a, b) => a.title.localeCompare(b.title)); @@ -267,7 +266,7 @@ export async function fetchAllSubDataPoints( pageSize: number; /** When true, metadata around guessed data categories should be included */ includeGuessedCategories?: boolean; - }, + } ): Promise { const subDataPoints: SubDataPoint[] = []; @@ -278,7 +277,7 @@ export async function fetchAllSubDataPoints( try { if (debug) { logger.log( - colors.magenta(`Pulling in subdatapoints for offset ${offset}`), + colors.magenta(`Pulling in subdatapoints for offset ${offset}`) ); } const { @@ -300,7 +299,7 @@ export async function fetchAllSubDataPoints( dataPoints: [dataPointId], }, offset, - }, + } ); subDataPoints.push(...nodes); @@ -310,20 +309,20 @@ export async function fetchAllSubDataPoints( if (debug) { logger.log( colors.green( - `Pulled in subdatapoints for offset ${offset} for dataPointId=${dataPointId}`, - ), + `Pulled in subdatapoints for offset ${offset} for dataPointId=${dataPointId}` + ) ); } - } catch (err) { + } catch (error) { logger.error( colors.red( - `An error fetching subdatapoints for offset ${offset} for dataPointId=${dataPointId}`, - ), + `An error fetching subdatapoints for offset ${offset} for dataPointId=${dataPointId}` + ) ); - throw err; + throw error; } } while (shouldContinue); - return sortBy(subDataPoints, 'name'); + return sortBy(subDataPoints, "name"); } /** @@ -351,7 +350,7 @@ export async function fetchAllDataPoints( skipSubDatapoints?: boolean; /** When true, metadata around guessed data categories should be included */ includeGuessedCategories?: boolean; - }, + } ): Promise { const dataPoints: DataPointWithSubDataPoint[] = []; @@ -384,22 +383,22 @@ export async function fetchAllDataPoints( if (debug) { logger.info( colors.magenta( - `Fetched ${nodes.length} datapoints at offset: ${offset}`, - ), + `Fetched ${nodes.length} datapoints at offset: ${offset}` + ) ); } if (!skipSubDatapoints) { await map( nodes, - /* eslint-disable no-loop-func */ + async (node) => { try { if (debug) { logger.info( colors.magenta( - `Fetching subdatapoints for ${node.name} for datapoint offset ${offset}`, - ), + `Fetching subdatapoints for ${node.name} for datapoint offset ${offset}` + ) ); } @@ -411,37 +410,37 @@ export async function fetchAllDataPoints( dataPoints.push({ ...node, subDataPoints: subDataPoints.sort((a, b) => - a.name.localeCompare(b.name), + a.name.localeCompare(b.name) ), }); if (debug) { logger.info( colors.green( - `Successfully fetched subdatapoints for ${node.name}`, - ), + `Successfully fetched subdatapoints for ${node.name}` + ) ); } - } catch (err) { + } catch (error) { logger.error( colors.red( - `An error fetching subdatapoints for ${node.name} datapoint offset ${offset}`, - ), + `An error fetching subdatapoints for ${node.name} datapoint offset ${offset}` + ) ); - throw err; + throw error; } }, - /* eslint-enable no-loop-func */ + { concurrency: 5, - }, + } ); if (debug) { logger.info( colors.green( - `Fetched all subdatapoints for page of datapoints at offset: ${offset}`, - ), + `Fetched all subdatapoints for page of datapoints at offset: ${offset}` + ) ); } } @@ -588,7 +587,7 @@ export async function fetchEnrichedDataSilos( skipSubDatapoints?: boolean; /** When true, metadata around guessed data categories should be included */ includeGuessedCategories?: boolean; - }, + } ): Promise<[DataSiloEnriched, DataPointWithSubDataPoint[]][]> { const dataSilos: [DataSiloEnriched, DataPointWithSubDataPoint[]][] = []; @@ -606,8 +605,8 @@ export async function fetchEnrichedDataSilos( await mapSeries(silos, async (silo, index) => { logger.info( colors.magenta( - `[${index + 1}/${silos.length}] Fetching data silo - ${silo.title}`, - ), + `[${index + 1}/${silos.length}] Fetching data silo - ${silo.title}` + ) ); const dataPoints = await fetchAllDataPoints(client, silo.id, { @@ -622,8 +621,8 @@ export async function fetchEnrichedDataSilos( colors.green( `[${index + 1}/${ silos.length - }] Successfully fetched datapoint for - ${silo.title}`, - ), + }] Successfully fetched datapoint for - ${silo.title}` + ) ); } @@ -633,8 +632,8 @@ export async function fetchEnrichedDataSilos( logger.info( colors.green( - `Successfully fetched all ${silos.length} data silo configurations`, - ), + `Successfully fetched all ${silos.length} data silo configurations` + ) ); return dataSilos; @@ -659,20 +658,20 @@ export async function syncDataSilos( /** Page size */ pageSize: number; /** The data subjects in the organization */ - dataSubjectsByName: { [type in string]: DataSubject }; + dataSubjectsByName: Record; /** API key title to API key */ - apiKeysByTitle: { [title in string]: ApiKey }; - }, + apiKeysByTitle: Record; + } ): Promise<{ /** Whether successfully updated */ success: boolean; /** A mapping between data silo title to data silo ID */ - dataSiloTitleToId: { [k in string]: string }; + dataSiloTitleToId: Record; }> { let encounteredError = false; // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); logger.info(colors.magenta(`Syncing "${dataSilos.length}" data silos...`)); // Determine the set of data silos that already exist @@ -682,20 +681,20 @@ export async function syncDataSilos( }); // Create a mapping of title -> existing silo, if it exists - const existingDataSiloByTitle = keyBy>( + const existingDataSiloByTitle = keyBy>( existingDataSilos, - 'title', + "title" ); // Create new silos that do not exist const newDataSiloInputs = dataSilos.filter( - ({ title }) => !existingDataSiloByTitle[title], + ({ title }) => !existingDataSiloByTitle[title] ); if (newDataSiloInputs.length > 0) { logger.info( colors.magenta( - `Creating "${newDataSiloInputs.length}" data silos that did not exist...`, - ), + `Creating "${newDataSiloInputs.length}" data silos that did not exist...` + ) ); // Batch the creation @@ -707,11 +706,11 @@ export async function syncDataSilos( /** Mutation result */ createDataSilos: { /** New data silos */ - dataSilos: Pick[]; + dataSilos: Pick[]; }; }>(client, CREATE_DATA_SILOS, { input: dependencyUpdateChunk.map((input) => ({ - name: input['outer-type'] || input.integrationName, + name: input["outer-type"] || input.integrationName, title: input.title, country: input.country, countrySubDivision: input.countrySubDivision, @@ -719,15 +718,15 @@ export async function syncDataSilos( }); // save mapping of title and id - dataSilos.forEach((silo) => { + for (const silo of dataSilos) { existingDataSiloByTitle[silo.title] = silo; - }); + } }); logger.info( colors.green( - `Successfully created "${newDataSiloInputs.length}" data silos!`, - ), + `Successfully created "${newDataSiloInputs.length}" data silos!` + ) ); } @@ -738,14 +737,14 @@ export async function syncDataSilos( colors.magenta( `[Batch ${ind + 1}/${chunkedUpdates.length}] Syncing "${ dataSiloUpdateChunk.length - }" data silos`, - ), + }" data silos` + ) ); await makeGraphQLRequest<{ /** Mutation result */ updateDataSilos: { /** New data silos */ - dataSilos: Pick[]; + dataSilos: Pick[]; }; }>(client, UPDATE_DATA_SILOS, { input: { @@ -756,37 +755,37 @@ export async function syncDataSilos( url: input.url, headers: input.headers, description: input.description, - identifiers: input['identity-keys'], + identifiers: input["identity-keys"], isLive: !input.disabled, ownerEmails: input.owners, teamNames: input.teams, // clear out if not specified, otherwise the update needs to be applied after // all data silos are created - dependedOnDataSiloTitles: input['deletion-dependencies'] + dependedOnDataSiloTitles: input["deletion-dependencies"] ? undefined : [], - apiKeyId: input['api-key-title'] - ? apiKeysByTitle[input['api-key-title']].id + apiKeyId: input["api-key-title"] + ? apiKeysByTitle[input["api-key-title"]].id : undefined, - dataSubjectBlockListIds: input['data-subjects'] + dataSubjectBlockListIds: input["data-subjects"] ? convertToDataSubjectBlockList( - input['data-subjects'], - dataSubjectsByName, + input["data-subjects"], + dataSubjectsByName ) : undefined, attributes: input.attributes, businessEntityTitles: input.businessEntityTitles, // AVC settings - notifyEmailAddress: input['email-settings']?.['notify-email-address'], + notifyEmailAddress: input["email-settings"]?.["notify-email-address"], promptAVendorEmailSendFrequency: - input['email-settings']?.['send-frequency'], - promptAVendorEmailSendType: input['email-settings']?.['send-type'], + input["email-settings"]?.["send-frequency"], + promptAVendorEmailSendType: input["email-settings"]?.["send-type"], promptAVendorEmailIncludeIdentifiersAttachment: - input['email-settings']?.['include-identifiers-attachment'], + input["email-settings"]?.["include-identifiers-attachment"], promptAVendorEmailCompletionLinkType: - input['email-settings']?.['completion-link-type'], + input["email-settings"]?.["completion-link-type"], manualWorkRetryFrequency: - input['email-settings']?.['manual-work-retry-frequency'], + input["email-settings"]?.["manual-work-retry-frequency"], })), }, }); @@ -794,8 +793,8 @@ export async function syncDataSilos( colors.green( `[Batch ${ind + 1}/${chunkedUpdates.length}] Synced "${ dataSiloUpdateChunk.length - }" data silos!`, - ), + }" data silos!` + ) ); }); @@ -804,18 +803,18 @@ export async function syncDataSilos( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); const dataSilosWithDataPoints = dataSilos.filter( - ({ datapoints = [] }) => datapoints.length > 0, + ({ datapoints = [] }) => datapoints.length > 0 ); const totalDataPoints = dataSilos .map(({ datapoints = [] }) => datapoints.length) - .reduce((acc, count) => acc + count, 0); + .reduce((accumulator, count) => accumulator + count, 0); logger.info( colors.magenta( - `Syncing "${totalDataPoints}" datapoints from "${dataSilosWithDataPoints.length}" data silos...`, - ), + `Syncing "${totalDataPoints}" datapoints from "${dataSilosWithDataPoints.length}" data silos...` + ) ); progressBar.start(totalDataPoints, 0); let total = 0; @@ -839,24 +838,24 @@ export async function syncDataSilos( ({ name: key, description, - categories: !categories - ? undefined - : categories.map((category) => ({ + categories: categories + ? categories.map((category) => ({ ...category, - name: category.name || 'Other', - })), - purposes: !purposes - ? undefined - : purposes.map((purpose) => ({ + name: category.name || "Other", + })) + : undefined, + purposes: purposes + ? purposes.map((purpose) => ({ ...purpose, - name: purpose.name || 'Other', - })), + name: purpose.name || "Other", + })) + : undefined, attributes, accessRequestVisibilityEnabled: - rest['access-request-visibility-enabled'], + rest["access-request-visibility-enabled"], erasureRequestRedactionEnabled: - rest['erasure-request-redaction-enabled'], - }), + rest["erasure-request-redaction-enabled"], + }) ) : undefined; @@ -876,27 +875,27 @@ export async function syncDataSilos( teamNames: datapoint.teams, } : {}), - ...(datapoint['data-collection-tag'] - ? { dataCollectionTag: datapoint['data-collection-tag'] } + ...(datapoint["data-collection-tag"] + ? { dataCollectionTag: datapoint["data-collection-tag"] } : {}), - querySuggestions: !datapoint['privacy-action-queries'] - ? undefined - : Object.entries(datapoint['privacy-action-queries']).map( + querySuggestions: datapoint["privacy-action-queries"] + ? Object.entries(datapoint["privacy-action-queries"]).map( ([key, value]) => ({ requestType: key, suggestedQuery: value, - }), - ), - enabledActions: datapoint['privacy-actions'] || [], // clear out when not specified + }) + ) + : undefined, + enabledActions: datapoint["privacy-actions"] || [], // clear out when not specified subDataPoints: fields, }; // Ensure no duplicate sub-datapoints are provided const subDataPointsToUpdate = (payload.subDataPoints || []).map( - ({ name }) => name, + ({ name }) => name ); const duplicateDataPoints = subDataPointsToUpdate.filter( - (name, index) => subDataPointsToUpdate.indexOf(name) !== index, + (name, index) => subDataPointsToUpdate.indexOf(name) !== index ); if (duplicateDataPoints.length > 0) { logger.info( @@ -904,9 +903,9 @@ export async function syncDataSilos( `\nCannot update datapoint "${ datapoint.key }" as it has duplicate sub-datapoints with the same name: \n${duplicateDataPoints.join( - '\n', - )}`, - ), + "\n" + )}` + ) ); encounteredError = true; } else { @@ -914,13 +913,13 @@ export async function syncDataSilos( await makeGraphQLRequest( client, UPDATE_OR_CREATE_DATA_POINT, - payload, + payload ); - } catch (err) { + } catch (error) { logger.info( colors.red( - `\nFailed to update datapoint "${datapoint.key}" for data silo "${title}"! - \n${err.message}`, - ), + `\nFailed to update datapoint "${datapoint.key}" for data silo "${title}"! - \n${error.message}` + ) ); encounteredError = true; } @@ -932,11 +931,11 @@ export async function syncDataSilos( }, { concurrency: 10, - }, + } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( @@ -945,8 +944,8 @@ export async function syncDataSilos( dataSilos.length }" data silos and "${totalDataPoints}" datapoints in "${ totalTime / 1000 - }" seconds!`, - ), + }" seconds!` + ) ); return { success: !encounteredError, @@ -963,13 +962,13 @@ export async function syncDataSilos( */ export async function syncDataSiloDependencies( client: GraphQLClient, - dependencyUpdates: [string, string[]][], + dependencyUpdates: [string, string[]][] ): Promise { let encounteredError = false; logger.info( colors.magenta( - `Syncing "${dependencyUpdates.length}" data silo dependencies...`, - ), + `Syncing "${dependencyUpdates.length}" data silo dependencies...` + ) ); // Batch the updates @@ -977,15 +976,15 @@ export async function syncDataSiloDependencies( await mapSeries(chunkedUpdates, async (dependencyUpdateChunk, ind) => { logger.info( colors.magenta( - `[Batch ${ind}/${dependencyUpdateChunk.length}] Updating "${dependencyUpdateChunk.length}" data silos...`, - ), + `[Batch ${ind}/${dependencyUpdateChunk.length}] Updating "${dependencyUpdateChunk.length}" data silos...` + ) ); try { await makeGraphQLRequest<{ /** Mutation result */ updateDataSilos: { /** New data silos */ - dataSilos: Pick[]; + dataSilos: Pick[]; }; }>(client, UPDATE_DATA_SILOS, { input: { @@ -993,26 +992,25 @@ export async function syncDataSiloDependencies( ([id, dependedOnDataSiloTitles]) => ({ id, dependedOnDataSiloTitles, - }), + }) ), }, }); logger.info( colors.green( `[Batch ${ind + 1}/${dependencyUpdateChunk.length}] ` + - `Synced "${dependencyUpdateChunk.length}" data silos!`, - ), + `Synced "${dependencyUpdateChunk.length}" data silos!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( `[Batch ${ind + 1}/${dependencyUpdateChunk.length}] ` + - `Failed to update "${dependencyUpdateChunk.length}" silos! - ${err.message}`, - ), + `Failed to update "${dependencyUpdateChunk.length}" silos! - ${error.message}` + ) ); } }); return !encounteredError; } -/* eslint-enable max-lines */ diff --git a/src/lib/graphql/syncDataSubject.ts b/src/lib/graphql/syncDataSubject.ts index 48d868df..e548d91c 100644 --- a/src/lib/graphql/syncDataSubject.ts +++ b/src/lib/graphql/syncDataSubject.ts @@ -1,7 +1,7 @@ -import { GraphQLClient } from 'graphql-request'; -import { DataSubjectInput } from '../../codecs'; -import { TOGGLE_DATA_SUBJECT, UPDATE_DATA_SUBJECT } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { GraphQLClient } from "graphql-request"; +import { DataSubjectInput } from "../../codecs"; +import { TOGGLE_DATA_SUBJECT, UPDATE_DATA_SUBJECT } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Sync the data subjects @@ -22,7 +22,7 @@ export async function syncDataSubject( dataSubjectId: string; /** When true, skip publishing to privacy center */ skipPublish?: boolean; - }, + } ): Promise { await makeGraphQLRequest(client, UPDATE_DATA_SUBJECT, { input: { @@ -31,11 +31,11 @@ export async function syncDataSubject( adminDashboardDefaultSilentMode: dataSubject.adminDashboardDefaultSilentMode, actions: dataSubject.actions, - skipPublish: skipPublish && typeof dataSubject.active === 'undefined', + skipPublish: skipPublish && dataSubject.active === undefined, }, }); - if (typeof dataSubject.active === 'boolean') { + if (typeof dataSubject.active === "boolean") { await makeGraphQLRequest(client, TOGGLE_DATA_SUBJECT, { input: { id: dataSubjectId, diff --git a/src/lib/graphql/syncEnrichers.ts b/src/lib/graphql/syncEnrichers.ts index 00d6a0a1..f6e557bf 100644 --- a/src/lib/graphql/syncEnrichers.ts +++ b/src/lib/graphql/syncEnrichers.ts @@ -4,13 +4,13 @@ import { IsoCountrySubdivisionCode, PreflightRequestStatus, RequestAction, -} from '@transcend-io/privacy-types'; -import { GraphQLClient } from 'graphql-request'; -import { EnricherInput } from '../../codecs'; -import { DataSubject } from './fetchDataSubjects'; -import { Identifier } from './fetchIdentifiers'; -import { CREATE_ENRICHER, ENRICHERS, UPDATE_ENRICHER } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "@transcend-io/privacy-types"; +import { GraphQLClient } from "graphql-request"; +import { EnricherInput } from "../../codecs"; +import { DataSubject } from "./fetchDataSubjects"; +import { Identifier } from "./fetchIdentifiers"; +import { CREATE_ENRICHER, ENRICHERS, UPDATE_ENRICHER } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; export interface Enricher { /** ID of enricher */ @@ -65,7 +65,7 @@ const PAGE_SIZE = 20; */ export async function fetchAllEnrichers( client: GraphQLClient, - title?: string, + title?: string ): Promise { const enrichers: Enricher[] = []; let offset = 0; @@ -110,19 +110,19 @@ export async function syncEnricher( /** The enricher input */ enricher: EnricherInput; /** Index of identifiers in the organization */ - identifierByName: { [name in string]: Identifier }; + identifierByName: Record; /** Lookup data subject by name */ - dataSubjectsByName: { [name in string]: DataSubject }; - }, + dataSubjectsByName: Record; + } ): Promise { // Whether to continue looping const matches = await fetchAllEnrichers(client, enricher.title); const existingEnricher = matches.find( - ({ title }) => title === enricher.title, + ({ title }) => title === enricher.title ); // Map to data subject Ids - const dataSubjectIds = enricher['data-subjects']?.map((subject) => { + const dataSubjectIds = enricher["data-subjects"]?.map((subject) => { const existing = dataSubjectsByName[subject]; if (!existing) { throw new Error(`Failed to find a data subject with name: ${subject}`); @@ -131,9 +131,9 @@ export async function syncEnricher( }); // If enricher exists, update it, else create new - const inputIdentifier = enricher['input-identifier']; + const inputIdentifier = enricher["input-identifier"]; const actionUpdates = - enricher['privacy-actions'] || Object.values(RequestAction); + enricher["privacy-actions"] || Object.values(RequestAction); if (existingEnricher) { await makeGraphQLRequest(client, UPDATE_ENRICHER, { input: { @@ -144,19 +144,19 @@ export async function syncEnricher( testRegex: enricher.testRegex, lookerQueryTitle: enricher.lookerQueryTitle, expirationDuration: - typeof enricher.expirationDuration === 'number' + typeof enricher.expirationDuration === "number" ? enricher.expirationDuration.toString() : undefined, transitionRequestStatus: enricher.transitionRequestStatus, phoneNumbers: enricher.phoneNumbers, regionList: enricher.regionList, dataSubjectIds, - description: enricher.description || '', + description: enricher.description || "", inputIdentifier: inputIdentifier ? identifierByName[inputIdentifier].id : undefined, - identifiers: enricher['output-identifiers'].map( - (id) => identifierByName[id].id, + identifiers: enricher["output-identifiers"].map( + (id) => identifierByName[id].id ), ...(existingEnricher.type === EnricherType.Sombra ? {} @@ -173,17 +173,17 @@ export async function syncEnricher( testRegex: enricher.testRegex, lookerQueryTitle: enricher.lookerQueryTitle, expirationDuration: - typeof enricher.expirationDuration === 'number' + typeof enricher.expirationDuration === "number" ? enricher.expirationDuration.toString() : undefined, transitionRequestStatus: enricher.transitionRequestStatus, phoneNumbers: enricher.phoneNumbers, dataSubjectIds, regionList: enricher.regionList, - description: enricher.description || '', + description: enricher.description || "", inputIdentifier: identifierByName[inputIdentifier].id, - identifiers: enricher['output-identifiers'].map( - (id) => identifierByName[id].id, + identifiers: enricher["output-identifiers"].map( + (id) => identifierByName[id].id ), actions: actionUpdates, }, diff --git a/src/lib/graphql/syncIdentifier.ts b/src/lib/graphql/syncIdentifier.ts index fd653c3c..35edd820 100644 --- a/src/lib/graphql/syncIdentifier.ts +++ b/src/lib/graphql/syncIdentifier.ts @@ -1,8 +1,8 @@ -import { GraphQLClient } from 'graphql-request'; -import { IdentifierInput } from '../../codecs'; -import type { DataSubject } from './fetchDataSubjects'; -import { UPDATE_IDENTIFIER } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { GraphQLClient } from "graphql-request"; +import { IdentifierInput } from "../../codecs"; +import type { DataSubject } from "./fetchDataSubjects"; +import { UPDATE_IDENTIFIER } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Sync the consent manager @@ -21,12 +21,12 @@ export async function syncIdentifier( /** Identifier update input */ identifier: IdentifierInput; /** Data subject lookup by name */ - dataSubjectsByName: { [k in string]: DataSubject }; + dataSubjectsByName: Record; /** Existing identifier Id */ identifierId: string; /** When true, skip publishing to privacy center */ skipPublish?: boolean; - }, + } ): Promise { await makeGraphQLRequest(client, UPDATE_IDENTIFIER, { input: { diff --git a/src/lib/graphql/syncIntlMessages.ts b/src/lib/graphql/syncIntlMessages.ts index e6d88fcb..d879db54 100644 --- a/src/lib/graphql/syncIntlMessages.ts +++ b/src/lib/graphql/syncIntlMessages.ts @@ -1,11 +1,11 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { chunk } from 'lodash-es'; -import { IntlMessageInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { UPDATE_INTL_MESSAGES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { chunk } from "lodash-es"; +import { IntlMessageInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; +import { UPDATE_INTL_MESSAGES } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; const MAX_PAGE_SIZE = 100; @@ -17,21 +17,21 @@ const MAX_PAGE_SIZE = 100; */ export async function updateIntlMessages( client: GraphQLClient, - messageInputs: IntlMessageInput[], + messageInputs: IntlMessageInput[] ): Promise { // Batch update messages await mapSeries(chunk(messageInputs, MAX_PAGE_SIZE), async (page) => { await makeGraphQLRequest(client, UPDATE_INTL_MESSAGES, { messages: page.map((message) => ({ - ...(message.id.includes('.') ? {} : { id: message.id }), + ...(message.id.includes(".") ? {} : { id: message.id }), defaultMessage: message.defaultMessage, targetReactIntlId: message.targetReactIntlId, - translations: !message.translations - ? undefined - : Object.entries(message.translations).map(([locale, value]) => ({ + translations: message.translations + ? Object.entries(message.translations).map(([locale, value]) => ({ locale, value, - })), + })) + : undefined, })), }); }); @@ -46,34 +46,34 @@ export async function updateIntlMessages( */ export async function syncIntlMessages( client: GraphQLClient, - messages: IntlMessageInput[], + messages: IntlMessageInput[] ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${messages.length}" messages...`)); // Ensure no duplicates are being uploaded const notUnique = messages.filter( - (message) => messages.filter((pol) => message.id === pol.id).length > 1, + (message) => messages.filter((pol) => message.id === pol.id).length > 1 ); if (notUnique.length > 0) { throw new Error( `Failed to upload messages as there were non-unique entries found: ${notUnique .map(({ id }) => id) - .join(',')}`, + .join(",")}` ); } try { logger.info( - colors.magenta(`Upserting "${messages.length}" new messages...`), + colors.magenta(`Upserting "${messages.length}" new messages...`) ); await updateIntlMessages(client, messages); logger.info( - colors.green(`Successfully synced ${messages.length} messages!`), + colors.green(`Successfully synced ${messages.length} messages!`) ); - } catch (err) { + } catch (error) { encounteredError = true; - logger.info(colors.red(`Failed to create messages! - ${err.message}`)); + logger.info(colors.red(`Failed to create messages! - ${error.message}`)); } return !encounteredError; diff --git a/src/lib/graphql/syncPartitions.ts b/src/lib/graphql/syncPartitions.ts index f0153e2b..cee07491 100644 --- a/src/lib/graphql/syncPartitions.ts +++ b/src/lib/graphql/syncPartitions.ts @@ -1,12 +1,12 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { difference } from 'lodash-es'; -import { PartitionInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { fetchConsentManagerId } from './fetchConsentManagerId'; -import { CONSENT_PARTITIONS, CREATE_CONSENT_PARTITION } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { difference } from "lodash-es"; +import { PartitionInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; +import { fetchConsentManagerId } from "./fetchConsentManagerId"; +import { CONSENT_PARTITIONS, CREATE_CONSENT_PARTITION } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; const PAGE_SIZE = 50; @@ -26,7 +26,7 @@ export interface TranscendPartition { * @returns Partition list */ export async function fetchPartitions( - client: GraphQLClient, + client: GraphQLClient ): Promise { const partitions: TranscendPartition[] = []; let offset = 0; @@ -63,7 +63,7 @@ export async function fetchPartitions( */ export async function syncPartitions( client: GraphQLClient, - partitionInputs: PartitionInput[], + partitionInputs: PartitionInput[] ): Promise { // Grab the bundleId associated with this API key const airgapBundleId = await fetchConsentManagerId(client); @@ -71,7 +71,7 @@ export async function syncPartitions( const partitions = await fetchPartitions(client); const newPartitionNames = difference( partitionInputs.map(({ name }) => name), - partitions.map(({ name }) => name), + partitions.map(({ name }) => name) ); await mapSeries(newPartitionNames, async (name) => { try { @@ -82,13 +82,13 @@ export async function syncPartitions( }, }); logger.info( - colors.green(`Successfully created consent partition: ${name}!`), + colors.green(`Successfully created consent partition: ${name}!`) ); - } catch (err) { + } catch (error) { logger.error( colors.red( - `Failed to create consent partition: ${name}! - ${err.message}`, - ), + `Failed to create consent partition: ${name}! - ${error.message}` + ) ); encounteredError = true; } diff --git a/src/lib/graphql/syncPolicies.ts b/src/lib/graphql/syncPolicies.ts index 46d0c0ff..c60cda73 100644 --- a/src/lib/graphql/syncPolicies.ts +++ b/src/lib/graphql/syncPolicies.ts @@ -1,13 +1,13 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { chunk, keyBy } from 'lodash-es'; -import { PolicyInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { fetchAllPolicies } from './fetchAllPolicies'; -import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; -import { UPDATE_POLICIES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { chunk, keyBy } from "lodash-es"; +import { PolicyInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; +import { fetchAllPolicies } from "./fetchAllPolicies"; +import { fetchPrivacyCenterId } from "./fetchPrivacyCenterId"; +import { UPDATE_POLICIES } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; const MAX_PAGE_SIZE = 100; @@ -19,7 +19,7 @@ const MAX_PAGE_SIZE = 100; */ export async function updatePolicies( client: GraphQLClient, - policyInputs: [PolicyInput, string | undefined][], + policyInputs: [PolicyInput, string | undefined][] ): Promise { const privacyCenterId = await fetchPrivacyCenterId(client); @@ -62,20 +62,20 @@ export async function updatePolicies( */ export async function syncPolicies( client: GraphQLClient, - policies: PolicyInput[], + policies: PolicyInput[] ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${policies.length}" policies...`)); // Ensure no duplicates are being uploaded const notUnique = policies.filter( - (policy) => policies.filter((pol) => policy.title === pol.title).length > 1, + (policy) => policies.filter((pol) => policy.title === pol.title).length > 1 ); if (notUnique.length > 0) { throw new Error( `Failed to upload policies as there were non-unique entries found: ${notUnique .map(({ title }) => title) - .join(',')}`, + .join(",")}` ); } @@ -83,23 +83,23 @@ export async function syncPolicies( const existingPolicies = await fetchAllPolicies(client); const policiesById = keyBy( existingPolicies, - ({ title }) => title.defaultMessage, + ({ title }) => title.defaultMessage ); try { logger.info( - colors.magenta(`Upserting "${policies.length}" new policies...`), + colors.magenta(`Upserting "${policies.length}" new policies...`) ); await updatePolicies( client, - policies.map((policy) => [policy, policiesById[policy.title]?.id]), + policies.map((policy) => [policy, policiesById[policy.title]?.id]) ); logger.info( - colors.green(`Successfully synced ${policies.length} policies!`), + colors.green(`Successfully synced ${policies.length} policies!`) ); - } catch (err) { + } catch (error) { encounteredError = true; - logger.info(colors.red(`Failed to create policies! - ${err.message}`)); + logger.info(colors.red(`Failed to create policies! - ${error.message}`)); } return !encounteredError; diff --git a/src/lib/graphql/syncPrivacyCenter.ts b/src/lib/graphql/syncPrivacyCenter.ts index b6e21db7..ac8c90ca 100644 --- a/src/lib/graphql/syncPrivacyCenter.ts +++ b/src/lib/graphql/syncPrivacyCenter.ts @@ -1,10 +1,10 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { PrivacyCenterInput } from '../../codecs'; -import { logger } from '../../logger'; -import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; -import { UPDATE_PRIVACY_CENTER } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { PrivacyCenterInput } from "../../codecs"; +import { logger } from "../../logger"; +import { fetchPrivacyCenterId } from "./fetchPrivacyCenterId"; +import { UPDATE_PRIVACY_CENTER } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Sync the privacy center @@ -15,10 +15,10 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; */ export async function syncPrivacyCenter( client: GraphQLClient, - privacyCenter: PrivacyCenterInput, + privacyCenter: PrivacyCenterInput ): Promise { let encounteredError = false; - logger.info(colors.magenta('Syncing privacy center...')); + logger.info(colors.magenta("Syncing privacy center...")); // Grab the privacy center ID const privacyCenterId = await fetchPrivacyCenterId(client); @@ -54,11 +54,11 @@ export async function syncPrivacyCenter( : {}), }, }); - logger.info(colors.green('Successfully synced privacy center!')); - } catch (err) { + logger.info(colors.green("Successfully synced privacy center!")); + } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create privacy center! - ${err.message}`), + colors.red(`Failed to create privacy center! - ${error.message}`) ); } diff --git a/src/lib/graphql/syncProcessingPurposes.ts b/src/lib/graphql/syncProcessingPurposes.ts index c8a74b37..52a334cc 100644 --- a/src/lib/graphql/syncProcessingPurposes.ts +++ b/src/lib/graphql/syncProcessingPurposes.ts @@ -1,18 +1,18 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy } from 'lodash-es'; -import { ProcessingPurposeInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { keyBy } from "lodash-es"; +import { ProcessingPurposeInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; import { fetchAllProcessingPurposes, ProcessingPurposeSubCategory, -} from './fetchAllProcessingPurposes'; +} from "./fetchAllProcessingPurposes"; import { CREATE_PROCESSING_PURPOSE_SUB_CATEGORY, UPDATE_PROCESSING_PURPOSE_SUB_CATEGORIES, -} from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Input to create a new processing purpose @@ -23,8 +23,8 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; */ export async function createProcessingPurpose( client: GraphQLClient, - processingPurpose: ProcessingPurposeInput, -): Promise> { + processingPurpose: ProcessingPurposeInput +): Promise> { const input = { name: processingPurpose.name, purpose: processingPurpose.purpose, @@ -52,7 +52,7 @@ export async function createProcessingPurpose( */ export async function updateProcessingPurposes( client: GraphQLClient, - processingPurposeIdPairs: [ProcessingPurposeInput, string][], + processingPurposeIdPairs: [ProcessingPurposeInput, string][] ): Promise { await makeGraphQLRequest(client, UPDATE_PROCESSING_PURPOSE_SUB_CATEGORIES, { input: { @@ -62,7 +62,7 @@ export async function updateProcessingPurposes( description: processingPurpose.description, // TODO: https://transcend.height.app/T-31994 - add teams, owners attributes: processingPurpose.attributes, - }), + }) ), }, }); @@ -77,11 +77,11 @@ export async function updateProcessingPurposes( */ export async function syncProcessingPurposes( client: GraphQLClient, - inputs: ProcessingPurposeInput[], + inputs: ProcessingPurposeInput[] ): Promise { // Fetch existing logger.info( - colors.magenta(`Syncing "${inputs.length}" processing purposes...`), + colors.magenta(`Syncing "${inputs.length}" processing purposes...`) ); let encounteredError = false; @@ -90,16 +90,17 @@ export async function syncProcessingPurposes( const existingProcessingPurposes = await fetchAllProcessingPurposes(client); // Look up by name - const processingPurposeByName: { - [k in string]: Pick; - } = keyBy( + const processingPurposeByName: Record< + string, + Pick + > = keyBy( existingProcessingPurposes, - ({ name, purpose }) => `${name}:${purpose}`, + ({ name, purpose }) => `${name}:${purpose}` ); // Create new processing purposes const newProcessingPurposes = inputs.filter( - (input) => !processingPurposeByName[`${input.name}:${input.purpose}`], + (input) => !processingPurposeByName[`${input.name}:${input.purpose}`] ); // Create new processing purposes @@ -107,22 +108,22 @@ export async function syncProcessingPurposes( try { const newProcessingPurpose = await createProcessingPurpose( client, - processingPurpose, + processingPurpose ); processingPurposeByName[ `${newProcessingPurpose.name}:${newProcessingPurpose.purpose}` ] = newProcessingPurpose; logger.info( colors.green( - `Successfully synced processing purpose "${processingPurpose.name}"!`, - ), + `Successfully synced processing purpose "${processingPurpose.name}"!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync processing purpose "${processingPurpose.name}"! - ${err.message}`, - ), + `Failed to sync processing purpose "${processingPurpose.name}"! - ${error.message}` + ) ); } }); @@ -130,26 +131,26 @@ export async function syncProcessingPurposes( // Update all processing purposes try { logger.info( - colors.magenta(`Updating "${inputs.length}" processing purposes!`), + colors.magenta(`Updating "${inputs.length}" processing purposes!`) ); await updateProcessingPurposes( client, inputs.map((input) => [ input, processingPurposeByName[`${input.name}:${input.purpose}`].id, - ]), + ]) ); logger.info( colors.green( - `Successfully synced "${inputs.length}" processing purposes!`, - ), + `Successfully synced "${inputs.length}" processing purposes!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" processing purposes ! - ${err.message}`, - ), + `Failed to sync "${inputs.length}" processing purposes ! - ${error.message}` + ) ); } diff --git a/src/lib/graphql/syncPromptGroups.ts b/src/lib/graphql/syncPromptGroups.ts index 82865c42..659249a3 100644 --- a/src/lib/graphql/syncPromptGroups.ts +++ b/src/lib/graphql/syncPromptGroups.ts @@ -1,13 +1,13 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy } from 'lodash-es'; -import { PromptGroupInput } from '../../codecs'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; -import { fetchAllPromptGroups } from './fetchPromptGroups'; -import { fetchAllPrompts } from './fetchPrompts'; -import { CREATE_PROMPT_GROUP, UPDATE_PROMPT_GROUPS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { keyBy } from "lodash-es"; +import { PromptGroupInput } from "../../codecs"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; +import { fetchAllPromptGroups } from "./fetchPromptGroups"; +import { fetchAllPrompts } from "./fetchPrompts"; +import { CREATE_PROMPT_GROUP, UPDATE_PROMPT_GROUPS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; export interface EditPromptGroupInput { /** Title of prompt group */ @@ -27,7 +27,7 @@ export interface EditPromptGroupInput { */ export async function createPromptGroup( client: GraphQLClient, - input: EditPromptGroupInput, + input: EditPromptGroupInput ): Promise { const { createPromptGroup: { promptGroup }, @@ -44,7 +44,7 @@ export async function createPromptGroup( input, }); logger.info( - colors.green(`Successfully created prompt group "${input.title}"!`), + colors.green(`Successfully created prompt group "${input.title}"!`) ); return promptGroup.id; } @@ -57,7 +57,7 @@ export async function createPromptGroup( */ export async function updatePromptGroups( client: GraphQLClient, - input: [EditPromptGroupInput, string][], + input: [EditPromptGroupInput, string][] ): Promise { await makeGraphQLRequest(client, UPDATE_PROMPT_GROUPS, { input: { @@ -68,7 +68,7 @@ export async function updatePromptGroups( }, }); logger.info( - colors.green(`Successfully updated ${input.length} prompt groups!`), + colors.green(`Successfully updated ${input.length} prompt groups!`) ); } @@ -83,18 +83,18 @@ export async function updatePromptGroups( export async function syncPromptGroups( client: GraphQLClient, promptGroups: PromptGroupInput[], - concurrency = 20, + concurrency = 20 ): Promise { let encounteredError = false; logger.info( - colors.magenta(`Syncing "${promptGroups.length}" prompt groups...`), + colors.magenta(`Syncing "${promptGroups.length}" prompt groups...`) ); // Index existing prompt groups const existing = await fetchAllPromptGroups(client); const existingPrompts = await fetchAllPrompts(client); - const promptByTitle = keyBy(existingPrompts, 'title'); - const promptGroupByTitle = keyBy(existing, 'title'); + const promptByTitle = keyBy(existingPrompts, "title"); + const promptGroupByTitle = keyBy(existing, "title"); // Determine which promptGroups are new vs existing const mapPromptGroupsToExisting = promptGroups.map((promptInput) => [ @@ -109,8 +109,8 @@ export async function syncPromptGroups( try { logger.info( colors.magenta( - `Creating "${newPromptGroups.length}" new prompt groups...`, - ), + `Creating "${newPromptGroups.length}" new prompt groups...` + ) ); await map( newPromptGroups, @@ -128,27 +128,29 @@ export async function syncPromptGroups( }, { concurrency, - }, + } ); logger.info( colors.green( - `Successfully synced ${newPromptGroups.length} prompt groups!`, - ), + `Successfully synced ${newPromptGroups.length} prompt groups!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; - logger.info(colors.red(`Failed to create prompt groups! - ${err.message}`)); + logger.info( + colors.red(`Failed to create prompt groups! - ${error.message}`) + ); } // Update existing promptGroups const existingPromptGroups = mapPromptGroupsToExisting.filter( - (x): x is [PromptGroupInput, string] => !!x[1], + (x): x is [PromptGroupInput, string] => !!x[1] ); try { logger.info( colors.magenta( - `Updating "${existingPromptGroups.length}" prompt groups...`, - ), + `Updating "${existingPromptGroups.length}" prompt groups...` + ) ); await updatePromptGroups( client, @@ -164,16 +166,18 @@ export async function syncPromptGroups( }), }, id, - ]), + ]) ); logger.info( colors.green( - `Successfully updated "${existingPromptGroups.length}" prompt groups!`, - ), + `Successfully updated "${existingPromptGroups.length}" prompt groups!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; - logger.info(colors.red(`Failed to create prompt groups! - ${err.message}`)); + logger.info( + colors.red(`Failed to create prompt groups! - ${error.message}`) + ); } logger.info(colors.green(`Synced "${promptGroups.length}" prompt groups!`)); diff --git a/src/lib/graphql/syncPromptPartials.ts b/src/lib/graphql/syncPromptPartials.ts index 424e245a..2668f030 100644 --- a/src/lib/graphql/syncPromptPartials.ts +++ b/src/lib/graphql/syncPromptPartials.ts @@ -1,12 +1,12 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy } from 'lodash-es'; -import { PromptPartialInput } from '../../codecs'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; -import { fetchAllPromptPartials } from './fetchPromptPartials'; -import { CREATE_PROMPT_PARTIAL, UPDATE_PROMPT_PARTIALS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { keyBy } from "lodash-es"; +import { PromptPartialInput } from "../../codecs"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; +import { fetchAllPromptPartials } from "./fetchPromptPartials"; +import { CREATE_PROMPT_PARTIAL, UPDATE_PROMPT_PARTIALS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Create a new prompt partial @@ -22,7 +22,7 @@ export async function createPromptPartial( title: string; /** Prompt content */ content: string; - }, + } ): Promise { const { createPromptPartial: { promptPartial }, @@ -39,7 +39,7 @@ export async function createPromptPartial( input, }); logger.info( - colors.green(`Successfully created prompt partial "${input.title}"!`), + colors.green(`Successfully created prompt partial "${input.title}"!`) ); return promptPartial.id; } @@ -52,7 +52,7 @@ export async function createPromptPartial( */ export async function updatePromptPartials( client: GraphQLClient, - input: [PromptPartialInput, string][], + input: [PromptPartialInput, string][] ): Promise { await makeGraphQLRequest(client, UPDATE_PROMPT_PARTIALS, { input: { @@ -63,7 +63,7 @@ export async function updatePromptPartials( }, }); logger.info( - colors.green(`Successfully updated ${input.length} prompt partials!`), + colors.green(`Successfully updated ${input.length} prompt partials!`) ); } @@ -78,16 +78,16 @@ export async function updatePromptPartials( export async function syncPromptPartials( client: GraphQLClient, promptPartials: PromptPartialInput[], - concurrency = 20, + concurrency = 20 ): Promise { let encounteredError = false; logger.info( - colors.magenta(`Syncing "${promptPartials.length}" prompt partials...`), + colors.magenta(`Syncing "${promptPartials.length}" prompt partials...`) ); // Index existing prompt partials const existing = await fetchAllPromptPartials(client); - const promptPartialByTitle = keyBy(existing, 'title'); + const promptPartialByTitle = keyBy(existing, "title"); // Determine which promptPartials are new vs existing const mapPromptPartialsToExisting = promptPartials.map((promptInput) => [ @@ -102,8 +102,8 @@ export async function syncPromptPartials( try { logger.info( colors.magenta( - `Creating "${newPromptPartials.length}" new prompt partials...`, - ), + `Creating "${newPromptPartials.length}" new prompt partials...` + ) ); await map( newPromptPartials, @@ -112,45 +112,45 @@ export async function syncPromptPartials( }, { concurrency, - }, + } ); logger.info( colors.green( - `Successfully synced ${newPromptPartials.length} prompt partials!`, - ), + `Successfully synced ${newPromptPartials.length} prompt partials!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create prompt partials! - ${err.message}`), + colors.red(`Failed to create prompt partials! - ${error.message}`) ); } // Update existing promptPartials const existingPromptPartials = mapPromptPartialsToExisting.filter( - (x): x is [PromptPartialInput, string] => !!x[1], + (x): x is [PromptPartialInput, string] => !!x[1] ); try { logger.info( colors.magenta( - `Updating "${existingPromptPartials.length}" prompt partials...`, - ), + `Updating "${existingPromptPartials.length}" prompt partials...` + ) ); await updatePromptPartials(client, existingPromptPartials); logger.info( colors.green( - `Successfully updated "${existingPromptPartials.length}" prompt partials!`, - ), + `Successfully updated "${existingPromptPartials.length}" prompt partials!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create prompt partials! - ${err.message}`), + colors.red(`Failed to create prompt partials! - ${error.message}`) ); } logger.info( - colors.green(`Synced "${promptPartials.length}" prompt partials!`), + colors.green(`Synced "${promptPartials.length}" prompt partials!`) ); // Return true upon success diff --git a/src/lib/graphql/syncPrompts.ts b/src/lib/graphql/syncPrompts.ts index 6e0819de..4864b8cf 100644 --- a/src/lib/graphql/syncPrompts.ts +++ b/src/lib/graphql/syncPrompts.ts @@ -1,12 +1,12 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy } from 'lodash-es'; -import { PromptInput } from '../../codecs'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; -import { fetchAllPrompts } from './fetchPrompts'; -import { CREATE_PROMPT, UPDATE_PROMPTS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { keyBy } from "lodash-es"; +import { PromptInput } from "../../codecs"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; +import { fetchAllPrompts } from "./fetchPrompts"; +import { CREATE_PROMPT, UPDATE_PROMPTS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Create a new prompt @@ -22,7 +22,7 @@ export async function createPrompt( title: string; /** Prompt content */ content: string; - }, + } ): Promise { const { createPrompt: { prompt }, @@ -51,7 +51,7 @@ export async function createPrompt( */ export async function updatePrompts( client: GraphQLClient, - input: [PromptInput, string][], + input: [PromptInput, string][] ): Promise { await makeGraphQLRequest(client, UPDATE_PROMPTS, { input: { @@ -75,14 +75,14 @@ export async function updatePrompts( export async function syncPrompts( client: GraphQLClient, prompts: PromptInput[], - concurrency = 20, + concurrency = 20 ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${prompts.length}" prompts...`)); // Index existing prompts const existing = await fetchAllPrompts(client); - const promptByTitle = keyBy(existing, 'title'); + const promptByTitle = keyBy(existing, "title"); // Determine which prompts are new vs existing const mapPromptsToExisting = prompts.map((promptInput) => [ @@ -96,7 +96,7 @@ export async function syncPrompts( .map(([promptInput]) => promptInput as PromptInput); try { logger.info( - colors.magenta(`Creating "${newPrompts.length}" new prompts...`), + colors.magenta(`Creating "${newPrompts.length}" new prompts...`) ); await map( newPrompts, @@ -105,31 +105,31 @@ export async function syncPrompts( }, { concurrency, - }, + } ); logger.info( - colors.green(`Successfully synced ${newPrompts.length} prompts!`), + colors.green(`Successfully synced ${newPrompts.length} prompts!`) ); - } catch (err) { + } catch (error) { encounteredError = true; - logger.info(colors.red(`Failed to create prompts! - ${err.message}`)); + logger.info(colors.red(`Failed to create prompts! - ${error.message}`)); } // Update existing prompts const existingPrompts = mapPromptsToExisting.filter( - (x): x is [PromptInput, string] => !!x[1], + (x): x is [PromptInput, string] => !!x[1] ); try { logger.info( - colors.magenta(`Updating "${existingPrompts.length}" prompts...`), + colors.magenta(`Updating "${existingPrompts.length}" prompts...`) ); await updatePrompts(client, existingPrompts); logger.info( - colors.green(`Successfully updated "${existingPrompts.length}" prompts!`), + colors.green(`Successfully updated "${existingPrompts.length}" prompts!`) ); - } catch (err) { + } catch (error) { encounteredError = true; - logger.info(colors.red(`Failed to create prompts! - ${err.message}`)); + logger.info(colors.red(`Failed to create prompts! - ${error.message}`)); } logger.info(colors.green(`Synced "${prompts.length}" prompts!`)); diff --git a/src/lib/graphql/syncRepositories.ts b/src/lib/graphql/syncRepositories.ts index 49fdfde8..61a8aac9 100644 --- a/src/lib/graphql/syncRepositories.ts +++ b/src/lib/graphql/syncRepositories.ts @@ -1,12 +1,12 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { chunk, keyBy } from 'lodash-es'; -import { RepositoryInput } from '../../codecs'; -import { logger } from '../../logger'; -import { map, mapSeries } from '../bluebird-replace'; -import { fetchAllRepositories, Repository } from './fetchAllRepositories'; -import { CREATE_REPOSITORY, UPDATE_REPOSITORIES } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { chunk, keyBy } from "lodash-es"; +import { RepositoryInput } from "../../codecs"; +import { logger } from "../../logger"; +import { map, mapSeries } from "../bluebird-replace"; +import { fetchAllRepositories, Repository } from "./fetchAllRepositories"; +import { CREATE_REPOSITORY, UPDATE_REPOSITORIES } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; const CHUNK_SIZE = 100; @@ -34,7 +34,7 @@ export async function createRepository( teamIds?: string[]; /** Team names */ teamNames?: string[]; - }, + } ): Promise { const { createRepository: { repository }, @@ -77,7 +77,7 @@ export async function updateRepositories( teamIds?: string[]; /** Team names */ teamNames?: string[]; - }[], + }[] ): Promise { const { updateRepositories: { repositories }, @@ -93,7 +93,7 @@ export async function updateRepositories( }, }); logger.info( - colors.green(`Successfully updated ${inputs.length} repositories!`), + colors.green(`Successfully updated ${inputs.length} repositories!`) ); return repositories; } @@ -109,7 +109,7 @@ export async function updateRepositories( export async function syncRepositories( client: GraphQLClient, repositories: RepositoryInput[], - concurrency = 20, + concurrency = 20 ): Promise<{ /** The repositories that were upserted */ repositories: Repository[]; @@ -121,7 +121,7 @@ export async function syncRepositories( // Index existing repositories const existing = await fetchAllRepositories(client); - const repositoryByName = keyBy(existing, 'name'); + const repositoryByName = keyBy(existing, "name"); // Determine which repositories are new vs existing const mapRepositoriesToExisting = repositories.map((repoInput) => [ @@ -135,9 +135,7 @@ export async function syncRepositories( .map(([repoInput]) => repoInput as RepositoryInput); try { logger.info( - colors.magenta( - `Creating "${newRepositories.length}" new repositories...`, - ), + colors.magenta(`Creating "${newRepositories.length}" new repositories...`) ); await map( newRepositories, @@ -147,25 +145,27 @@ export async function syncRepositories( }, { concurrency, - }, + } ); logger.info( colors.green( - `Successfully synced ${newRepositories.length} repositories!`, - ), + `Successfully synced ${newRepositories.length} repositories!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; - logger.info(colors.red(`Failed to create repositories! - ${err.message}`)); + logger.info( + colors.red(`Failed to create repositories! - ${error.message}`) + ); } // Update existing repositories const existingRepositories = mapRepositoriesToExisting.filter( - (x): x is [RepositoryInput, string] => !!x[1], + (x): x is [RepositoryInput, string] => !!x[1] ); const chunks = chunk(existingRepositories, CHUNK_SIZE); logger.info( - colors.magenta(`Updating "${existingRepositories.length}" repositories...`), + colors.magenta(`Updating "${existingRepositories.length}" repositories...`) ); await mapSeries(chunks, async (chunk) => { @@ -175,18 +175,18 @@ export async function syncRepositories( chunk.map(([input, id]) => ({ ...input, id, - })), + })) ); repos.push(...updatedRepos); logger.info( colors.green( - `Successfully updated "${existingRepositories.length}" repositories!`, - ), + `Successfully updated "${existingRepositories.length}" repositories!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to update repositories! - ${err.message}`), + colors.red(`Failed to update repositories! - ${error.message}`) ); } diff --git a/src/lib/graphql/syncSoftwareDevelopmentKits.ts b/src/lib/graphql/syncSoftwareDevelopmentKits.ts index fff8a0f9..227fede5 100644 --- a/src/lib/graphql/syncSoftwareDevelopmentKits.ts +++ b/src/lib/graphql/syncSoftwareDevelopmentKits.ts @@ -1,19 +1,19 @@ -import { CodePackageType } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { chunk, keyBy } from 'lodash-es'; -import { SoftwareDevelopmentKitInput } from '../../codecs'; -import { logger } from '../../logger'; -import { map, mapSeries } from '../bluebird-replace'; +import { CodePackageType } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { chunk, keyBy } from "lodash-es"; +import { SoftwareDevelopmentKitInput } from "../../codecs"; +import { logger } from "../../logger"; +import { map, mapSeries } from "../bluebird-replace"; import { fetchAllSoftwareDevelopmentKits, SoftwareDevelopmentKit, -} from './fetchAllSoftwareDevelopmentKits'; +} from "./fetchAllSoftwareDevelopmentKits"; import { CREATE_SOFTWARE_DEVELOPMENT_KIT, UPDATE_SOFTWARE_DEVELOPMENT_KITS, -} from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +} from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; const CHUNK_SIZE = 100; @@ -51,7 +51,7 @@ export async function createSoftwareDevelopmentKit( teamIds?: string[]; /** Team names */ teamNames?: string[]; - }, + } ): Promise { const { createSoftwareDevelopmentKit: { softwareDevelopmentKit }, @@ -66,8 +66,8 @@ export async function createSoftwareDevelopmentKit( }); logger.info( colors.green( - `Successfully created software development kit "${input.name}"!`, - ), + `Successfully created software development kit "${input.name}"!` + ) ); return softwareDevelopmentKit; } @@ -106,7 +106,7 @@ export async function updateSoftwareDevelopmentKits( teamIds?: string[]; /** Team names */ teamNames?: string[]; - }[], + }[] ): Promise { const { updateSoftwareDevelopmentKits: { softwareDevelopmentKits }, @@ -123,8 +123,8 @@ export async function updateSoftwareDevelopmentKits( }); logger.info( colors.green( - `Successfully updated ${inputs.length} software development kits!`, - ), + `Successfully updated ${inputs.length} software development kits!` + ) ); return softwareDevelopmentKits; } @@ -140,7 +140,7 @@ export async function updateSoftwareDevelopmentKits( export async function syncSoftwareDevelopmentKits( client: GraphQLClient, softwareDevelopmentKits: SoftwareDevelopmentKitInput[], - concurrency = 20, + concurrency = 20 ): Promise<{ /** The SDKs that were upserted */ softwareDevelopmentKits: SoftwareDevelopmentKit[]; @@ -149,13 +149,13 @@ export async function syncSoftwareDevelopmentKits( }> { let encounteredError = false; const sdks: SoftwareDevelopmentKit[] = []; - logger.info(colors.magenta('Syncing software development kits...')); + logger.info(colors.magenta("Syncing software development kits...")); // Index existing software development kits const existing = await fetchAllSoftwareDevelopmentKits(client); const softwareDevelopmentKitByTitle = keyBy( existing, - ({ name, codePackageType }) => JSON.stringify({ name, codePackageType }), + ({ name, codePackageType }) => JSON.stringify({ name, codePackageType }) ); // Determine which software development kits are new vs existing @@ -168,7 +168,7 @@ export async function syncSoftwareDevelopmentKits( codePackageType: sdkInput.codePackageType, }) ]?.id, - ], + ] ); // Create the new software development kits @@ -178,8 +178,8 @@ export async function syncSoftwareDevelopmentKits( try { logger.info( colors.magenta( - `Creating "${newSoftwareDevelopmentKits.length}" new software development kits...`, - ), + `Creating "${newSoftwareDevelopmentKits.length}" new software development kits...` + ) ); await map( newSoftwareDevelopmentKits, @@ -189,32 +189,32 @@ export async function syncSoftwareDevelopmentKits( }, { concurrency, - }, + } ); logger.info( colors.green( - `Successfully synced ${newSoftwareDevelopmentKits.length} software development kits!`, - ), + `Successfully synced ${newSoftwareDevelopmentKits.length} software development kits!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to create software development kits! - ${err.message}`, - ), + `Failed to create software development kits! - ${error.message}` + ) ); } // Update existing software development kits const existingSoftwareDevelopmentKits = mapSoftwareDevelopmentKitsToExisting.filter( - (x): x is [SoftwareDevelopmentKitInput, string] => !!x[1], + (x): x is [SoftwareDevelopmentKitInput, string] => !!x[1] ); const chunks = chunk(existingSoftwareDevelopmentKits, CHUNK_SIZE); logger.info( colors.magenta( - `Updating "${existingSoftwareDevelopmentKits.length}" software development kits...`, - ), + `Updating "${existingSoftwareDevelopmentKits.length}" software development kits...` + ) ); await mapSeries(chunks, async (chunk) => { @@ -225,27 +225,27 @@ export async function syncSoftwareDevelopmentKits( chunk.map(([{ codePackageType, ...input }, id]) => ({ ...input, id, - })), + })) ); sdks.push(...updatedSdks); logger.info( colors.green( - `Successfully updated "${existingSoftwareDevelopmentKits.length}" software development kits!`, - ), + `Successfully updated "${existingSoftwareDevelopmentKits.length}" software development kits!` + ) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to update software development kits! - ${err.message}`, - ), + `Failed to update software development kits! - ${error.message}` + ) ); } logger.info( colors.green( - `Synced "${softwareDevelopmentKits.length}" software development kits!`, - ), + `Synced "${softwareDevelopmentKits.length}" software development kits!` + ) ); }); diff --git a/src/lib/graphql/syncTeams.ts b/src/lib/graphql/syncTeams.ts index 998ddc58..f22199bc 100644 --- a/src/lib/graphql/syncTeams.ts +++ b/src/lib/graphql/syncTeams.ts @@ -1,12 +1,12 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy } from 'lodash-es'; -import { TeamInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { fetchAllTeams, Team } from './fetchAllTeams'; -import { CREATE_TEAM, UPDATE_TEAM } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { keyBy } from "lodash-es"; +import { TeamInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; +import { fetchAllTeams, Team } from "./fetchAllTeams"; +import { CREATE_TEAM, UPDATE_TEAM } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Input to create a new team @@ -17,14 +17,14 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; */ export async function createTeam( client: GraphQLClient, - team: TeamInput, -): Promise> { + team: TeamInput +): Promise> { const input = { name: team.name, description: team.description, - ssoTitle: team['sso-title'], - ssoDepartment: team['sso-department'], - ssoGroup: team['sso-group'], + ssoTitle: team["sso-title"], + ssoDepartment: team["sso-department"], + ssoGroup: team["sso-group"], scopes: team.scopes, userEmails: team.users, }; @@ -52,8 +52,8 @@ export async function createTeam( export async function updateTeam( client: GraphQLClient, input: TeamInput, - teamId: string, -): Promise> { + teamId: string +): Promise> { const { updateTeam } = await makeGraphQLRequest<{ /** Update team mutation */ updateTeam: { @@ -65,9 +65,9 @@ export async function updateTeam( id: teamId, name: input.name, description: input.description, - ssoTitle: input['sso-title'], - ssoDepartment: input['sso-department'], - ssoGroup: input['sso-group'], + ssoTitle: input["sso-title"], + ssoDepartment: input["sso-department"], + ssoGroup: input["sso-group"], scopes: input.scopes, userEmails: input.users, }, @@ -84,7 +84,7 @@ export async function updateTeam( */ export async function syncTeams( client: GraphQLClient, - inputs: TeamInput[], + inputs: TeamInput[] ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" teams...`)); @@ -95,9 +95,9 @@ export async function syncTeams( const existingTeams = await fetchAllTeams(client); // Look up by name - const teamsByName: { [k in string]: Pick } = keyBy( + const teamsByName: Record> = keyBy( existingTeams, - 'name', + "name" ); // Create new teams @@ -110,10 +110,10 @@ export async function syncTeams( const newTeam = await createTeam(client, team); teamsByName[newTeam.name] = newTeam; logger.info(colors.green(`Successfully created team "${team.name}"!`)); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync team "${team.name}"! - ${err.message}`), + colors.red(`Failed to sync team "${team.name}"! - ${error.message}`) ); } }); @@ -124,14 +124,14 @@ export async function syncTeams( const newTeam = await updateTeam( client, input, - teamsByName[input.name].id, + teamsByName[input.name].id ); teamsByName[newTeam.name] = newTeam; logger.info(colors.green(`Successfully updated team "${input.name}"!`)); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync team "${input.name}"! - ${err.message}`), + colors.red(`Failed to sync team "${input.name}"! - ${error.message}`) ); } }); diff --git a/src/lib/graphql/syncVendors.ts b/src/lib/graphql/syncVendors.ts index 9df26c69..cd9e60c4 100644 --- a/src/lib/graphql/syncVendors.ts +++ b/src/lib/graphql/syncVendors.ts @@ -1,12 +1,12 @@ -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy } from 'lodash-es'; -import { VendorInput } from '../../codecs'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { fetchAllVendors, Vendor } from './fetchAllVendors'; -import { CREATE_VENDOR, UPDATE_VENDORS } from './gqls'; -import { makeGraphQLRequest } from './makeGraphQLRequest'; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { keyBy } from "lodash-es"; +import { VendorInput } from "../../codecs"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; +import { fetchAllVendors, Vendor } from "./fetchAllVendors"; +import { CREATE_VENDOR, UPDATE_VENDORS } from "./gqls"; +import { makeGraphQLRequest } from "./makeGraphQLRequest"; /** * Input to create a new vendor @@ -17,8 +17,8 @@ import { makeGraphQLRequest } from './makeGraphQLRequest'; */ export async function createVendor( client: GraphQLClient, - vendor: VendorInput, -): Promise> { + vendor: VendorInput +): Promise> { const input = { title: vendor.title, description: vendor.description, @@ -52,7 +52,7 @@ export async function createVendor( */ export async function updateVendors( client: GraphQLClient, - vendorIdParis: [VendorInput, string][], + vendorIdParis: [VendorInput, string][] ): Promise { await makeGraphQLRequest(client, UPDATE_VENDORS, { input: { @@ -83,7 +83,7 @@ export async function updateVendors( */ export async function syncVendors( client: GraphQLClient, - inputs: VendorInput[], + inputs: VendorInput[] ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" vendors...`)); @@ -94,9 +94,9 @@ export async function syncVendors( const existingVendors = await fetchAllVendors(client); // Look up by title - const vendorByTitle: { [k in string]: Pick } = keyBy( + const vendorByTitle: Record> = keyBy( existingVendors, - 'title', + "title" ); // Create new vendors @@ -108,12 +108,14 @@ export async function syncVendors( const newVendor = await createVendor(client, vendor); vendorByTitle[newVendor.title] = newVendor; logger.info( - colors.green(`Successfully synced vendor "${vendor.title}"!`), + colors.green(`Successfully synced vendor "${vendor.title}"!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync vendor "${vendor.title}"! - ${err.message}`), + colors.red( + `Failed to sync vendor "${vendor.title}"! - ${error.message}` + ) ); } }); @@ -123,17 +125,17 @@ export async function syncVendors( logger.info(colors.magenta(`Updating "${inputs.length}" vendors!`)); await updateVendors( client, - inputs.map((input) => [input, vendorByTitle[input.title].id]), + inputs.map((input) => [input, vendorByTitle[input.title].id]) ); logger.info( - colors.green(`Successfully synced "${inputs.length}" vendors!`), + colors.green(`Successfully synced "${inputs.length}" vendors!`) ); - } catch (err) { + } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" vendors ! - ${err.message}`, - ), + `Failed to sync "${inputs.length}" vendors ! - ${error.message}` + ) ); } diff --git a/src/lib/helpers/inquirer.ts b/src/lib/helpers/inquirer.ts index 21b501c8..911d90ca 100644 --- a/src/lib/helpers/inquirer.ts +++ b/src/lib/helpers/inquirer.ts @@ -1,7 +1,7 @@ -import { ObjByString } from '@transcend-io/type-utils'; -import inquirer from 'inquirer'; -import autoCompletePrompt from 'inquirer-autocomplete-prompt'; -import { fuzzySearch } from '../requests'; +import { ObjByString } from "@transcend-io/type-utils"; +import inquirer from "inquirer"; +import autoCompletePrompt from "inquirer-autocomplete-prompt"; +import { fuzzySearch } from "../requests"; /** * Inquirer confirm text @@ -20,9 +20,9 @@ export async function inquirerConfirmBoolean({ response: boolean; }>([ { - name: 'response', + name: "response", message, - type: 'confirm', + type: "confirm", }, ]); return response; @@ -45,9 +45,9 @@ export async function inquirerConfirmText({ response: string; }>([ { - name: 'response', + name: "response", message, - type: 'text', + type: "text", validate: (x) => x.trim().length > 0, }, ]); @@ -72,22 +72,20 @@ export async function inquirerAutoComplete({ /** Values to select */ values: string[]; }): Promise { - inquirer.registerPrompt('autocomplete', autoCompletePrompt); + inquirer.registerPrompt("autocomplete", autoCompletePrompt); const { response } = await inquirer.prompt<{ /** confirmation */ response: string; }>([ { - name: 'response', + name: "response", message, - type: 'autocomplete', + type: "autocomplete", default: defaultValue, source: (answersSoFar: ObjByString, input: string) => - !input - ? values - : values.filter( - (x) => typeof x === 'string' && fuzzySearch(input, x), - ), + input + ? values.filter((x) => typeof x === "string" && fuzzySearch(input, x)) + : values, }, ]); return response; diff --git a/src/lib/helpers/parseVariablesFromString.ts b/src/lib/helpers/parseVariablesFromString.ts index 3f909cc6..e58df764 100644 --- a/src/lib/helpers/parseVariablesFromString.ts +++ b/src/lib/helpers/parseVariablesFromString.ts @@ -4,20 +4,20 @@ * @param variables - Variables as string * @returns Variables as object */ -export function parseVariablesFromString(variables: string): { - [k in string]: string; -} { +export function parseVariablesFromString( + variables: string +): Record { // Parse out the variables - const splitVars = variables.split(',').filter((x) => !!x); - const vars: { [k in string]: string } = {}; - splitVars.forEach((variable) => { - const [k, v] = variable.split(':'); + const splitVariables = variables.split(",").filter((x) => !!x); + const variables_: Record = {}; + for (const variable of splitVariables) { + const [k, v] = variable.split(":"); if (!k || !v) { throw new Error( - `Invalid variable: ${variable}. Expected format: key:value`, + `Invalid variable: ${variable}. Expected format: key:value` ); } - vars[k] = v; - }); - return vars; + variables_[k] = v; + } + return variables_; } diff --git a/src/lib/manual-enrichment/enrichPrivacyRequest.ts b/src/lib/manual-enrichment/enrichPrivacyRequest.ts index 1d7859e5..e8999237 100644 --- a/src/lib/manual-enrichment/enrichPrivacyRequest.ts +++ b/src/lib/manual-enrichment/enrichPrivacyRequest.ts @@ -1,12 +1,12 @@ -import colors from 'colors'; -import type { Got } from 'got'; -import * as t from 'io-ts'; -import { uniq } from 'lodash-es'; -import { logger } from '../../logger'; -import { splitCsvToList } from '../requests/splitCsvToList'; +import colors from "colors"; +import type { Got } from "got"; +import * as t from "io-ts"; +import { uniq } from "lodash-es"; +import { logger } from "../../logger"; +import { splitCsvToList } from "../requests/splitCsvToList"; const ADMIN_URL = - 'https://app.transcend.io/privacy-requests/incoming-requests/'; + "https://app.transcend.io/privacy-requests/incoming-requests/"; /** * Minimal set required to mark as completed */ @@ -28,41 +28,40 @@ export async function enrichPrivacyRequest( sombra: Got, { id: rawId, ...rest }: EnrichPrivacyRequest, enricherId: string, - index?: number, + index?: number ): Promise { if (!rawId) { // error - const msg = `Request ID must be provided to enricher request.${ - index ? ` Found error in row: ${index}` : '' + const message = `Request ID must be provided to enricher request.${ + index ? ` Found error in row: ${index}` : "" }`; - logger.error(colors.red(msg)); - throw new Error(msg); + logger.error(colors.red(message)); + throw new Error(message); } const id = rawId.toLowerCase(); // Pull out the identifiers - const enrichedIdentifiers = Object.entries(rest).reduce( - (acc, [key, value]) => { - const values = uniq(splitCsvToList(value)); - return values.length === 0 - ? acc - : Object.assign(acc, { - [key]: uniq(splitCsvToList(value)).map((val) => ({ - value: key === 'email' ? val.toLowerCase() : val, - })), - }); - }, - {} as Record, - ); + const enrichedIdentifiers = Object.entries(rest).reduce< + Record + >((accumulator, [key, value]) => { + const values = uniq(splitCsvToList(value)); + return values.length === 0 + ? accumulator + : Object.assign(accumulator, { + [key]: uniq(splitCsvToList(value)).map((value_) => ({ + value: key === "email" ? value_.toLowerCase() : value_, + })), + }); + }, {}); // Make the GraphQL request try { await sombra - .post('v1/enrich-identifiers', { + .post("v1/enrich-identifiers", { headers: { - 'x-transcend-request-id': id, - 'x-transcend-enricher-id': enricherId, + "x-transcend-request-id": id, + "x-transcend-enricher-id": enricherId, }, json: { enrichedIdentifiers, @@ -71,19 +70,19 @@ export async function enrichPrivacyRequest( .json(); logger.error( - colors.green(`Successfully enriched request: ${ADMIN_URL}${id}`), + colors.green(`Successfully enriched request: ${ADMIN_URL}${id}`) ); return true; - } catch (err) { + } catch (error) { // skip if already enriched if ( - typeof err.response.body === 'string' && - err.response.body.includes('Cannot update a resolved RequestEnricher') + typeof error.response.body === "string" && + error.response.body.includes("Cannot update a resolved RequestEnricher") ) { logger.warn( colors.magenta( - `Skipped enrichment for request: ${ADMIN_URL}${id}, request is no longer in the enriching phase.`, - ), + `Skipped enrichment for request: ${ADMIN_URL}${id}, request is no longer in the enriching phase.` + ) ); return false; } @@ -91,9 +90,9 @@ export async function enrichPrivacyRequest( // error logger.error( colors.red( - `Failed to enricher identifiers for request with id: ${ADMIN_URL}${id} - ${err.message} - ${err.response.body}`, - ), + `Failed to enricher identifiers for request with id: ${ADMIN_URL}${id} - ${error.message} - ${error.response.body}` + ) ); - throw err; + throw error; } } diff --git a/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts b/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts index 37ff3f59..053e9184 100644 --- a/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts +++ b/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts @@ -1,10 +1,10 @@ -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { groupBy, uniq } from 'lodash-es'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; -import { writeCsv } from '../cron/writeCsv'; +import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { groupBy, uniq } from "lodash-es"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; +import { writeCsv } from "../cron/writeCsv"; import { buildTranscendGraphQLClient, createSombraGotInstance, @@ -14,7 +14,7 @@ import { PrivacyRequest, RequestEnricher, RequestIdentifier, -} from '../graphql'; +} from "../graphql"; export interface PrivacyRequestWithIdentifiers extends PrivacyRequest { /** Request Enrichers */ @@ -57,9 +57,9 @@ export async function pullManualEnrichmentIdentifiersToCsv({ logger.info( colors.magenta( `Pulling manual enrichment requests, filtered for actions: ${requestActions.join( - ',', - )}`, - ), + "," + )}` + ) ); // Pull all privacy requests @@ -82,7 +82,7 @@ export async function pullManualEnrichmentIdentifiersToCsv({ // Check if manual enrichment exists for that request const hasManualEnrichment = requestEnrichers.filter( - ({ status }) => status === 'ACTION_REQUIRED', + ({ status }) => status === "ACTION_REQUIRED" ); // Save request to queue @@ -92,7 +92,7 @@ export async function pullManualEnrichmentIdentifiersToCsv({ sombra, { requestId: request.id, - }, + } ); savedRequests.push({ ...request, @@ -103,7 +103,7 @@ export async function pullManualEnrichmentIdentifiersToCsv({ }, { concurrency, - }, + } ); const data = savedRequests.map( @@ -115,32 +115,28 @@ export async function pullManualEnrichmentIdentifiersToCsv({ }) => ({ ...request, // flatten identifiers - ...Object.entries(groupBy(requestIdentifiers, 'name')).reduce( - (acc, [key, values]) => - Object.assign(acc, { - [key]: values.map(({ value }) => value).join(','), - }), - {}, + ...Object.fromEntries( + Object.entries(groupBy(requestIdentifiers, "name")).map( + ([key, values]) => [key, values.map(({ value }) => value).join(",")] + ) ), // flatten attributes - ...Object.entries(groupBy(attributeValues, 'attributeKey.name')).reduce( - (acc, [key, values]) => - Object.assign(acc, { - [key]: values.map(({ name }) => name).join(','), - }), - {}, + ...Object.fromEntries( + Object.entries(groupBy(attributeValues, "attributeKey.name")).map( + ([key, values]) => [key, values.map(({ name }) => name).join(",")] + ) ), - }), + }) ); // Write out to CSV - const headers = uniq(data.map((d) => Object.keys(d)).flat()); + const headers = uniq(data.flatMap((d) => Object.keys(d))); writeCsv(file, data, headers); logger.info( colors.green( - `Successfully wrote ${savedRequests.length} requests to file "${file}"`, - ), + `Successfully wrote ${savedRequests.length} requests to file "${file}"` + ) ); return savedRequests; diff --git a/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts b/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts index 248be712..67720d5f 100644 --- a/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts +++ b/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts @@ -1,18 +1,18 @@ -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, createSombraGotInstance, makeGraphQLRequest, UPDATE_PRIVACY_REQUEST, -} from '../graphql'; -import { readCsv } from '../requests'; +} from "../graphql"; +import { readCsv } from "../requests"; import { enrichPrivacyRequest, EnrichPrivacyRequest, -} from './enrichPrivacyRequest'; +} from "./enrichPrivacyRequest"; /** * Push a CSV of enriched requests back into Transcend @@ -54,7 +54,7 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ // Notify Transcend logger.info( - colors.magenta(`Enriching "${activeResults.length}" privacy requests.`), + colors.magenta(`Enriching "${activeResults.length}" privacy requests.`) ); let successCount = 0; @@ -75,7 +75,7 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ }); logger.info( - colors.magenta(`Mark request as silent mode - ${request.id}`), + colors.magenta(`Mark request as silent mode - ${request.id}`) ); } @@ -83,24 +83,24 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ sombra, request, enricherId, - index, + index ); if (result) { successCount += 1; } else { skippedCount += 1; } - } catch (err) { + } catch { errorCount += 1; } }, - { concurrency }, + { concurrency } ); logger.info( colors.green( - `Successfully notified Transcend! \n Success count: ${successCount}.`, - ), + `Successfully notified Transcend! \n Success count: ${successCount}.` + ) ); if (skippedCount > 0) { diff --git a/src/lib/mergeTranscendInputs.ts b/src/lib/mergeTranscendInputs.ts index de6c04e4..71c2c8ee 100644 --- a/src/lib/mergeTranscendInputs.ts +++ b/src/lib/mergeTranscendInputs.ts @@ -1,5 +1,5 @@ -import { getEntries } from '@transcend-io/type-utils'; -import { TranscendInput } from '../codecs'; +import { getEntries } from "@transcend-io/type-utils"; +import { TranscendInput } from "../codecs"; /** * Combine a set of TranscendInput yaml files into a single yaml @@ -14,7 +14,7 @@ export function mergeTranscendInputs( ): TranscendInput { // eslint-disable-next-line @typescript-eslint/no-explicit-any const cloned: any = JSON.parse(JSON.stringify(base)); - inputs.forEach((input) => { + for (const input of inputs) { // eslint-disable-next-line @typescript-eslint/no-explicit-any getEntries(input).forEach(([key, value]: [any, any]) => { if (cloned[key] === undefined) { @@ -25,6 +25,6 @@ export function mergeTranscendInputs( cloned[key] = value; } }); - }); + } return cloned; } diff --git a/src/lib/oneTrust/helpers/convertToEmptyStrings.ts b/src/lib/oneTrust/helpers/convertToEmptyStrings.ts index 24757d82..5cbff11a 100644 --- a/src/lib/oneTrust/helpers/convertToEmptyStrings.ts +++ b/src/lib/oneTrust/helpers/convertToEmptyStrings.ts @@ -36,7 +36,7 @@ export function convertToEmptyStrings(input: T): any { // Handle null/undefined if (input === null || input === undefined) { - return ''; + return ""; } // Handle arrays @@ -45,16 +45,15 @@ export function convertToEmptyStrings(input: T): any { } // Handle objects - if (typeof input === 'object') { - return Object.entries(input).reduce( - (acc, [key, value]) => ({ - ...acc, - [key]: convertToEmptyStrings(value), - }), - {} as Record, + if (typeof input === "object") { + return Object.fromEntries( + Object.entries(input).map>(([key, value]) => [ + key, + convertToEmptyStrings(value), + ]) ); } // Handle primitives - return ''; + return ""; } diff --git a/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts b/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts index e8c343b8..9e7c38f4 100644 --- a/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts +++ b/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts @@ -1,11 +1,11 @@ -import colors from 'colors'; -import yargs from 'yargs-parser'; +import colors from "colors"; +import yargs from "yargs-parser"; import { OneTrustFileFormat, OneTrustPullResource, OneTrustPullSource, -} from '../../../enums'; -import { logger } from '../../../logger'; +} from "../../../enums"; +import { logger } from "../../../logger"; const VALID_RESOURCES = Object.values(OneTrustPullResource); @@ -48,21 +48,21 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { source, } = yargs(process.argv.slice(2), { string: [ - 'file', - 'hostname', - 'oneTrustAuth', - 'resource', - 'dryRun', - 'transcendAuth', - 'transcendUrl', - 'source', + "file", + "hostname", + "oneTrustAuth", + "resource", + "dryRun", + "transcendAuth", + "transcendUrl", + "source", ], - boolean: ['debug', 'dryRun'], + boolean: ["debug", "dryRun"], default: { resource: OneTrustPullResource.Assessments, debug: false, dryRun: false, - transcendUrl: 'https://api.transcend.io', + transcendUrl: "https://api.transcend.io", source: OneTrustPullSource.OneTrust, }, }); @@ -71,18 +71,16 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (!dryRun && !transcendAuth) { logger.error( colors.red( - // eslint-disable-next-line no-template-curly-in-string - 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}', - ), + 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}' + ) ); return process.exit(1); } if (!dryRun && !transcendUrl) { logger.error( colors.red( - // eslint-disable-next-line max-len - 'Must specify a "transcendUrl" parameter to sync resources to Transcend. e.g. --transcendUrl=https://api.transcend.io', - ), + 'Must specify a "transcendUrl" parameter to sync resources to Transcend. e.g. --transcendUrl=https://api.transcend.io' + ) ); return process.exit(1); } @@ -91,19 +89,19 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (dryRun && !file) { logger.error( colors.red( - 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json', - ), + 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json' + ) ); return process.exit(1); } if (file) { - const splitFile = file.split('.'); + const splitFile = file.split("."); if (splitFile.length < 2) { logger.error( colors.red( - 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.', - ), + 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.' + ) ); return process.exit(1); } @@ -112,8 +110,8 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { colors.red( `Expected the format of the "file" parameters '${file}' to be '${ OneTrustFileFormat.Json - }', but got '${splitFile.at(-1)}'.`, - ), + }', but got '${splitFile.at(-1)}'.` + ) ); return process.exit(1); } @@ -125,8 +123,8 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (!hostname) { logger.error( colors.red( - 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com', - ), + 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com' + ) ); return process.exit(1); } @@ -134,8 +132,8 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (!oneTrustAuth) { logger.error( colors.red( - 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN', - ), + 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN' + ) ); return process.exit(1); } @@ -144,8 +142,8 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (!file) { logger.error( colors.red( - 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json', - ), + 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json' + ) ); return process.exit(1); } @@ -154,9 +152,9 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (dryRun) { logger.error( colors.red( - 'Cannot read and write to a file simultaneously.' + - ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.`, - ), + "Cannot read and write to a file simultaneously." + + ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.` + ) ); return process.exit(1); } @@ -166,9 +164,9 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { logger.error( colors.red( `Received invalid resource value: "${resource}". Allowed: ${VALID_RESOURCES.join( - ',', - )}`, - ), + "," + )}` + ) ); return process.exit(1); } diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts index 89545f17..885e33d5 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts @@ -1,8 +1,8 @@ -import fs from 'fs'; -import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { logger } from '../../../logger'; -import { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson'; +import fs from "node:fs"; +import { OneTrustEnrichedAssessment } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { logger } from "../../../logger"; +import { oneTrustAssessmentToJson } from "./oneTrustAssessmentToJson"; /** * Write the assessment to disk at the specified file path. @@ -29,8 +29,8 @@ export const syncOneTrustAssessmentToDisk = ({ colors.magenta( `Writing enriched assessment ${ index + 1 - } of ${total} to file "${file}"...`, - ), + } of ${total} to file "${file}"...` + ) ); if (index === 0) { @@ -41,7 +41,7 @@ export const syncOneTrustAssessmentToDisk = ({ index, total, wrap: false, - }), + }) ); } else { fs.appendFileSync( @@ -51,7 +51,7 @@ export const syncOneTrustAssessmentToDisk = ({ index, total, wrap: false, - }), + }) ); } }; diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts index c0bc771b..80127ee8 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts @@ -1,13 +1,13 @@ -import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { ImportOnetrustAssessmentsInput } from '../../../codecs'; -import { logger } from '../../../logger'; +import { OneTrustEnrichedAssessment } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { ImportOnetrustAssessmentsInput } from "../../../codecs"; +import { logger } from "../../../logger"; import { IMPORT_ONE_TRUST_ASSESSMENT_FORMS, makeGraphQLRequest, -} from '../../graphql'; -import { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson'; +} from "../../graphql"; +import { oneTrustAssessmentToJson } from "./oneTrustAssessmentToJson"; export interface AssessmentForm { /** ID of Assessment Form */ @@ -40,9 +40,9 @@ export const syncOneTrustAssessmentToTranscend = async ({ logger.info( colors.magenta( `Writing enriched assessment ${index + 1} ${ - total ? `of ${total} ` : ' ' - }to Transcend...`, - ), + total ? `of ${total} ` : " " + }to Transcend...` + ) ); // convert the OneTrust assessment object into a json record @@ -67,14 +67,14 @@ export const syncOneTrustAssessmentToTranscend = async ({ }>(transcend, IMPORT_ONE_TRUST_ASSESSMENT_FORMS, { input, }); - } catch (e) { + } catch { logger.error( colors.red( `Failed to sync assessment ${index + 1} ${ - total ? `of ${total} ` : ' ' + total ? `of ${total} ` : " " }to Transcend.\n` + - `\tAssessment Title: ${assessment.name}. Template Title: ${assessment.template.name}\n`, - ), + `\tAssessment Title: ${assessment.name}. Template Title: ${assessment.template.name}\n` + ) ); } }; diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts index cad3b78e..3d9b69a4 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts @@ -1,11 +1,11 @@ -import { createReadStream } from 'fs'; -import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; -import { decodeCodec } from '@transcend-io/type-utils'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import JSONStream from 'JSONStream'; -import { logger } from '../../../logger'; -import { syncOneTrustAssessmentToTranscend } from './syncOneTrustAssessmentToTranscend'; +import { createReadStream } from "node:fs"; +import { OneTrustEnrichedAssessment } from "@transcend-io/privacy-types"; +import { decodeCodec } from "@transcend-io/type-utils"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import JSONStream from "JSONStream"; +import { logger } from "../../../logger"; +import { syncOneTrustAssessmentToTranscend } from "./syncOneTrustAssessmentToTranscend"; /** * Reads assessments from a file and syncs them to Transcend. @@ -26,12 +26,12 @@ export const syncOneTrustAssessmentsFromFile = ({ return new Promise((resolve, reject) => { // Create a readable stream from the file const fileStream = createReadStream(file, { - encoding: 'utf-8', + encoding: "utf-8", highWaterMark: 64 * 1024, // 64KB chunks }); // Create a JSONStream parser to parse the array of OneTrust assessments from the file - const parser = JSONStream.parse('*'); // '*' matches each element in the root array + const parser = JSONStream.parse("*"); // '*' matches each element in the root array let index = 0; @@ -39,7 +39,7 @@ export const syncOneTrustAssessmentsFromFile = ({ fileStream.pipe(parser); // Handle each parsed assessment object - parser.on('data', async (assessment) => { + parser.on("data", async (assessment) => { try { // Pause the stream while processing to avoid overwhelming memory parser.pause(); @@ -47,7 +47,7 @@ export const syncOneTrustAssessmentsFromFile = ({ // Decode and validate the assessment const parsedAssessment = decodeCodec( OneTrustEnrichedAssessment, - assessment, + assessment ); // Sync the assessment to transcend @@ -61,33 +61,33 @@ export const syncOneTrustAssessmentsFromFile = ({ // Resume the stream after processing parser.resume(); - } catch (e) { + } catch (error) { // if failed to parse a line, report error and continue logger.error( colors.red( - `Failed to parse the assessment ${index} from file '${file}': ${e.message}.`, - ), + `Failed to parse the assessment ${index} from file '${file}': ${error.message}.` + ) ); } }); // Handle completion - parser.on('end', () => { + parser.on("end", () => { logger.info(`Finished processing ${index} assessments from file ${file}`); resolve(); }); // Handle stream or parsing errors - parser.on('error', (error) => { + parser.on("error", (error) => { logger.error( - colors.red(`Error parsing file '${file}': ${error.message}`), + colors.red(`Error parsing file '${file}': ${error.message}`) ); reject(error); }); - fileStream.on('error', (error) => { + fileStream.on("error", (error) => { logger.error( - colors.red(`Error reading file '${file}': ${error.message}`), + colors.red(`Error reading file '${file}': ${error.message}`) ); reject(error); }); diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts index 071e4817..d33d0897 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts @@ -4,22 +4,22 @@ import { OneTrustEnrichedAssessment, OneTrustGetRiskResponse, OneTrustGetUserResponse, -} from '@transcend-io/privacy-types'; -import colors from 'colors'; -import type { Got } from 'got'; -import { GraphQLClient } from 'graphql-request'; -import { uniq } from 'lodash-es'; -import { logger } from '../../../logger'; -import { map, mapSeries } from '../../bluebird-replace'; +} from "@transcend-io/privacy-types"; +import colors from "colors"; +import type { Got } from "got"; +import { GraphQLClient } from "graphql-request"; +import { uniq } from "lodash-es"; +import { logger } from "../../../logger"; +import { map, mapSeries } from "../../bluebird-replace"; import { getListOfOneTrustAssessments, getOneTrustAssessment, getOneTrustRisk, getOneTrustUser, -} from '../endpoints'; -import { enrichOneTrustAssessment } from './enrichOneTrustAssessment'; -import { syncOneTrustAssessmentToDisk } from './syncOneTrustAssessmentToDisk'; -import { syncOneTrustAssessmentToTranscend } from './syncOneTrustAssessmentToTranscend'; +} from "../endpoints"; +import { enrichOneTrustAssessment } from "./enrichOneTrustAssessment"; +import { syncOneTrustAssessmentToDisk } from "./syncOneTrustAssessmentToDisk"; +import { syncOneTrustAssessmentToTranscend } from "./syncOneTrustAssessmentToTranscend"; export interface AssessmentForm { /** ID of Assessment Form */ @@ -49,7 +49,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ file?: string; }): Promise => { // fetch the list of all assessments in the OneTrust organization - logger.info('Getting list of all assessments from OneTrust...'); + logger.info("Getting list of all assessments from OneTrust..."); const assessments = await getListOfOneTrustAssessments({ oneTrust }); // a cache of OneTrust users so we avoid requesting already fetched users @@ -61,7 +61,8 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ { length: Math.ceil(assessments.length / BATCH_SIZE), }, - (_, i) => assessments.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE), + (_, index) => + assessments.slice(index * BATCH_SIZE, (index + 1) * BATCH_SIZE) ); // process each batch and sync the batch right away so it's garbage collected and we don't run out of memory @@ -74,7 +75,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ async (assessment, index) => { const assessmentNumber = BATCH_SIZE * batch + index + 1; logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching details...`, + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching details...` ); const { templateName, assessmentId } = assessment; const assessmentDetails = await getOneTrustAssessment({ @@ -86,7 +87,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ let creator = oneTrustCachedUsers[creatorId]; if (!creator) { logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching creator...`, + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching creator...` ); try { creator = await getOneTrustUser({ @@ -94,12 +95,12 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ userId: creatorId, }); oneTrustCachedUsers[creatorId] = creator; - } catch (e) { + } catch { logger.warn( colors.yellow( `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch form creator.` + - `\tcreatorId: ${creatorId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`, - ), + `\tcreatorId: ${creatorId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}` + ) ); } } @@ -109,7 +110,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ let approversDetails: OneTrustGetUserResponse[][] = []; if (approvers.length > 0) { logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching approvers...`, + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching approvers...` ); approversDetails = await map( approvers.map(({ id }) => id), @@ -121,17 +122,17 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ oneTrustCachedUsers[userId] = approver; } return [approver]; - } catch (e) { + } catch { logger.warn( colors.yellow( `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch a form approver.` + - `\tapproverId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`, - ), + `\tapproverId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}` + ) ); return []; } }, - { concurrency: 5 }, + { concurrency: 5 } ); } @@ -139,12 +140,12 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ const { respondents } = assessmentDetails; // if a user is an internal respondents, their 'name' field can't be an email. const internalRespondents = respondents.filter( - (r) => !r.name.includes('@'), + (r) => !r.name.includes("@") ); let respondentsDetails: OneTrustGetUserResponse[][] = []; if (internalRespondents.length > 0) { logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching respondents...`, + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching respondents...` ); respondentsDetails = await map( internalRespondents.map(({ id }) => id), @@ -156,17 +157,17 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ oneTrustCachedUsers[userId] = respondent; } return [respondent]; - } catch (e) { + } catch { logger.warn( colors.yellow( `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch a respondent.` + - `\trespondentId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`, - ), + `\trespondentId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}` + ) ); return []; } }, - { concurrency: 5 }, + { concurrency: 5 } ); } @@ -175,20 +176,20 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ const riskIds = uniq( assessmentDetails.sections.flatMap((s: OneTrustAssessmentSection) => s.questions.flatMap((q: OneTrustAssessmentQuestion) => - (q.risks ?? []).flatMap((r) => r.riskId), - ), - ), + (q.risks ?? []).flatMap((r) => r.riskId) + ) + ) ); if (riskIds.length > 0) { logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching risks...`, + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching risks...` ); riskDetails = await map( riskIds, - (riskId) => getOneTrustRisk({ oneTrust, riskId: riskId as string }), + (riskId) => getOneTrustRisk({ oneTrust, riskId: riskId }), { concurrency: 5, - }, + } ); } @@ -204,7 +205,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ batchEnrichedAssessments.push(enrichedAssessment); }, - { concurrency: BATCH_SIZE }, + { concurrency: BATCH_SIZE } ); // sync assessments in series to avoid concurrency bugs @@ -231,7 +232,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ index: globalIndex, }); } - }, + } ); }); }; diff --git a/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts b/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts index 0a633c09..91024c53 100644 --- a/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts +++ b/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts @@ -1,51 +1,51 @@ -import { describe, expect, it } from 'vitest'; -import { convertToEmptyStrings } from '../convertToEmptyStrings'; +import { describe, expect, it } from "vitest"; +import { convertToEmptyStrings } from "../convertToEmptyStrings"; -describe('buildDefaultCodecWrapper', () => { - it('should correctly build a default codec for null', () => { +describe("buildDefaultCodecWrapper", () => { + it("should correctly build a default codec for null", () => { const result = convertToEmptyStrings(null); - expect(result).to.equal(''); + expect(result).to.equal(""); }); - it('should correctly build a default codec for number', () => { + it("should correctly build a default codec for number", () => { const result = convertToEmptyStrings(0); - expect(result).to.equal(''); + expect(result).to.equal(""); }); - it('should correctly build a default codec for boolean', () => { + it("should correctly build a default codec for boolean", () => { const result = convertToEmptyStrings(false); - expect(result).to.equal(''); + expect(result).to.equal(""); }); - it('should correctly build a default codec for undefined', () => { - const result = convertToEmptyStrings(undefined); - expect(result).to.equal(''); + it("should correctly build a default codec for undefined", () => { + const result = convertToEmptyStrings(); + expect(result).to.equal(""); }); - it('should correctly build a default codec for string', () => { - const result = convertToEmptyStrings('1'); - expect(result).to.equal(''); + it("should correctly build a default codec for string", () => { + const result = convertToEmptyStrings("1"); + expect(result).to.equal(""); }); - it('should correctly build a default codec for an object', () => { - const result = convertToEmptyStrings({ name: 'joe' }); - expect(result).to.deep.equal({ name: '' }); + it("should correctly build a default codec for an object", () => { + const result = convertToEmptyStrings({ name: "joe" }); + expect(result).to.deep.equal({ name: "" }); }); - it('should correctly build a default codec for an array of primitive types', () => { - const result = convertToEmptyStrings(['name', 0, false]); - expect(result).to.deep.equal(['', '', '']); + it("should correctly build a default codec for an array of primitive types", () => { + const result = convertToEmptyStrings(["name", 0, false]); + expect(result).to.deep.equal(["", "", ""]); }); - it('should correctly build a default codec for an array of object', () => { + it("should correctly build a default codec for an array of object", () => { const result = convertToEmptyStrings([ - { name: 'john', age: 52 }, - { name: 'jane', age: 15, isAdult: true }, + { name: "john", age: 52 }, + { name: "jane", age: 15, isAdult: true }, ]); // should default to the array with object if the union contains an array of objects expect(result).to.deep.equal([ - { name: '', age: '' }, - { name: '', age: '', isAdult: '' }, + { name: "", age: "" }, + { name: "", age: "", isAdult: "" }, ]); }); }); diff --git a/src/lib/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts b/src/lib/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts index 17b5525c..3a822364 100644 --- a/src/lib/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts +++ b/src/lib/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts @@ -2,8 +2,8 @@ import { PreferenceQueryResponseItem, PreferenceStorePurposeResponse, PreferenceTopicType, -} from '@transcend-io/privacy-types'; -import { PreferenceTopic } from '../graphql'; +} from "@transcend-io/privacy-types"; +import { PreferenceTopic } from "../graphql"; /** * Check if the pending set of updates are exactly the same as the current consent record. @@ -19,9 +19,10 @@ export function checkIfPendingPreferenceUpdatesAreNoOp({ /** The current consent record */ currentConsentRecord: PreferenceQueryResponseItem; /** The pending updates */ - pendingUpdates: { - [purposeName in string]: Omit; - }; + pendingUpdates: Record< + string, + Omit + >; /** The preference topic configurations */ preferenceTopics: PreferenceTopic[]; }): boolean { @@ -30,7 +31,7 @@ export function checkIfPendingPreferenceUpdatesAreNoOp({ ([purposeName, { preferences = [], enabled }]) => { // Ensure the purpose exists const currentPurpose = currentConsentRecord.purposes.find( - (existingPurpose) => existingPurpose.purpose === purposeName, + (existingPurpose) => existingPurpose.purpose === purposeName ); // Ensure purpose.enabled is in sync @@ -42,52 +43,55 @@ export function checkIfPendingPreferenceUpdatesAreNoOp({ } // Compare the preferences are in sync - return preferences.every( - ({ topic, choice }) => - // ensure preferences exist on record - currentPurpose.preferences && - currentPurpose.preferences.find((existingPreference) => { - // find matching topic - if (existingPreference.topic !== topic) { - return false; - } + return preferences.every(({ topic, choice }) => + // ensure preferences exist on record + currentPurpose.preferences?.find((existingPreference) => { + // find matching topic + if (existingPreference.topic !== topic) { + return false; + } + + // Determine type of preference topic + const preferenceTopic = preferenceTopics.find( + (x) => x.slug === topic && x.purpose.trackingType === purposeName + ); + if (!preferenceTopic) { + throw new Error(`Could not find preference topic for ${topic}`); + } - // Determine type of preference topic - const preferenceTopic = preferenceTopics.find( - (x) => x.slug === topic && x.purpose.trackingType === purposeName, - ); - if (!preferenceTopic) { - throw new Error(`Could not find preference topic for ${topic}`); + // Handle comparison based on type + switch (preferenceTopic.type) { + case PreferenceTopicType.Boolean: { + return ( + existingPreference.choice.booleanValue === choice.booleanValue + ); } + case PreferenceTopicType.Select: { + return ( + existingPreference.choice.selectValue === choice.selectValue + ); + } + case PreferenceTopicType.MultiSelect: { + const sortedCurrentValues = ( + existingPreference.choice.selectValues || [] + ).sort(); - // Handle comparison based on type - switch (preferenceTopic.type) { - case PreferenceTopicType.Boolean: - return ( - existingPreference.choice.booleanValue === choice.booleanValue - ); - case PreferenceTopicType.Select: - return ( - existingPreference.choice.selectValue === choice.selectValue - ); - case PreferenceTopicType.MultiSelect: - // eslint-disable-next-line no-case-declarations - const sortedCurrentValues = ( - existingPreference.choice.selectValues || [] - ).sort(); - // eslint-disable-next-line no-case-declarations - const sortedNewValues = (choice.selectValues || []).sort(); - return ( - sortedCurrentValues.length === sortedNewValues.length && - sortedCurrentValues.every((x, i) => x === sortedNewValues[i]) - ); - default: - throw new Error( - `Unknown preference topic type: ${preferenceTopic.type}`, - ); + const sortedNewValues = (choice.selectValues || []).sort(); + return ( + sortedCurrentValues.length === sortedNewValues.length && + sortedCurrentValues.every( + (x, index) => x === sortedNewValues[index] + ) + ); + } + default: { + throw new Error( + `Unknown preference topic type: ${preferenceTopic.type}` + ); } - }), + } + }) ); - }, + } ); } diff --git a/src/lib/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts b/src/lib/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts index a14b4845..06bd39b7 100644 --- a/src/lib/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts +++ b/src/lib/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts @@ -2,8 +2,8 @@ import { PreferenceQueryResponseItem, PreferenceStorePurposeResponse, PreferenceTopicType, -} from '@transcend-io/privacy-types'; -import { PreferenceTopic } from '../graphql'; +} from "@transcend-io/privacy-types"; +import { PreferenceTopic } from "../graphql"; /** * Check if the pending set of updates will result in a change of @@ -20,9 +20,10 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ /** The current consent record */ currentConsentRecord: PreferenceQueryResponseItem; /** The pending updates */ - pendingUpdates: { - [purposeName in string]: Omit; - }; + pendingUpdates: Record< + string, + Omit + >; /** The preference topic configurations */ preferenceTopics: PreferenceTopic[]; }): boolean { @@ -31,7 +32,7 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ ([purposeName, { preferences = [], enabled }]) => { // Ensure the purpose exists const currentPurpose = currentConsentRecord.purposes.find( - (existingPurpose) => existingPurpose.purpose === purposeName, + (existingPurpose) => existingPurpose.purpose === purposeName ); // If no purpose exists, then it is not a conflict @@ -48,7 +49,7 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ return !!preferences.find(({ topic, choice }) => { // find matching topic const currentPreference = (currentPurpose.preferences || []).find( - (existingPreference) => existingPreference.topic === topic, + (existingPreference) => existingPreference.topic === topic ); // if no topic exists, no conflict @@ -58,7 +59,7 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ // Determine type of preference topic const preferenceTopic = preferenceTopics.find( - (x) => x.slug === topic && x.purpose.trackingType === purposeName, + (x) => x.slug === topic && x.purpose.trackingType === purposeName ); if (!preferenceTopic) { throw new Error(`Could not find preference topic for ${topic}`); @@ -66,29 +67,34 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ // Handle comparison based on type switch (preferenceTopic.type) { - case PreferenceTopicType.Boolean: + case PreferenceTopicType.Boolean: { return ( currentPreference.choice.booleanValue !== choice.booleanValue ); - case PreferenceTopicType.Select: + } + case PreferenceTopicType.Select: { return currentPreference.choice.selectValue !== choice.selectValue; - case PreferenceTopicType.MultiSelect: - // eslint-disable-next-line no-case-declarations + } + case PreferenceTopicType.MultiSelect: { const sortedCurrentValues = ( currentPreference.choice.selectValues || [] ).sort(); - // eslint-disable-next-line no-case-declarations + const sortedNewValues = (choice.selectValues || []).sort(); return ( sortedCurrentValues.length !== sortedNewValues.length || - !sortedCurrentValues.every((x, i) => x === sortedNewValues[i]) + !sortedCurrentValues.every( + (x, index) => x === sortedNewValues[index] + ) ); - default: + } + default: { throw new Error( - `Unknown preference topic type: ${preferenceTopic.type}`, + `Unknown preference topic type: ${preferenceTopic.type}` ); + } } }); - }, + } ); } diff --git a/src/lib/preference-management/getPreferenceUpdatesFromRow.ts b/src/lib/preference-management/getPreferenceUpdatesFromRow.ts index cabbd8cd..7d9a5801 100644 --- a/src/lib/preference-management/getPreferenceUpdatesFromRow.ts +++ b/src/lib/preference-management/getPreferenceUpdatesFromRow.ts @@ -1,11 +1,11 @@ import { PreferenceStorePurposeResponse, PreferenceTopicType, -} from '@transcend-io/privacy-types'; -import { apply } from '@transcend-io/type-utils'; -import { PreferenceTopic } from '../graphql'; -import { splitCsvToList } from '../requests'; -import { PurposeRowMapping } from './codecs'; +} from "@transcend-io/privacy-types"; +import { apply } from "@transcend-io/type-utils"; +import { PreferenceTopic } from "../graphql"; +import { splitCsvToList } from "../requests"; +import { PurposeRowMapping } from "./codecs"; /** * Parse an arbitrary object to the Transcend PUT /v1/preference update shape @@ -42,155 +42,154 @@ export function getPreferenceUpdatesFromRow({ purposeSlugs: string[]; /** The preference topics */ preferenceTopics: PreferenceTopic[]; -}): { - [k in string]: Omit; -} { +}): Record> { // Create a result object to store the parsed preferences - const result: { - [k in string]: Partial; - } = {}; + const result: Record> = {}; // Iterate over each column and map to the purpose or preference - Object.entries(columnToPurposeName).forEach( - ([columnName, { purpose, preference, valueMapping }]) => { - // Ensure the purpose is valid - if (!purposeSlugs.includes(purpose)) { + for (const [ + columnName, + { purpose, preference, valueMapping }, + ] of Object.entries(columnToPurposeName)) { + // Ensure the purpose is valid + if (!purposeSlugs.includes(purpose)) { + throw new Error( + `Invalid purpose slug: ${purpose}, expected: ${purposeSlugs.join(", ")}` + ); + } + + // CHeck if parsing a preference or just the top level purpose + if (preference) { + const preferenceTopic = preferenceTopics.find( + (x) => x.slug === preference && x.purpose.trackingType === purpose + ); + if (!preferenceTopic) { + const allowedTopics = preferenceTopics + .filter((x) => x.purpose.trackingType === purpose) + .map((x) => x.slug); throw new Error( - `Invalid purpose slug: ${purpose}, expected: ${purposeSlugs.join( - ', ', - )}`, + `Invalid preference slug: ${preference} for purpose: ${purpose}. ` + + `Allowed preference slugs for purpose are: ${allowedTopics.join( + "," + )}` ); } - // CHeck if parsing a preference or just the top level purpose - if (preference) { - const preferenceTopic = preferenceTopics.find( - (x) => x.slug === preference && x.purpose.trackingType === purpose, - ); - if (!preferenceTopic) { - const allowedTopics = preferenceTopics - .filter((x) => x.purpose.trackingType === purpose) - .map((x) => x.slug); - throw new Error( - `Invalid preference slug: ${preference} for purpose: ${purpose}. ` + - `Allowed preference slugs for purpose are: ${allowedTopics.join( - ',', - )}`, - ); - } - - // If parsing preferences, default to an empty array - if (!result[purpose]) { - result[purpose] = { - preferences: [], - }; - } - if (!result[purpose].preferences) { - result[purpose].preferences = []; - } + // If parsing preferences, default to an empty array + if (!result[purpose]) { + result[purpose] = { + preferences: [], + }; + } + if (!result[purpose].preferences) { + result[purpose].preferences = []; + } - // The value to parse - const rawValue = row[columnName]; - const rawMapping = valueMapping[rawValue]; - const trimmedMapping = - typeof rawMapping === 'string' ? rawMapping.trim() || null : null; + // The value to parse + const rawValue = row[columnName]; + const rawMapping = valueMapping[rawValue]; + const trimmedMapping = + typeof rawMapping === "string" ? rawMapping.trim() || null : null; - // handle each type of preference - switch (preferenceTopic.type) { - case PreferenceTopicType.Boolean: - if (typeof rawMapping !== 'boolean') { - throw new Error( - `Invalid value for boolean preference: ${preference}, expected boolean, got: ${rawValue}`, - ); - } - result[purpose].preferences!.push({ - topic: preference, - choice: { - booleanValue: rawMapping, - }, - }); - break; - case PreferenceTopicType.Select: - if (typeof rawMapping !== 'string' && rawMapping !== null) { - throw new Error( - `Invalid value for select preference: ${preference}, expected string or null, got: ${rawValue}`, - ); - } + // handle each type of preference + switch (preferenceTopic.type) { + case PreferenceTopicType.Boolean: { + if (typeof rawMapping !== "boolean") { + throw new TypeError( + `Invalid value for boolean preference: ${preference}, expected boolean, got: ${rawValue}` + ); + } + result[purpose].preferences.push({ + topic: preference, + choice: { + booleanValue: rawMapping, + }, + }); + break; + } + case PreferenceTopicType.Select: { + if (typeof rawMapping !== "string" && rawMapping !== null) { + throw new Error( + `Invalid value for select preference: ${preference}, expected string or null, got: ${rawValue}` + ); + } - if ( - trimmedMapping && - !preferenceTopic.preferenceOptionValues - .map(({ slug }) => slug) - .includes(trimmedMapping) - ) { - throw new Error( - `Invalid value for select preference: ${preference}, expected one of: ` + - `${preferenceTopic.preferenceOptionValues - .map(({ slug }) => slug) - .join(', ')}, got: ${rawValue}`, - ); - } + if ( + trimmedMapping && + !preferenceTopic.preferenceOptionValues + .map(({ slug }) => slug) + .includes(trimmedMapping) + ) { + throw new Error( + `Invalid value for select preference: ${preference}, expected one of: ` + + `${preferenceTopic.preferenceOptionValues + .map(({ slug }) => slug) + .join(", ")}, got: ${rawValue}` + ); + } - // Update preferences - result[purpose].preferences!.push({ - topic: preference, - choice: { - selectValue: trimmedMapping, - }, - }); - break; - case PreferenceTopicType.MultiSelect: - if (typeof rawValue !== 'string') { - throw new Error( - `Invalid value for multi select preference: ${preference}, expected string, got: ${rawValue}`, - ); - } - // Update preferences - result[purpose].preferences!.push({ - topic: preference, - choice: { - selectValues: splitCsvToList(rawValue) - .map((val) => { - const result = valueMapping[val]; - if (typeof result !== 'string') { - throw new Error( - `Invalid value for multi select preference: ${preference}, ` + - `expected one of: ${preferenceTopic.preferenceOptionValues - .map(({ slug }) => slug) - .join(', ')}, got: ${val}`, - ); - } - return result; - }) - .sort((a, b) => a.localeCompare(b)), - }, - }); - break; - default: - throw new Error(`Unknown preference type: ${preferenceTopic.type}`); + // Update preferences + result[purpose].preferences.push({ + topic: preference, + choice: { + selectValue: trimmedMapping, + }, + }); + break; + } + case PreferenceTopicType.MultiSelect: { + if (typeof rawValue !== "string") { + throw new TypeError( + `Invalid value for multi select preference: ${preference}, expected string, got: ${rawValue}` + ); + } + // Update preferences + result[purpose].preferences.push({ + topic: preference, + choice: { + selectValues: splitCsvToList(rawValue) + .map((value) => { + const result = valueMapping[value]; + if (typeof result !== "string") { + throw new TypeError( + `Invalid value for multi select preference: ${preference}, ` + + `expected one of: ${preferenceTopic.preferenceOptionValues + .map(({ slug }) => slug) + .join(", ")}, got: ${value}` + ); + } + return result; + }) + .sort((a, b) => a.localeCompare(b)), + }, + }); + break; + } + default: { + throw new Error(`Unknown preference type: ${preferenceTopic.type}`); } - } else if (!result[purpose]) { - // Handle updating top level purpose for the first time - result[purpose] = { - enabled: valueMapping[row[columnName]] === true, - }; - } else { - // Handle updating top level purpose but preserve preference updates - result[purpose].enabled = valueMapping[row[columnName]] === true; } - }, - ); + } else if (result[purpose]) { + // Handle updating top level purpose but preserve preference updates + result[purpose].enabled = valueMapping[row[columnName]] === true; + } else { + // Handle updating top level purpose for the first time + result[purpose] = { + enabled: valueMapping[row[columnName]] === true, + }; + } + } // Ensure that enabled is provided return apply(result, (x, purposeName) => { - if (typeof x.enabled !== 'boolean') { - throw new Error( - `No mapping provided for purpose.enabled=true/false value: ${purposeName}`, + if (typeof x.enabled !== "boolean") { + throw new TypeError( + `No mapping provided for purpose.enabled=true/false value: ${purposeName}` ); } return { ...x, - enabled: x.enabled!, + enabled: x.enabled, }; }); } diff --git a/src/lib/preference-management/getPreferencesForIdentifiers.ts b/src/lib/preference-management/getPreferencesForIdentifiers.ts index 10bf46f8..3f523344 100644 --- a/src/lib/preference-management/getPreferencesForIdentifiers.ts +++ b/src/lib/preference-management/getPreferencesForIdentifiers.ts @@ -1,12 +1,12 @@ -import { PreferenceQueryResponseItem } from '@transcend-io/privacy-types'; -import { decodeCodec } from '@transcend-io/type-utils'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import type { Got } from 'got'; -import * as t from 'io-ts'; -import { chunk } from 'lodash-es'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { PreferenceQueryResponseItem } from "@transcend-io/privacy-types"; +import { decodeCodec } from "@transcend-io/type-utils"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import type { Got } from "got"; +import * as t from "io-ts"; +import { chunk } from "lodash-es"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; const PreferenceRecordsQueryResponse = t.intersection([ t.type({ @@ -19,10 +19,10 @@ const PreferenceRecordsQueryResponse = t.intersection([ ]); const MSGS = [ - 'ENOTFOUND', - 'ETIMEDOUT', - '504 Gateway Time-out', - 'Task timed out after', + "ENOTFOUND", + "ETIMEDOUT", + "504 Gateway Time-out", + "Task timed out after", ]; /** @@ -48,16 +48,16 @@ export async function getPreferencesForIdentifiers( partitionKey: string; /** Whether to skip logging */ skipLogging?: boolean; - }, + } ): Promise { const results: PreferenceQueryResponseItem[] = []; const groupedIdentifiers = chunk(identifiers, 100); // create a new progress bar instance and use shades_classic theme - const t0 = new Date().getTime(); + const t0 = Date.now(); const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); if (!skipLogging) { progressBar.start(identifiers.length, 0); @@ -88,40 +88,40 @@ export async function getPreferencesForIdentifiers( total += group.length; progressBar.update(total); break; // Exit loop if successful - } catch (err) { + } catch (error) { attempts += 1; - const msg = err?.response?.body || err?.message || ''; + const message = error?.response?.body || error?.message || ""; if ( attempts >= maxAttempts || - !MSGS.some((errorMessage) => msg.includes(errorMessage)) + !MSGS.some((errorMessage) => message.includes(errorMessage)) ) { throw new Error( - `Received an error from server after ${attempts} attempts: ${msg}`, + `Received an error from server after ${attempts} attempts: ${message}` ); } logger.warn( colors.yellow( `[RETRYING FAILED REQUEST - Attempt ${attempts}] ` + - `Failed to fetch ${group.length} user preferences from partition ${partitionKey}: ${msg}`, - ), + `Failed to fetch ${group.length} user preferences from partition ${partitionKey}: ${message}` + ) ); } } }, { concurrency: 40, - }, + } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; if (!skipLogging) { // Log completion time logger.info( - colors.green(`Completed download in "${totalTime / 1000}" seconds.`), + colors.green(`Completed download in "${totalTime / 1000}" seconds.`) ); } diff --git a/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts b/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts index fe4b6322..bd7240f1 100644 --- a/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts @@ -1,14 +1,12 @@ -import { PreferenceTopicType } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import inquirer from 'inquirer'; -import { difference, uniq } from 'lodash-es'; -import { logger } from '../../logger'; -import { mapSeries } from '../bluebird-replace'; -import { PreferenceTopic } from '../graphql'; -import { splitCsvToList } from '../requests'; -import { FileMetadataState } from './codecs'; - -/* eslint-disable no-param-reassign */ +import { PreferenceTopicType } from "@transcend-io/privacy-types"; +import colors from "colors"; +import inquirer from "inquirer"; +import { difference, uniq } from "lodash-es"; +import { logger } from "../../logger"; +import { mapSeries } from "../bluebird-replace"; +import { PreferenceTopic } from "../graphql"; +import { splitCsvToList } from "../requests"; +import { FileMetadataState } from "./codecs"; /** * Parse out the purpose.enabled and preference values from a CSV file @@ -32,10 +30,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( preferenceTopics: PreferenceTopic[]; /** Force workflow triggers */ forceTriggerWorkflows: boolean; - }, + } ): Promise { // Determine columns to map - const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()); + const columnNames = uniq(preferences.flatMap((x) => Object.keys(x))); // Determine the columns that could potentially be used for identifier const otherColumns = difference(columnNames, [ @@ -46,7 +44,7 @@ export async function parsePreferenceAndPurposeValuesFromCsv( if (forceTriggerWorkflows) { return currentState; } - throw new Error('No other columns to process'); + throw new Error("No other columns to process"); } // The purpose and preferences to map to @@ -65,8 +63,8 @@ export async function parsePreferenceAndPurposeValuesFromCsv( if (purposeMapping) { logger.info( colors.magenta( - `Column "${col}" is associated with purpose "${purposeMapping.purpose}"`, - ), + `Column "${col}" is associated with purpose "${purposeMapping.purpose}"` + ) ); } else { const { purposeName } = await inquirer.prompt<{ @@ -74,14 +72,14 @@ export async function parsePreferenceAndPurposeValuesFromCsv( purposeName: string; }>([ { - name: 'purposeName', + name: "purposeName", message: `Choose the purpose that column ${col} is associated with`, - type: 'list', + type: "list", default: purposeNames.find((x) => x.startsWith(purposeSlugs[0])), choices: purposeNames, }, ]); - const [purposeSlug, preferenceSlug] = purposeName.split('->'); + const [purposeSlug, preferenceSlug] = purposeName.split("->"); purposeMapping = { purpose: purposeSlug, preference: preferenceSlug || null, @@ -94,8 +92,8 @@ export async function parsePreferenceAndPurposeValuesFromCsv( if (purposeMapping.valueMapping[value] !== undefined) { logger.info( colors.magenta( - `Value "${value}" is associated with purpose value "${purposeMapping.valueMapping[value]}"`, - ), + `Value "${value}" is associated with purpose value "${purposeMapping.valueMapping[value]}"` + ) ); return; } @@ -106,10 +104,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( purposeValue: boolean; }>([ { - name: 'purposeValue', + name: "purposeValue", message: `Choose the purpose value for value "${value}" associated with purpose "${purposeMapping.purpose}"`, - type: 'confirm', - default: value !== 'false', + type: "confirm", + default: value !== "false", }, ]); purposeMapping.valueMapping[value] = purposeValue; @@ -118,18 +116,18 @@ export async function parsePreferenceAndPurposeValuesFromCsv( // if preference is not null, this column is for a specific preference if (purposeMapping.preference !== null) { const preferenceTopic = preferenceTopics.find( - (x) => x.slug === purposeMapping.preference, + (x) => x.slug === purposeMapping.preference ); if (!preferenceTopic) { logger.error( colors.red( - `Preference topic "${purposeMapping.preference}" not found`, - ), + `Preference topic "${purposeMapping.preference}" not found` + ) ); return; } const preferenceOptions = preferenceTopic.preferenceOptionValues.map( - ({ slug }) => slug, + ({ slug }) => slug ); if (preferenceTopic.type === PreferenceTopicType.Boolean) { @@ -138,12 +136,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( preferenceValue: boolean; }>([ { - name: 'preferenceValue', - message: - // eslint-disable-next-line max-len - `Choose the preference value for "${preferenceTopic.slug}" value "${value}" associated with purpose "${purposeMapping.purpose}"`, - type: 'confirm', - default: value !== 'false', + name: "preferenceValue", + message: `Choose the preference value for "${preferenceTopic.slug}" value "${value}" associated with purpose "${purposeMapping.purpose}"`, + type: "confirm", + default: value !== "false", }, ]); purposeMapping.valueMapping[value] = preferenceValue; @@ -156,10 +152,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( preferenceValue: boolean; }>([ { - name: 'preferenceValue', - // eslint-disable-next-line max-len + name: "preferenceValue", + message: `Choose the preference value for "${preferenceTopic.slug}" value "${value}" associated with purpose "${purposeMapping.purpose}"`, - type: 'list', + type: "list", choices: preferenceOptions, default: preferenceOptions.find((x) => x === value), }, @@ -181,10 +177,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( preferenceValue: boolean; }>([ { - name: 'preferenceValue', - // eslint-disable-next-line max-len + name: "preferenceValue", + message: `Choose the preference value for "${preferenceTopic.slug}" value "${parsedValue}" associated with purpose "${purposeMapping.purpose}"`, - type: 'list', + type: "list", choices: preferenceOptions, default: preferenceOptions.find((x) => x === parsedValue), }, @@ -195,7 +191,7 @@ export async function parsePreferenceAndPurposeValuesFromCsv( } throw new Error( - `Unknown preference topic type: ${preferenceTopic.type}`, + `Unknown preference topic type: ${preferenceTopic.type}` ); } }); @@ -205,4 +201,3 @@ export async function parsePreferenceAndPurposeValuesFromCsv( return currentState; } -/* eslint-enable no-param-reassign */ diff --git a/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts b/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts index 5aea479b..3487eedc 100644 --- a/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts @@ -1,11 +1,9 @@ -import colors from 'colors'; -import inquirer from 'inquirer'; -import { difference, groupBy, uniq } from 'lodash-es'; -import { logger } from '../../logger'; -import { inquirerConfirmBoolean } from '../helpers'; -import { FileMetadataState } from './codecs'; - -/* eslint-disable no-param-reassign */ +import colors from "colors"; +import inquirer from "inquirer"; +import { difference, groupBy, uniq } from "lodash-es"; +import { logger } from "../../logger"; +import { inquirerConfirmBoolean } from "../helpers"; +import { FileMetadataState } from "./codecs"; /** * Parse identifiers from a CSV list of preferences @@ -19,7 +17,7 @@ import { FileMetadataState } from './codecs'; */ export async function parsePreferenceIdentifiersFromCsv( preferences: Record[], - currentState: FileMetadataState, + currentState: FileMetadataState ): Promise<{ /** The updated state */ currentState: FileMetadataState; @@ -27,7 +25,7 @@ export async function parsePreferenceIdentifiersFromCsv( preferences: Record[]; }> { // Determine columns to map - const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()); + const columnNames = uniq(preferences.flatMap((x) => Object.keys(x))); // Determine the columns that could potentially be used for identifier const remainingColumnsForIdentifier = difference(columnNames, [ @@ -42,13 +40,13 @@ export async function parsePreferenceIdentifiersFromCsv( identifierName: string; }>([ { - name: 'identifierName', + name: "identifierName", message: - 'Choose the column that will be used as the identifier to upload consent preferences by', - type: 'list', + "Choose the column that will be used as the identifier to upload consent preferences by", + type: "list", default: remainingColumnsForIdentifier.find((col) => - col.toLowerCase().includes('email'), + col.toLowerCase().includes("email") ) || remainingColumnsForIdentifier[0], choices: remainingColumnsForIdentifier, }, @@ -56,9 +54,7 @@ export async function parsePreferenceIdentifiersFromCsv( currentState.identifierColumn = identifierName; } logger.info( - colors.magenta( - `Using identifier column "${currentState.identifierColumn}"`, - ), + colors.magenta(`Using identifier column "${currentState.identifierColumn}"`) ); // Validate that the identifier column is present for all rows and unique @@ -67,72 +63,71 @@ export async function parsePreferenceIdentifiersFromCsv( .filter((x): x is number[] => !!x) .flat(); if (identifierColumnsMissing.length > 0) { - const msg = `The identifier column "${ + const message = `The identifier column "${ currentState.identifierColumn }" is missing a value for the following rows: ${identifierColumnsMissing.join( - ', ', + ", " )}`; - logger.warn(colors.yellow(msg)); + logger.warn(colors.yellow(message)); // Ask user if they would like to skip rows missing an identifier const skip = await inquirerConfirmBoolean({ - message: 'Would you like to skip rows missing an identifier?', + message: "Would you like to skip rows missing an identifier?", }); if (!skip) { - throw new Error(msg); + throw new Error(message); } // Filter out rows missing an identifier const previous = preferences.length; preferences = preferences.filter( - (pref) => pref[currentState.identifierColumn!], + (pref) => pref[currentState.identifierColumn!] ); logger.info( colors.yellow( - `Skipped ${previous - preferences.length} rows missing an identifier`, - ), + `Skipped ${previous - preferences.length} rows missing an identifier` + ) ); } logger.info( colors.magenta( - `The identifier column "${currentState.identifierColumn}" is present for all rows`, - ), + `The identifier column "${currentState.identifierColumn}" is present for all rows` + ) ); // Validate that all identifiers are unique const rowsByUserId = groupBy(preferences, currentState.identifierColumn); const duplicateIdentifiers = Object.entries(rowsByUserId).filter( - ([, rows]) => rows.length > 1, + ([, rows]) => rows.length > 1 ); if (duplicateIdentifiers.length > 0) { - const msg = `The identifier column "${ + const message = `The identifier column "${ currentState.identifierColumn }" has duplicate values for the following rows: ${duplicateIdentifiers .slice(0, 10) .map(([userId, rows]) => `${userId} (${rows.length})`) - .join('\n')}`; - logger.warn(colors.yellow(msg)); + .join("\n")}`; + logger.warn(colors.yellow(message)); // Ask user if they would like to take the most recent update // for each duplicate identifier const skip = await inquirerConfirmBoolean({ - message: 'Would you like to automatically take the latest update?', + message: "Would you like to automatically take the latest update?", }); if (!skip) { - throw new Error(msg); + throw new Error(message); } preferences = Object.entries(rowsByUserId) .map(([, rows]) => { const sorted = rows.sort( (a, b) => new Date(b[currentState.timestampColum!]).getTime() - - new Date(a[currentState.timestampColum!]).getTime(), + new Date(a[currentState.timestampColum!]).getTime() ); return sorted[0]; }) - .filter((x) => x); + .filter(Boolean); } return { currentState, preferences }; } -/* eslint-enable no-param-reassign */ diff --git a/src/lib/preference-management/parsePreferenceManagementCsv.ts b/src/lib/preference-management/parsePreferenceManagementCsv.ts index b2db315d..13415db4 100644 --- a/src/lib/preference-management/parsePreferenceManagementCsv.ts +++ b/src/lib/preference-management/parsePreferenceManagementCsv.ts @@ -1,19 +1,19 @@ -import { PersistedState } from '@transcend-io/persisted-state'; -import colors from 'colors'; -import type { Got } from 'got'; -import * as t from 'io-ts'; -import { keyBy } from 'lodash-es'; -import { logger } from '../../logger'; -import { PreferenceTopic } from '../graphql'; -import { readCsv } from '../requests'; -import { checkIfPendingPreferenceUpdatesAreNoOp } from './checkIfPendingPreferenceUpdatesAreNoOp'; -import { checkIfPendingPreferenceUpdatesCauseConflict } from './checkIfPendingPreferenceUpdatesCauseConflict'; -import { FileMetadataState, PreferenceState } from './codecs'; -import { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers'; -import { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow'; -import { parsePreferenceAndPurposeValuesFromCsv } from './parsePreferenceAndPurposeValuesFromCsv'; -import { parsePreferenceIdentifiersFromCsv } from './parsePreferenceIdentifiersFromCsv'; -import { parsePreferenceTimestampsFromCsv } from './parsePreferenceTimestampsFromCsv'; +import { PersistedState } from "@transcend-io/persisted-state"; +import colors from "colors"; +import type { Got } from "got"; +import * as t from "io-ts"; +import { keyBy } from "lodash-es"; +import { logger } from "../../logger"; +import { PreferenceTopic } from "../graphql"; +import { readCsv } from "../requests"; +import { checkIfPendingPreferenceUpdatesAreNoOp } from "./checkIfPendingPreferenceUpdatesAreNoOp"; +import { checkIfPendingPreferenceUpdatesCauseConflict } from "./checkIfPendingPreferenceUpdatesCauseConflict"; +import { FileMetadataState, PreferenceState } from "./codecs"; +import { getPreferencesForIdentifiers } from "./getPreferencesForIdentifiers"; +import { getPreferenceUpdatesFromRow } from "./getPreferenceUpdatesFromRow"; +import { parsePreferenceAndPurposeValuesFromCsv } from "./parsePreferenceAndPurposeValuesFromCsv"; +import { parsePreferenceIdentifiersFromCsv } from "./parsePreferenceIdentifiersFromCsv"; +import { parsePreferenceTimestampsFromCsv } from "./parsePreferenceTimestampsFromCsv"; /** * Parse a file into the cache @@ -48,13 +48,13 @@ export async function parsePreferenceManagementCsvWithCache( /** Wheather to force workflow triggers */ forceTriggerWorkflows: boolean; }, - cache: PersistedState, + cache: PersistedState ): Promise { // Start the timer - const t0 = new Date().getTime(); + const t0 = Date.now(); // Get the current metadata - const fileMetadata = cache.getValue('fileMetadata'); + const fileMetadata = cache.getValue("fileMetadata"); // Read in the file logger.info(colors.magenta(`Reading in file: "${file}"`)); @@ -74,20 +74,20 @@ export async function parsePreferenceManagementCsvWithCache( // Validate that all timestamps are present in the file currentState = await parsePreferenceTimestampsFromCsv( preferences, - currentState, + currentState ); fileMetadata[file] = currentState; - await cache.setValue(fileMetadata, 'fileMetadata'); + await cache.setValue(fileMetadata, "fileMetadata"); // Validate that all identifiers are present and unique const result = await parsePreferenceIdentifiersFromCsv( preferences, - currentState, + currentState ); currentState = result.currentState; preferences = result.preferences; fileMetadata[file] = currentState; - await cache.setValue(fileMetadata, 'fileMetadata'); + await cache.setValue(fileMetadata, "fileMetadata"); // Ensure all other columns are mapped to purpose and preference // slug values @@ -98,14 +98,14 @@ export async function parsePreferenceManagementCsvWithCache( preferenceTopics, purposeSlugs, forceTriggerWorkflows, - }, + } ); fileMetadata[file] = currentState; - await cache.setValue(fileMetadata, 'fileMetadata'); + await cache.setValue(fileMetadata, "fileMetadata"); // Grab existing preference store records const identifiers = preferences.map( - (pref) => pref[currentState.identifierColumn!], + (pref) => pref[currentState.identifierColumn!] ); const existingConsentRecords = skipExistingRecordCheck ? [] @@ -113,7 +113,7 @@ export async function parsePreferenceManagementCsvWithCache( identifiers: identifiers.map((x) => ({ value: x })), partitionKey, }); - const consentRecordByIdentifier = keyBy(existingConsentRecords, 'userId'); + const consentRecordByIdentifier = keyBy(existingConsentRecords, "userId"); // Clear out previous updates currentState.pendingConflictUpdates = {}; @@ -121,7 +121,7 @@ export async function parsePreferenceManagementCsvWithCache( currentState.skippedUpdates = {}; // Process each row - preferences.forEach((pref) => { + for (const pref of preferences) { // Grab unique Id for the user const userId = pref[currentState.identifierColumn!]; @@ -138,7 +138,7 @@ export async function parsePreferenceManagementCsvWithCache( if (forceTriggerWorkflows && !currentConsentRecord) { throw new Error( `No existing consent record found for user with id: ${userId}. - When 'forceTriggerWorkflows' is set all the user identifiers should contain a consent record`, + When 'forceTriggerWorkflows' is set all the user identifiers should contain a consent record` ); } // Check if the update can be skipped @@ -154,7 +154,7 @@ export async function parsePreferenceManagementCsvWithCache( !forceTriggerWorkflows ) { currentState.skippedUpdates[userId] = pref; - return; + continue; } // Determine if there are any conflicts @@ -170,20 +170,20 @@ export async function parsePreferenceManagementCsvWithCache( row: pref, record: currentConsentRecord, }; - return; + continue; } // Add to pending updates currentState.pendingSafeUpdates[userId] = pref; - }); + } // Read in the file fileMetadata[file] = currentState; - await cache.setValue(fileMetadata, 'fileMetadata'); - const t1 = new Date().getTime(); + await cache.setValue(fileMetadata, "fileMetadata"); + const t1 = Date.now(); logger.info( colors.green( - `Successfully pre-processed file: "${file}" in ${(t1 - t0) / 1000}s`, - ), + `Successfully pre-processed file: "${file}" in ${(t1 - t0) / 1000}s` + ) ); } diff --git a/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts b/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts index cfe5db8c..e59ca765 100644 --- a/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts @@ -1,12 +1,10 @@ -import colors from 'colors'; -import inquirer from 'inquirer'; -import { difference, uniq } from 'lodash-es'; -import { logger } from '../../logger'; -import { FileMetadataState } from './codecs'; +import colors from "colors"; +import inquirer from "inquirer"; +import { difference, uniq } from "lodash-es"; +import { logger } from "../../logger"; +import { FileMetadataState } from "./codecs"; -export const NONE_PREFERENCE_MAP = '[NONE]'; - -/* eslint-disable no-param-reassign */ +export const NONE_PREFERENCE_MAP = "[NONE]"; /** * Parse timestamps from a CSV list of preferences @@ -22,10 +20,10 @@ export const NONE_PREFERENCE_MAP = '[NONE]'; */ export async function parsePreferenceTimestampsFromCsv( preferences: Record[], - currentState: FileMetadataState, + currentState: FileMetadataState ): Promise { // Determine columns to map - const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat()); + const columnNames = uniq(preferences.flatMap((x) => Object.keys(x))); // Determine the columns that could potentially be used for timestamp const remainingColumnsForTimestamp = difference(columnNames, [ @@ -40,16 +38,16 @@ export async function parsePreferenceTimestampsFromCsv( timestampName: string; }>([ { - name: 'timestampName', + name: "timestampName", message: - 'Choose the column that will be used as the timestamp of last preference update', - type: 'list', + "Choose the column that will be used as the timestamp of last preference update", + type: "list", default: remainingColumnsForTimestamp.find((col) => - col.toLowerCase().includes('date'), + col.toLowerCase().includes("date") ) || remainingColumnsForTimestamp.find((col) => - col.toLowerCase().includes('time'), + col.toLowerCase().includes("time") ) || remainingColumnsForTimestamp[0], choices: [...remainingColumnsForTimestamp, NONE_PREFERENCE_MAP], @@ -58,7 +56,7 @@ export async function parsePreferenceTimestampsFromCsv( currentState.timestampColum = timestampName; } logger.info( - colors.magenta(`Using timestamp column "${currentState.timestampColum}"`), + colors.magenta(`Using timestamp column "${currentState.timestampColum}"`) ); // Validate that all rows have valid timestamp @@ -72,16 +70,15 @@ export async function parsePreferenceTimestampsFromCsv( `The timestamp column "${ currentState.timestampColum }" is missing a value for the following rows: ${timestampColumnsMissing.join( - '\n', - )}`, + "\n" + )}` ); } logger.info( colors.magenta( - `The timestamp column "${currentState.timestampColum}" is present for all row`, - ), + `The timestamp column "${currentState.timestampColum}" is present for all row` + ) ); } return currentState; } -/* eslint-enable no-param-reassign */ diff --git a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts index cc467f2c..baa9f464 100644 --- a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts +++ b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts @@ -1,16 +1,15 @@ -/* eslint-disable max-lines */ -import { PreferenceTopicType } from '@transcend-io/privacy-types'; -import { describe, expect, it } from 'vitest'; -import { PreferenceTopic } from '../../graphql'; -import { checkIfPendingPreferenceUpdatesAreNoOp } from '../index'; +import { PreferenceTopicType } from "@transcend-io/privacy-types"; +import { describe, expect, it } from "vitest"; +import { PreferenceTopic } from "../../graphql"; +import { checkIfPendingPreferenceUpdatesAreNoOp } from "../index"; const DEFAULT_VALUES = { - userId: 'test@transcend.io', - timestamp: '2024-11-30T00:00:15.327Z', - partition: 'd9c0b9ca-2253-4418-89d2-88776d654223', + userId: "test@transcend.io", + timestamp: "2024-11-30T00:00:15.327Z", + partition: "d9c0b9ca-2253-4418-89d2-88776d654223", system: { - decryptionStatus: 'DECRYPTED' as const, - updatedAt: '2024-11-30T00:00:16.506Z', + decryptionStatus: "DECRYPTED" as const, + updatedAt: "2024-11-30T00:00:16.506Z", }, consentManagement: { usp: null, @@ -23,122 +22,122 @@ const DEFAULT_VALUES = { const PREFERENCE_TOPICS: PreferenceTopic[] = [ { - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'BooleanPreference1', + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "BooleanPreference1", type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, title: { - defaultMessage: 'Boolean Preference 1', - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3', + defaultMessage: "Boolean Preference 1", + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3", }, displayDescription: { - defaultMessage: 'This is a boolean preference for testing.', - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', + defaultMessage: "This is a boolean preference for testing.", + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", }, - defaultConfiguration: '', + defaultConfiguration: "", showInPrivacyCenter: true, }, { - id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'BooleanPreference2', + id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "BooleanPreference2", type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, title: { - defaultMessage: 'Boolean Preference 2', - id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', + defaultMessage: "Boolean Preference 2", + id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", }, displayDescription: { - defaultMessage: 'This is another boolean preference for testing.', - id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b5', + defaultMessage: "This is another boolean preference for testing.", + id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b5", }, - defaultConfiguration: '', + defaultConfiguration: "", showInPrivacyCenter: true, }, { - id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'MultiSelectPreference', + id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "MultiSelectPreference", type: PreferenceTopicType.MultiSelect, title: { - defaultMessage: 'Multi Select Preference', - id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b0', + defaultMessage: "Multi Select Preference", + id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b0", }, displayDescription: { - defaultMessage: 'This is a multi-select preference for testing.', - id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', + defaultMessage: "This is a multi-select preference for testing.", + id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", }, - defaultConfiguration: '', + defaultConfiguration: "", showInPrivacyCenter: true, preferenceOptionValues: [ { - slug: 'Value1', + slug: "Value1", title: { - defaultMessage: 'Value 1', - id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', + defaultMessage: "Value 1", + id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", }, }, { - slug: 'Value2', + slug: "Value2", title: { - defaultMessage: 'Value 2', - id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2', + defaultMessage: "Value 2", + id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2", }, }, ], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, { - id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'SingleSelectPreference', + id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "SingleSelectPreference", type: PreferenceTopicType.Select, preferenceOptionValues: [ { - slug: 'Value1', + slug: "Value1", title: { - defaultMessage: 'Value 1', - id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', + defaultMessage: "Value 1", + id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", }, }, { - slug: 'Value2', + slug: "Value2", title: { - defaultMessage: 'Value 2', - id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2', + defaultMessage: "Value 2", + id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2", }, }, ], title: { - defaultMessage: 'Single Select Preference', - id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b0', + defaultMessage: "Single Select Preference", + id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b0", }, displayDescription: { - defaultMessage: 'This is a single-select preference for testing.', - id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', + defaultMessage: "This is a single-select preference for testing.", + id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", }, - defaultConfiguration: '', + defaultConfiguration: "", showInPrivacyCenter: true, purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, ]; -describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { - it('should return true for simple purpose comparison', () => { +describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { + it("should return true for simple purpose comparison", () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'SalesOutreach', + purpose: "SalesOutreach", enabled: false, preferences: [], }, @@ -150,18 +149,18 @@ describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(true); }); - it('should return false for simple purpose comparison', () => { + it("should return false for simple purpose comparison", () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'SalesOutreach', + purpose: "SalesOutreach", enabled: true, preferences: [], }, @@ -173,22 +172,22 @@ describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(false); }); - it('should return true for simple purpose comparison with extra preference', () => { + it("should return true for simple purpose comparison with extra preference", () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'SalesOutreach', + purpose: "SalesOutreach", enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, @@ -203,18 +202,18 @@ describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(true); }); - it('should return false for simple purpose comparison with extra preference in update', () => { + it("should return false for simple purpose comparison with extra preference in update", () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'SalesOutreach', + purpose: "SalesOutreach", enabled: false, }, ], @@ -224,7 +223,7 @@ describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, @@ -233,36 +232,36 @@ describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(false); }); - it('should return true for preferences being same', () => { + it("should return true for preferences being same", () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'Marketing', + purpose: "Marketing", enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], @@ -274,57 +273,57 @@ describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(true); }); - it('should return false for boolean preference changing', () => { + it("should return false for boolean preference changing", () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'Marketing', + purpose: "Marketing", enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: false, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], @@ -336,57 +335,57 @@ describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(false); }); - it('should return false for single select preference changing', () => { + it("should return false for single select preference changing", () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'Marketing', + purpose: "Marketing", enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value2', + selectValue: "Value2", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], @@ -398,57 +397,57 @@ describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(false); }); - it('should return false for multi select preference changing', () => { + it("should return false for multi select preference changing", () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'Marketing', + purpose: "Marketing", enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value2'], + selectValues: ["Value2"], }, }, ], @@ -460,29 +459,28 @@ describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(false); }); }); -/* eslint-enable max-lines */ diff --git a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts index bca6bb10..11fffaf9 100644 --- a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts +++ b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts @@ -1,16 +1,15 @@ -/* eslint-disable max-lines */ -import { PreferenceTopicType } from '@transcend-io/privacy-types'; -import { describe, expect, it } from 'vitest'; -import { PreferenceTopic } from '../../graphql'; -import { checkIfPendingPreferenceUpdatesCauseConflict } from '../index'; +import { PreferenceTopicType } from "@transcend-io/privacy-types"; +import { describe, expect, it } from "vitest"; +import { PreferenceTopic } from "../../graphql"; +import { checkIfPendingPreferenceUpdatesCauseConflict } from "../index"; const DEFAULT_VALUES = { - userId: 'test@transcend.io', - timestamp: '2024-11-30T00:00:15.327Z', - partition: 'd9c0b9ca-2253-4418-89d2-88776d654223', + userId: "test@transcend.io", + timestamp: "2024-11-30T00:00:15.327Z", + partition: "d9c0b9ca-2253-4418-89d2-88776d654223", system: { - decryptionStatus: 'DECRYPTED' as const, - updatedAt: '2024-11-30T00:00:16.506Z', + decryptionStatus: "DECRYPTED" as const, + updatedAt: "2024-11-30T00:00:16.506Z", }, consentManagement: { usp: null, @@ -23,125 +22,125 @@ const DEFAULT_VALUES = { const PREFERENCE_TOPICS: PreferenceTopic[] = [ { - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'BooleanPreference1', + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "BooleanPreference1", type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, title: { - defaultMessage: 'Boolean Preference 1', - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3', + defaultMessage: "Boolean Preference 1", + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3", }, displayDescription: { - defaultMessage: 'This is a boolean preference for testing purposes.', - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', + defaultMessage: "This is a boolean preference for testing purposes.", + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", }, { - id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'BooleanPreference2', + id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "BooleanPreference2", type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, title: { - defaultMessage: 'Boolean Preference 2', - id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', + defaultMessage: "Boolean Preference 2", + id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", }, displayDescription: { defaultMessage: - 'This is another boolean preference for testing purposes.', - id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b5', + "This is another boolean preference for testing purposes.", + id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b5", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", }, { - id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'MultiSelectPreference', + id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "MultiSelectPreference", type: PreferenceTopicType.MultiSelect, preferenceOptionValues: [ { - slug: 'Value1', + slug: "Value1", title: { - defaultMessage: 'Value 1', - id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', + defaultMessage: "Value 1", + id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", }, }, { - slug: 'Value2', + slug: "Value2", title: { - defaultMessage: 'Value 2', - id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2', + defaultMessage: "Value 2", + id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2", }, }, ], title: { - defaultMessage: 'Multi Select Preference', - id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3', + defaultMessage: "Multi Select Preference", + id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3", }, displayDescription: { - defaultMessage: 'This is a multi-select preference for testing purposes.', - id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', + defaultMessage: "This is a multi-select preference for testing purposes.", + id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, { - id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'SingleSelectPreference', + id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "SingleSelectPreference", type: PreferenceTopicType.Select, preferenceOptionValues: [ { - slug: 'Value1', + slug: "Value1", title: { - defaultMessage: 'Value 1', - id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', + defaultMessage: "Value 1", + id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", }, }, { - slug: 'Value2', + slug: "Value2", title: { - defaultMessage: 'Value 2', - id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2', + defaultMessage: "Value 2", + id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2", }, }, ], title: { - defaultMessage: 'Single Select Preference', - id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3', + defaultMessage: "Single Select Preference", + id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3", }, displayDescription: { defaultMessage: - 'This is a single-select preference for testing purposes.', - id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', + "This is a single-select preference for testing purposes.", + id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", // This preference is associated with the Marketing purpose purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, ]; -describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { - it('should return false for simple purpose comparison', () => { +describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { + it("should return false for simple purpose comparison", () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'SalesOutreach', + purpose: "SalesOutreach", enabled: false, preferences: [], }, @@ -153,18 +152,18 @@ describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(false); }); - it('should return false if purpose missing', () => { + it("should return false if purpose missing", () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'SalesOutreach', + purpose: "SalesOutreach", enabled: false, preferences: [], }, @@ -176,18 +175,18 @@ describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(false); }); - it('should return true for simple purpose comparison', () => { + it("should return true for simple purpose comparison", () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'SalesOutreach', + purpose: "SalesOutreach", enabled: true, preferences: [], }, @@ -199,22 +198,22 @@ describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(true); }); - it('should return true for simple purpose comparison with extra preference', () => { + it("should return true for simple purpose comparison with extra preference", () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'SalesOutreach', + purpose: "SalesOutreach", enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, @@ -229,18 +228,18 @@ describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(false); }); - it('should return false for simple purpose comparison with extra preference in update', () => { + it("should return false for simple purpose comparison with extra preference in update", () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'SalesOutreach', + purpose: "SalesOutreach", enabled: false, }, ], @@ -250,7 +249,7 @@ describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, @@ -259,36 +258,36 @@ describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(false); }); - it('should return false for preferences being same', () => { + it("should return false for preferences being same", () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'Marketing', + purpose: "Marketing", enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], @@ -300,57 +299,57 @@ describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(false); }); - it('should return true for boolean preference changing', () => { + it("should return true for boolean preference changing", () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'Marketing', + purpose: "Marketing", enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: false, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], @@ -362,57 +361,57 @@ describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(true); }); - it('should return true for single select preference changing', () => { + it("should return true for single select preference changing", () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'Marketing', + purpose: "Marketing", enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value2', + selectValue: "Value2", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], @@ -424,57 +423,57 @@ describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(true); }); - it('should return true for multi select preference changing', () => { + it("should return true for multi select preference changing", () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: 'Marketing', + purpose: "Marketing", enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value2'], + selectValues: ["Value2"], }, }, ], @@ -486,29 +485,28 @@ describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { enabled: false, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }), + }) ).to.equal(true); }); }); -/* eslint-enable max-lines */ diff --git a/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts b/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts index 0ad9374c..587d3353 100644 --- a/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts +++ b/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts @@ -1,61 +1,60 @@ -/* eslint-disable max-lines */ -import { PreferenceTopicType } from '@transcend-io/privacy-types'; -import { describe, expect, it } from 'vitest'; -import { getPreferenceUpdatesFromRow } from '../index'; +import { PreferenceTopicType } from "@transcend-io/privacy-types"; +import { describe, expect, it } from "vitest"; +import { getPreferenceUpdatesFromRow } from "../index"; -describe('getPreferenceUpdatesFromRow', () => { - it('should parse boolean updates', () => { +describe("getPreferenceUpdatesFromRow", () => { + it("should parse boolean updates", () => { expect( getPreferenceUpdatesFromRow({ row: { - my_purpose: 'true', - has_topic_1: 'true', - has_topic_2: 'false', + my_purpose: "true", + has_topic_1: "true", + has_topic_2: "false", }, - purposeSlugs: ['Marketing'], + purposeSlugs: ["Marketing"], preferenceTopics: [ { - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'BooleanPreference1', + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "BooleanPreference1", type: PreferenceTopicType.Boolean, preferenceOptionValues: [], title: { - defaultMessage: 'Marketing Preferences', - id: '12345678-1234-1234-1234-123456789012', + defaultMessage: "Marketing Preferences", + id: "12345678-1234-1234-1234-123456789012", }, displayDescription: { - defaultMessage: 'Enable marketing tracking', - id: '12345678-1234-1234-1234-123456789013', + defaultMessage: "Enable marketing tracking", + id: "12345678-1234-1234-1234-123456789013", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, { - id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'BooleanPreference2', + id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "BooleanPreference2", type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", title: { - defaultMessage: 'Advertising Preferences', - id: '12345678-1234-1234-1234-123456789014', + defaultMessage: "Advertising Preferences", + id: "12345678-1234-1234-1234-123456789014", }, displayDescription: { - defaultMessage: 'Enable advertising tracking', - id: '12345678-1234-1234-1234-123456789015', + defaultMessage: "Enable advertising tracking", + id: "12345678-1234-1234-1234-123456789015", }, }, ], columnToPurposeName: { my_purpose: { - purpose: 'Marketing', + purpose: "Marketing", preference: null, valueMapping: { true: true, @@ -63,35 +62,35 @@ describe('getPreferenceUpdatesFromRow', () => { }, }, has_topic_1: { - purpose: 'Marketing', - preference: 'BooleanPreference1', + purpose: "Marketing", + preference: "BooleanPreference1", valueMapping: { true: true, false: false, }, }, has_topic_2: { - purpose: 'Marketing', - preference: 'BooleanPreference2', + purpose: "Marketing", + preference: "BooleanPreference2", valueMapping: { true: true, false: false, }, }, }, - }), + }) ).to.deep.equal({ Marketing: { enabled: true, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'BooleanPreference2', + topic: "BooleanPreference2", choice: { booleanValue: false, }, @@ -101,53 +100,53 @@ describe('getPreferenceUpdatesFromRow', () => { }); }); - it('should parse a single select', () => { + it("should parse a single select", () => { expect( getPreferenceUpdatesFromRow({ row: { - my_purpose: 'true', - has_topic_3: 'Option 1', + my_purpose: "true", + has_topic_3: "Option 1", }, - purposeSlugs: ['Marketing'], + purposeSlugs: ["Marketing"], preferenceTopics: [ { - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'SingleSelectPreference', - defaultConfiguration: '', + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "SingleSelectPreference", + defaultConfiguration: "", title: { - defaultMessage: 'Single Select Preference', - id: '12345678-1234-1234-1234-123456789010', + defaultMessage: "Single Select Preference", + id: "12345678-1234-1234-1234-123456789010", }, displayDescription: { - defaultMessage: 'Choose one option', - id: '12345678-1234-1234-1234-123456789011', + defaultMessage: "Choose one option", + id: "12345678-1234-1234-1234-123456789011", }, showInPrivacyCenter: true, type: PreferenceTopicType.Select, preferenceOptionValues: [ { - slug: 'Value1', + slug: "Value1", title: { - defaultMessage: 'Option 1', - id: '12345678-1234-1234-1234-123456789016', + defaultMessage: "Option 1", + id: "12345678-1234-1234-1234-123456789016", }, }, { - slug: 'Value2', + slug: "Value2", title: { - defaultMessage: 'Option 2', - id: '12345678-1234-1234-1234-123456789017', + defaultMessage: "Option 2", + id: "12345678-1234-1234-1234-123456789017", }, }, ], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, ], columnToPurposeName: { my_purpose: { - purpose: 'Marketing', + purpose: "Marketing", preference: null, valueMapping: { true: true, @@ -155,23 +154,23 @@ describe('getPreferenceUpdatesFromRow', () => { }, }, has_topic_3: { - purpose: 'Marketing', - preference: 'SingleSelectPreference', + purpose: "Marketing", + preference: "SingleSelectPreference", valueMapping: { - 'Option 1': 'Value1', - 'Option 2': 'Value2', + "Option 1": "Value1", + "Option 2": "Value2", }, }, }, - }), + }) ).to.deep.equal({ Marketing: { enabled: true, preferences: [ { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, ], @@ -179,54 +178,54 @@ describe('getPreferenceUpdatesFromRow', () => { }); }); - it('should parse a multi select example', () => { + it("should parse a multi select example", () => { expect( getPreferenceUpdatesFromRow({ row: { - my_purpose: 'true', - has_topic_4: 'Option 2,Option 1', - has_topic_5: 'Option 1', + my_purpose: "true", + has_topic_4: "Option 2,Option 1", + has_topic_5: "Option 1", }, - purposeSlugs: ['Marketing'], + purposeSlugs: ["Marketing"], preferenceTopics: [ { - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'MultiSelectPreference', + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "MultiSelectPreference", type: PreferenceTopicType.MultiSelect, - defaultConfiguration: '', + defaultConfiguration: "", title: { - defaultMessage: 'Multi Select Preference', - id: '12345678-1234-1234-1234-123456789020', + defaultMessage: "Multi Select Preference", + id: "12345678-1234-1234-1234-123456789020", }, displayDescription: { - defaultMessage: 'Choose multiple options', - id: '12345678-1234-1234-1234-123456789021', + defaultMessage: "Choose multiple options", + id: "12345678-1234-1234-1234-123456789021", }, showInPrivacyCenter: true, preferenceOptionValues: [ { - slug: 'Value1', + slug: "Value1", title: { - defaultMessage: 'Option 1', - id: '12345678-1234-1234-1234-123456789018', + defaultMessage: "Option 1", + id: "12345678-1234-1234-1234-123456789018", }, }, { - slug: 'Value2', + slug: "Value2", title: { - defaultMessage: 'Option 2', - id: '12345678-1234-1234-1234-123456789019', + defaultMessage: "Option 2", + id: "12345678-1234-1234-1234-123456789019", }, }, ], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, ], columnToPurposeName: { my_purpose: { - purpose: 'Marketing', + purpose: "Marketing", preference: null, valueMapping: { true: true, @@ -234,37 +233,37 @@ describe('getPreferenceUpdatesFromRow', () => { }, }, has_topic_4: { - purpose: 'Marketing', - preference: 'MultiSelectPreference', + purpose: "Marketing", + preference: "MultiSelectPreference", valueMapping: { - 'Option 1': 'Value1', - 'Option 2': 'Value2', + "Option 1": "Value1", + "Option 2": "Value2", }, }, has_topic_5: { - purpose: 'Marketing', - preference: 'MultiSelectPreference', + purpose: "Marketing", + preference: "MultiSelectPreference", valueMapping: { - 'Option 1': 'Value1', - 'Option 2': 'Value2', + "Option 1": "Value1", + "Option 2": "Value2", }, }, }, - }), + }) ).to.deep.equal({ Marketing: { enabled: true, preferences: [ { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1'], + selectValues: ["Value1"], }, }, ], @@ -272,128 +271,128 @@ describe('getPreferenceUpdatesFromRow', () => { }); }); - it('should parse boolean, single select, multi select example', () => { + it("should parse boolean, single select, multi select example", () => { expect( getPreferenceUpdatesFromRow({ row: { - my_purpose: 'true', - has_topic_1: 'true', - has_topic_2: 'false', - has_topic_3: 'Option 1', - has_topic_4: 'Option 2,Option 1', + my_purpose: "true", + has_topic_1: "true", + has_topic_2: "false", + has_topic_3: "Option 1", + has_topic_4: "Option 2,Option 1", }, - purposeSlugs: ['Marketing'], + purposeSlugs: ["Marketing"], preferenceTopics: [ { - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'BooleanPreference1', + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "BooleanPreference1", type: PreferenceTopicType.Boolean, preferenceOptionValues: [], title: { - defaultMessage: 'Boolean Preference 1', - id: '12345678-1234-1234-1234-123456789022', + defaultMessage: "Boolean Preference 1", + id: "12345678-1234-1234-1234-123456789022", }, displayDescription: { - defaultMessage: 'Enable this preference', - id: '12345678-1234-1234-1234-123456789023', + defaultMessage: "Enable this preference", + id: "12345678-1234-1234-1234-123456789023", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, { - id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'BooleanPreference2', + id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "BooleanPreference2", type: PreferenceTopicType.Boolean, preferenceOptionValues: [], title: { - defaultMessage: 'Boolean Preference 2', - id: '12345678-1234-1234-1234-123456789024', + defaultMessage: "Boolean Preference 2", + id: "12345678-1234-1234-1234-123456789024", }, displayDescription: { - defaultMessage: 'Disable this preference', - id: '12345678-1234-1234-1234-123456789025', + defaultMessage: "Disable this preference", + id: "12345678-1234-1234-1234-123456789025", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, { - id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'MultiSelectPreference', + id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "MultiSelectPreference", type: PreferenceTopicType.MultiSelect, - defaultConfiguration: '', + defaultConfiguration: "", title: { - defaultMessage: 'Multi Select Preference', - id: '12345678-1234-1234-1234-123456789028', + defaultMessage: "Multi Select Preference", + id: "12345678-1234-1234-1234-123456789028", }, displayDescription: { - defaultMessage: 'Choose multiple options', - id: '12345678-1234-1234-1234-123456789029', + defaultMessage: "Choose multiple options", + id: "12345678-1234-1234-1234-123456789029", }, showInPrivacyCenter: true, preferenceOptionValues: [ { - slug: 'Value1', + slug: "Value1", title: { - defaultMessage: 'Option 1', - id: '12345678-1234-1234-1234-123456789026', + defaultMessage: "Option 1", + id: "12345678-1234-1234-1234-123456789026", }, }, { - slug: 'Value2', + slug: "Value2", title: { - defaultMessage: 'Option 2', - id: '12345678-1234-1234-1234-123456789027', + defaultMessage: "Option 2", + id: "12345678-1234-1234-1234-123456789027", }, }, ], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, { - id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'SingleSelectPreference', + id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "SingleSelectPreference", type: PreferenceTopicType.Select, - defaultConfiguration: '', + defaultConfiguration: "", title: { - defaultMessage: 'Single Select Preference', - id: '12345678-1234-1234-1234-123456789030', + defaultMessage: "Single Select Preference", + id: "12345678-1234-1234-1234-123456789030", }, displayDescription: { - defaultMessage: 'Choose one option', - id: '12345678-1234-1234-1234-123456789031', + defaultMessage: "Choose one option", + id: "12345678-1234-1234-1234-123456789031", }, showInPrivacyCenter: true, preferenceOptionValues: [ { - slug: 'Value1', + slug: "Value1", title: { - defaultMessage: 'Option 1', - id: '12345678-1234-1234-1234-123456789030', + defaultMessage: "Option 1", + id: "12345678-1234-1234-1234-123456789030", }, }, { - slug: 'Value2', + slug: "Value2", title: { - defaultMessage: 'Option 2', - id: '12345678-1234-1234-1234-123456789031', + defaultMessage: "Option 2", + id: "12345678-1234-1234-1234-123456789031", }, }, ], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, ], columnToPurposeName: { my_purpose: { - purpose: 'Marketing', + purpose: "Marketing", preference: null, valueMapping: { true: true, @@ -401,65 +400,65 @@ describe('getPreferenceUpdatesFromRow', () => { }, }, has_topic_1: { - purpose: 'Marketing', - preference: 'BooleanPreference1', + purpose: "Marketing", + preference: "BooleanPreference1", valueMapping: { true: true, false: false, }, }, has_topic_2: { - purpose: 'Marketing', - preference: 'BooleanPreference2', + purpose: "Marketing", + preference: "BooleanPreference2", valueMapping: { true: true, false: false, }, }, has_topic_3: { - purpose: 'Marketing', - preference: 'SingleSelectPreference', + purpose: "Marketing", + preference: "SingleSelectPreference", valueMapping: { - 'Option 1': 'Value1', - 'Option 2': 'Value2', + "Option 1": "Value1", + "Option 2": "Value2", }, }, has_topic_4: { - purpose: 'Marketing', - preference: 'MultiSelectPreference', + purpose: "Marketing", + preference: "MultiSelectPreference", valueMapping: { - 'Option 1': 'Value1', - 'Option 2': 'Value2', + "Option 1": "Value1", + "Option 2": "Value2", }, }, }, - }), + }) ).to.deep.equal({ Marketing: { enabled: true, preferences: [ { - topic: 'BooleanPreference1', + topic: "BooleanPreference1", choice: { booleanValue: true, }, }, { - topic: 'BooleanPreference2', + topic: "BooleanPreference2", choice: { booleanValue: false, }, }, { - topic: 'SingleSelectPreference', + topic: "SingleSelectPreference", choice: { - selectValue: 'Value1', + selectValue: "Value1", }, }, { - topic: 'MultiSelectPreference', + topic: "MultiSelectPreference", choice: { - selectValues: ['Value1', 'Value2'], + selectValues: ["Value1", "Value2"], }, }, ], @@ -467,57 +466,57 @@ describe('getPreferenceUpdatesFromRow', () => { }); }); - it('should error if missing purpose', () => { + it("should error if missing purpose", () => { try { getPreferenceUpdatesFromRow({ row: { - has_topic_1: 'true', + has_topic_1: "true", }, - purposeSlugs: ['Marketing'], + purposeSlugs: ["Marketing"], preferenceTopics: [ { - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'BooleanPreference1', + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "BooleanPreference1", type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, displayDescription: { - defaultMessage: 'Enable this preference', - id: '12345678-1234-1234-1234-123456789032', + defaultMessage: "Enable this preference", + id: "12345678-1234-1234-1234-123456789032", }, title: { - defaultMessage: 'Boolean Preference 1', - id: '12345678-1234-1234-1234-123456789033', + defaultMessage: "Boolean Preference 1", + id: "12345678-1234-1234-1234-123456789033", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", }, { - id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'BooleanPreference2', + id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "BooleanPreference2", type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, displayDescription: { - defaultMessage: 'Disable this preference', - id: '12345678-1234-1234-1234-123456789034', + defaultMessage: "Disable this preference", + id: "12345678-1234-1234-1234-123456789034", }, title: { - defaultMessage: 'Boolean Preference 2', - id: '12345678-1234-1234-1234-123456789035', + defaultMessage: "Boolean Preference 2", + id: "12345678-1234-1234-1234-123456789035", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", }, ], columnToPurposeName: { has_topic_1: { - purpose: 'Marketing', - preference: 'BooleanPreference1', + purpose: "Marketing", + preference: "BooleanPreference1", valueMapping: { true: true, false: false, @@ -525,63 +524,63 @@ describe('getPreferenceUpdatesFromRow', () => { }, }, }); - expect.fail('Should have thrown'); - } catch (err) { - expect(err.message).to.include('No mapping provided'); + expect.fail("Should have thrown"); + } catch (error) { + expect(error.message).to.include("No mapping provided"); } }); - it('should error if purpose name is not valid', () => { + it("should error if purpose name is not valid", () => { try { getPreferenceUpdatesFromRow({ row: { - has_topic_1: 'true', + has_topic_1: "true", }, - purposeSlugs: ['Marketing', 'Advertising'], + purposeSlugs: ["Marketing", "Advertising"], preferenceTopics: [ { - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'BooleanPreference1', + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "BooleanPreference1", type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, displayDescription: { - defaultMessage: 'Enable this preference', - id: '12345678-1234-1234-1234-123456789036', + defaultMessage: "Enable this preference", + id: "12345678-1234-1234-1234-123456789036", }, title: { - defaultMessage: 'Boolean Preference 1', - id: '12345678-1234-1234-1234-123456789037', + defaultMessage: "Boolean Preference 1", + id: "12345678-1234-1234-1234-123456789037", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", }, { - id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'BooleanPreference2', + id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "BooleanPreference2", type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, displayDescription: { - defaultMessage: 'Disable this preference', - id: '12345678-1234-1234-1234-123456789038', + defaultMessage: "Disable this preference", + id: "12345678-1234-1234-1234-123456789038", }, title: { - defaultMessage: 'Boolean Preference 2', - id: '12345678-1234-1234-1234-123456789039', + defaultMessage: "Boolean Preference 2", + id: "12345678-1234-1234-1234-123456789039", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", }, ], columnToPurposeName: { has_topic_1: { - purpose: 'InvalidPurpose', - preference: 'BooleanPreference1', + purpose: "InvalidPurpose", + preference: "BooleanPreference1", valueMapping: { true: true, false: false, @@ -589,136 +588,135 @@ describe('getPreferenceUpdatesFromRow', () => { }, }, }); - expect.fail('Should have thrown'); - } catch (err) { - expect(err.message).to.equal( - 'Invalid purpose slug: InvalidPurpose, expected: Marketing, Advertising', + expect.fail("Should have thrown"); + } catch (error) { + expect(error.message).to.equal( + "Invalid purpose slug: InvalidPurpose, expected: Marketing, Advertising" ); } }); - it('should error if single select option is invalid', () => { + it("should error if single select option is invalid", () => { try { getPreferenceUpdatesFromRow({ row: { - has_topic_1: 'true', + has_topic_1: "true", }, - purposeSlugs: ['Marketing'], + purposeSlugs: ["Marketing"], preferenceTopics: [ { - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'SingleSelectPreference', + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "SingleSelectPreference", type: PreferenceTopicType.Select, preferenceOptionValues: [ { - slug: 'Value1', + slug: "Value1", title: { - defaultMessage: 'Option 1', - id: '12345678-1234-1234-1234-123456789040', + defaultMessage: "Option 1", + id: "12345678-1234-1234-1234-123456789040", }, }, { - slug: 'Value2', + slug: "Value2", title: { - defaultMessage: 'Option 2', - id: '12345678-1234-1234-1234-123456789041', + defaultMessage: "Option 2", + id: "12345678-1234-1234-1234-123456789041", }, }, ], title: { - defaultMessage: 'Single Select Preference', - id: '12345678-1234-1234-1234-123456789042', + defaultMessage: "Single Select Preference", + id: "12345678-1234-1234-1234-123456789042", }, displayDescription: { - defaultMessage: 'Choose one option', - id: '12345678-1234-1234-1234-123456789043', + defaultMessage: "Choose one option", + id: "12345678-1234-1234-1234-123456789043", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, ], columnToPurposeName: { has_topic_1: { - purpose: 'Marketing', - preference: 'SingleSelectPreference', + purpose: "Marketing", + preference: "SingleSelectPreference", valueMapping: { - 'Option 1': 'Value1', - 'Option 2': 'Value2', + "Option 1": "Value1", + "Option 2": "Value2", }, }, }, }); - expect.fail('Should have thrown'); - } catch (err) { - expect(err.message).to.equal( - 'Invalid value for select preference: SingleSelectPreference, expected string or null, got: true', + expect.fail("Should have thrown"); + } catch (error) { + expect(error.message).to.equal( + "Invalid value for select preference: SingleSelectPreference, expected string or null, got: true" ); } }); - it('should error if multi select value is invalid', () => { + it("should error if multi select value is invalid", () => { try { getPreferenceUpdatesFromRow({ row: { - has_topic_1: 'true', + has_topic_1: "true", }, - purposeSlugs: ['Marketing'], + purposeSlugs: ["Marketing"], preferenceTopics: [ { - id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', - slug: 'MultiSelectPreference', + id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", + slug: "MultiSelectPreference", type: PreferenceTopicType.MultiSelect, preferenceOptionValues: [ { title: { - defaultMessage: 'Option 1', - id: '12345678-1234-1234-1234-123456789044', + defaultMessage: "Option 1", + id: "12345678-1234-1234-1234-123456789044", }, - slug: 'Value1', + slug: "Value1", }, { title: { - defaultMessage: 'Option 2', - id: '12345678-1234-1234-1234-123456789045', + defaultMessage: "Option 2", + id: "12345678-1234-1234-1234-123456789045", }, - slug: 'Value2', + slug: "Value2", }, ], title: { - defaultMessage: 'Multi Select Preference', - id: '12345678-1234-1234-1234-123456789046', + defaultMessage: "Multi Select Preference", + id: "12345678-1234-1234-1234-123456789046", }, displayDescription: { - defaultMessage: 'Choose multiple options', - id: '12345678-1234-1234-1234-123456789047', + defaultMessage: "Choose multiple options", + id: "12345678-1234-1234-1234-123456789047", }, showInPrivacyCenter: true, - defaultConfiguration: '', + defaultConfiguration: "", purpose: { - trackingType: 'Marketing', + trackingType: "Marketing", }, }, ], columnToPurposeName: { has_topic_1: { - purpose: 'Marketing', - preference: 'MultiSelectPreference', + purpose: "Marketing", + preference: "MultiSelectPreference", valueMapping: { - 'Option 1': 'Value1', - 'Option 2': 'Value2', + "Option 1": "Value1", + "Option 2": "Value2", }, }, }, }); - expect.fail('Should have thrown'); - } catch (err) { - expect(err.message).to.equal( - 'Invalid value for multi select preference: MultiSelectPreference, expected one of: Value1, Value2, got: true', + expect.fail("Should have thrown"); + } catch (error) { + expect(error.message).to.equal( + "Invalid value for multi select preference: MultiSelectPreference, expected one of: Value1, Value2, got: true" ); } }); }); -/* eslint-enable max-lines */ diff --git a/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts b/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts index ed8cd7a4..06da6e1f 100644 --- a/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts +++ b/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts @@ -1,12 +1,12 @@ -import { PersistedState } from '@transcend-io/persisted-state'; -import { PreferenceUpdateItem } from '@transcend-io/privacy-types'; -import { apply } from '@transcend-io/type-utils'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { chunk } from 'lodash-es'; -import { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { PersistedState } from "@transcend-io/persisted-state"; +import { PreferenceUpdateItem } from "@transcend-io/privacy-types"; +import { apply } from "@transcend-io/type-utils"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { chunk } from "lodash-es"; +import { DEFAULT_TRANSCEND_CONSENT_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, createSombraGotInstance, @@ -14,12 +14,12 @@ import { fetchAllPurposes, PreferenceTopic, Purpose, -} from '../graphql'; -import { parseAttributesFromString } from '../requests'; -import { PreferenceState } from './codecs'; -import { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow'; -import { parsePreferenceManagementCsvWithCache } from './parsePreferenceManagementCsv'; -import { NONE_PREFERENCE_MAP } from './parsePreferenceTimestampsFromCsv'; +} from "../graphql"; +import { parseAttributesFromString } from "../requests"; +import { PreferenceState } from "./codecs"; +import { getPreferenceUpdatesFromRow } from "./getPreferenceUpdatesFromRow"; +import { parsePreferenceManagementCsvWithCache } from "./parsePreferenceManagementCsv"; +import { NONE_PREFERENCE_MAP } from "./parsePreferenceTimestampsFromCsv"; /** * Upload a set of consent preferences @@ -80,13 +80,13 @@ export async function uploadPreferenceManagementPreferencesInteractive({ failingUpdates: {}, pendingUpdates: {}, }); - const failingRequests = preferenceState.getValue('failingUpdates'); - const pendingRequests = preferenceState.getValue('pendingUpdates'); - let fileMetadata = preferenceState.getValue('fileMetadata'); + const failingRequests = preferenceState.getValue("failingUpdates"); + const pendingRequests = preferenceState.getValue("pendingUpdates"); + let fileMetadata = preferenceState.getValue("fileMetadata"); logger.info( colors.magenta( - 'Restored cache, there are: \n' + + "Restored cache, there are: \n" + `${ Object.values(failingRequests).length } failing requests to be retried\n` + @@ -94,12 +94,12 @@ export async function uploadPreferenceManagementPreferencesInteractive({ Object.values(pendingRequests).length } pending requests to be processed\n` + `The following files are stored in cache and will be used:\n${Object.keys( - fileMetadata, + fileMetadata ) .map((x) => x) - .join('\n')}\n` + - `The following file will be processed: ${file}\n`, - ), + .join("\n")}\n` + + `The following file will be processed: ${file}\n` + ) ); // Create GraphQL client to connect to Transcend backend @@ -128,43 +128,43 @@ export async function uploadPreferenceManagementPreferencesInteractive({ skipExistingRecordCheck, forceTriggerWorkflows, }, - preferenceState, + preferenceState ); // Construct the pending updates const pendingUpdates: Record = {}; - fileMetadata = preferenceState.getValue('fileMetadata'); + fileMetadata = preferenceState.getValue("fileMetadata"); const metadata = fileMetadata[file]; logger.info( colors.magenta( `Found ${ Object.entries(metadata.pendingSafeUpdates).length - } safe updates in ${file}`, - ), + } safe updates in ${file}` + ) ); logger.info( colors.magenta( `Found ${ Object.entries(metadata.pendingConflictUpdates).length - } conflict updates in ${file}`, - ), + } conflict updates in ${file}` + ) ); logger.info( colors.magenta( `Found ${ Object.entries(metadata.skippedUpdates).length - } skipped updates in ${file}`, - ), + } skipped updates in ${file}` + ) ); // Update either safe updates only or safe + conflict - Object.entries({ + for (const [userId, update] of Object.entries({ ...metadata.pendingSafeUpdates, ...(skipConflictUpdates ? {} : apply(metadata.pendingConflictUpdates, ({ row }) => row)), - }).forEach(([userId, update]) => { + })) { // Determine timestamp const timestamp = metadata.timestampColum === NONE_PREFERENCE_MAP @@ -192,9 +192,9 @@ export async function uploadPreferenceManagementPreferencesInteractive({ }, })), }; - }); - await preferenceState.setValue(pendingUpdates, 'pendingUpdates'); - await preferenceState.setValue({}, 'failingUpdates'); + } + await preferenceState.setValue(pendingUpdates, "pendingUpdates"); + await preferenceState.setValue({}, "failingUpdates"); // Exist early if dry run if (dryRun) { @@ -202,8 +202,8 @@ export async function uploadPreferenceManagementPreferencesInteractive({ colors.green( `Dry run complete, exiting. ${ Object.values(pendingUpdates).length - } pending updates. Check file: ${receiptFilepath}`, - ), + } pending updates. Check file: ${receiptFilepath}` + ) ); return; } @@ -212,17 +212,17 @@ export async function uploadPreferenceManagementPreferencesInteractive({ colors.magenta( `Uploading ${ Object.values(pendingUpdates).length - } preferences to partition: ${partition}`, - ), + } preferences to partition: ${partition}` + ) ); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Build a GraphQL client @@ -236,7 +236,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ // Make the request try { await sombra - .put('v1/preferences', { + .put("v1/preferences", { json: { records: currentChunk.map(([, update]) => update), skipWorkflowTriggers, @@ -244,13 +244,13 @@ export async function uploadPreferenceManagementPreferencesInteractive({ }, }) .json(); - } catch (err) { + } catch (error) { try { - const parsed = JSON.parse(err?.response?.body || '{}'); + const parsed = JSON.parse(error?.response?.body || "{}"); if (parsed.error) { logger.error(colors.red(`Error: ${parsed.error}`)); } - } catch (e) { + } catch { // continue } logger.error( @@ -258,19 +258,19 @@ export async function uploadPreferenceManagementPreferencesInteractive({ `Failed to upload ${ currentChunk.length } user preferences to partition ${partition}: ${ - err?.response?.body || err?.message - }`, - ), + error?.response?.body || error?.message + }` + ) ); - const failingUpdates = preferenceState.getValue('failingUpdates'); - currentChunk.forEach(([userId, update]) => { + const failingUpdates = preferenceState.getValue("failingUpdates"); + for (const [userId, update] of currentChunk) { failingUpdates[userId] = { uploadedAt: new Date().toISOString(), update, - error: err?.response?.body || err?.message || 'Unknown error', + error: error?.response?.body || error?.message || "Unknown error", }; - }); - await preferenceState.setValue(failingUpdates, 'failingUpdates'); + } + await preferenceState.setValue(failingUpdates, "failingUpdates"); } total += currentChunk.length; @@ -278,11 +278,11 @@ export async function uploadPreferenceManagementPreferencesInteractive({ }, { concurrency: 40, - }, + } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( @@ -290,7 +290,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ updatesToRun.length } user preferences to partition ${partition} in "${ totalTime / 1000 - }" seconds!`, - ), + }" seconds!` + ) ); } diff --git a/src/lib/readTranscendYaml.ts b/src/lib/readTranscendYaml.ts index b7a54ee8..d31ed407 100644 --- a/src/lib/readTranscendYaml.ts +++ b/src/lib/readTranscendYaml.ts @@ -1,10 +1,10 @@ -import { readFileSync, writeFileSync } from 'fs'; -import { decodeCodec, ObjByString } from '@transcend-io/type-utils'; -import yaml from 'js-yaml'; -import { TranscendInput } from '../codecs'; +import { readFileSync, writeFileSync } from "node:fs"; +import { decodeCodec, ObjByString } from "@transcend-io/type-utils"; +import yaml from "js-yaml"; +import { TranscendInput } from "../codecs"; export const VARIABLE_PARAMETERS_REGEXP = /<>/; -export const VARIABLE_PARAMETERS_NAME = 'parameters'; +export const VARIABLE_PARAMETERS_NAME = "parameters"; /** * Function that replaces variables in a text file. @@ -18,15 +18,15 @@ export const VARIABLE_PARAMETERS_NAME = 'parameters'; export function replaceVariablesInYaml( input: string, variables: ObjByString, - extraErrorMessage = '', + extraErrorMessage = "" ): string { let contents = input; // Replace variables - Object.entries(variables).forEach(([name, value]) => { + for (const [name, value] of Object.entries(variables)) { contents = contents .split(`<<${VARIABLE_PARAMETERS_NAME}.${name}>>`) .join(value); - }); + } // Throw error if unfilled variables if (VARIABLE_PARAMETERS_REGEXP.test(contents)) { @@ -34,7 +34,7 @@ export function replaceVariablesInYaml( throw new Error( `Found variable that was not set: ${name}. Make sure you are passing all parameters through the --${VARIABLE_PARAMETERS_NAME}=${name}:value-for-param flag. -${extraErrorMessage}`, +${extraErrorMessage}` ); } @@ -51,16 +51,16 @@ ${extraErrorMessage}`, */ export function readTranscendYaml( filePath: string, - variables: ObjByString = {}, + variables: ObjByString = {} ): TranscendInput { // Read in contents - const fileContents = readFileSync(filePath, 'utf-8'); + const fileContents = readFileSync(filePath, "utf-8"); // Replace variables const replacedVariables = replaceVariablesInYaml( fileContents, variables, - `Also check that there are no extra variables defined in your yaml: ${filePath}`, + `Also check that there are no extra variables defined in your yaml: ${filePath}` ); // Validate shape @@ -75,7 +75,7 @@ export function readTranscendYaml( */ export function writeTranscendYaml( filePath: string, - input: TranscendInput, + input: TranscendInput ): void { writeFileSync(filePath, yaml.dump(decodeCodec(TranscendInput, input))); } diff --git a/src/lib/requests/approvePrivacyRequests.ts b/src/lib/requests/approvePrivacyRequests.ts index 851fded5..bfe2eebe 100644 --- a/src/lib/requests/approvePrivacyRequests.ts +++ b/src/lib/requests/approvePrivacyRequests.ts @@ -2,19 +2,19 @@ import { RequestAction, RequestOrigin, RequestStatus, -} from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +} from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { APPROVE_PRIVACY_REQUEST, buildTranscendGraphQLClient, fetchAllRequests, makeGraphQLRequest, UPDATE_PRIVACY_REQUEST, -} from '../graphql'; +} from "../graphql"; /** * Approve a set of privacy requests @@ -53,11 +53,11 @@ export async function approvePrivacyRequests({ const client = buildTranscendGraphQLClient(transcendUrl, auth); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Pull in the requests @@ -97,8 +97,8 @@ export async function approvePrivacyRequests({ await makeGraphQLRequest(client, APPROVE_PRIVACY_REQUEST, { input: { requestId: requestToApprove.id }, }); - } catch (err) { - if (err.message.includes('Request must be in an approving state,')) { + } catch (error) { + if (error.message.includes("Request must be in an approving state,")) { skipped += 1; } } @@ -106,11 +106,11 @@ export async function approvePrivacyRequests({ total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; if (skipped > 0) { logger.info(colors.yellow(`${skipped} requests were skipped.`)); @@ -119,8 +119,8 @@ export async function approvePrivacyRequests({ colors.green( `Successfully approved ${total} requests in "${ totalTime / 1000 - }" seconds!`, - ), + }" seconds!` + ) ); return allRequests.length; } diff --git a/src/lib/requests/bulkRestartRequests.ts b/src/lib/requests/bulkRestartRequests.ts index 8f4b4f2f..b022421b 100644 --- a/src/lib/requests/bulkRestartRequests.ts +++ b/src/lib/requests/bulkRestartRequests.ts @@ -1,22 +1,22 @@ -import { join } from 'path'; -import { PersistedState } from '@transcend-io/persisted-state'; -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import * as t from 'io-ts'; -import { difference } from 'lodash-es'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { join } from "node:path"; +import { PersistedState } from "@transcend-io/persisted-state"; +import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import * as t from "io-ts"; +import { difference } from "lodash-es"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequestIdentifiers, fetchAllRequests, -} from '../graphql'; -import { SuccessfulRequest } from './constants'; -import { extractClientError } from './extractClientError'; -import { restartPrivacyRequest } from './restartPrivacyRequest'; +} from "../graphql"; +import { SuccessfulRequest } from "./constants"; +import { extractClientError } from "./extractClientError"; +import { restartPrivacyRequest } from "./restartPrivacyRequest"; /** Minimal state we need to keep a list of requests */ const ErrorRequest = t.intersection([ @@ -92,17 +92,17 @@ export async function bulkRestartRequests({ concurrency?: number; }): Promise { // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Create a new state file to store the requests from this run const cacheFile = join( requestReceiptFolder, - `tr-request-restart-${new Date().toISOString()}`, + `tr-request-restart-${new Date().toISOString()}` ); const state = new PersistedState(cacheFile, CachedRequestState, { restartedRequests: [], @@ -122,33 +122,33 @@ export async function bulkRestartRequests({ createdAtAfter, }); const requests = allRequests.filter( - (request) => new Date(request.createdAt) < createdAt, + (request) => new Date(request.createdAt) < createdAt ); logger.info(`Found ${requests.length} requests to process`); if (copyIdentifiers) { - logger.info('copyIdentifiers detected - All Identifiers will be copied.'); + logger.info("copyIdentifiers detected - All Identifiers will be copied."); } if (sendEmailReceipt) { - logger.info('sendEmailReceipt detected - Email receipts will be sent.'); + logger.info("sendEmailReceipt detected - Email receipts will be sent."); } if (skipWaitingPeriod) { - logger.info('skipWaitingPeriod detected - Waiting period will be skipped.'); + logger.info("skipWaitingPeriod detected - Waiting period will be skipped."); } // Validate request IDs if (requestIds.length > 0 && requestIds.length !== requests.length) { const missingRequests = difference( requestIds, - requests.map(({ id }) => id), + requests.map(({ id }) => id) ); if (missingRequests.length > 0) { logger.error( colors.red( `Failed to find the following requests by ID: ${missingRequests.join( - ',', - )}.`, - ), + "," + )}.` + ) ); process.exit(1); } @@ -185,11 +185,11 @@ export async function bulkRestartRequests({ skipWaitingPeriod, sendEmailReceipt, emailIsVerified, - }, + } ); // Cache successful upload - const restartedRequests = state.getValue('restartedRequests'); + const restartedRequests = state.getValue("restartedRequests"); restartedRequests.push({ id: requestResponse.id, link: requestResponse.link, @@ -197,50 +197,50 @@ export async function bulkRestartRequests({ coreIdentifier: requestResponse.coreIdentifier, attemptedAt: new Date().toISOString(), }); - await state.setValue(restartedRequests, 'restartedRequests'); - } catch (err) { - const msg = `${err.message} - ${JSON.stringify( - err.response?.body, + await state.setValue(restartedRequests, "restartedRequests"); + } catch (error) { + const message = `${error.message} - ${JSON.stringify( + error.response?.body, null, - 2, + 2 )}`; - const clientError = extractClientError(msg); + const clientError = extractClientError(message); - const failingRequests = state.getValue('failingRequests'); + const failingRequests = state.getValue("failingRequests"); failingRequests.push({ id: request.id, link: request.link, rowIndex: ind, coreIdentifier: request.coreIdentifier, attemptedAt: new Date().toISOString(), - error: clientError || msg, + error: clientError || message, }); - await state.setValue(failingRequests, 'failingRequests'); + await state.setValue(failingRequests, "failingRequests"); } total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; // Log completion time logger.info( colors.green( - `Completed restarting of requests in "${totalTime / 1000}" seconds.`, - ), + `Completed restarting of requests in "${totalTime / 1000}" seconds.` + ) ); // Log errors - if (state.getValue('failingRequests').length > 0) { + if (state.getValue("failingRequests").length > 0) { logger.error( colors.red( - `Encountered "${state.getValue('failingRequests').length}" errors. ` + - `See "${cacheFile}" to review the error messages and inputs.`, - ), + `Encountered "${state.getValue("failingRequests").length}" errors. ` + + `See "${cacheFile}" to review the error messages and inputs.` + ) ); process.exit(1); } diff --git a/src/lib/requests/bulkRetryEnrichers.ts b/src/lib/requests/bulkRetryEnrichers.ts index e101d8c1..7e83e71f 100644 --- a/src/lib/requests/bulkRetryEnrichers.ts +++ b/src/lib/requests/bulkRetryEnrichers.ts @@ -2,19 +2,19 @@ import { RequestAction, RequestEnricherStatus, RequestStatus, -} from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { difference } from 'lodash-es'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +} from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { difference } from "lodash-es"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, fetchAllRequestEnrichers, fetchAllRequests, retryRequestEnricher, -} from '../graphql'; +} from "../graphql"; /** * Restart a bunch of request enrichers @@ -52,17 +52,17 @@ export async function bulkRetryEnrichers({ concurrency?: number; }): Promise { // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Find all requests made before createdAt that are in a removing data state const client = buildTranscendGraphQLClient(transcendUrl, auth); - logger.info(colors.magenta('Fetching requests to restart...')); + logger.info(colors.magenta("Fetching requests to restart...")); const requests = await fetchAllRequests(client, { actions: requestActions, @@ -78,15 +78,15 @@ export async function bulkRetryEnrichers({ if (requestIds.length > 0 && requestIds.length !== requests.length) { const missingRequests = difference( requestIds, - requests.map(({ id }) => id), + requests.map(({ id }) => id) ); if (missingRequests.length > 0) { logger.error( colors.red( `Failed to find the following requests by ID: ${missingRequests.join( - ',', - )}.`, - ), + "," + )}.` + ) ); process.exit(1); } @@ -105,7 +105,7 @@ export async function bulkRetryEnrichers({ const requestEnrichersToRestart = requestEnrichers.filter( (requestEnricher) => requestEnricher.enricher.id === enricherId && - requestEnricherStatuses.includes(requestEnricher.status), + requestEnricherStatuses.includes(requestEnricher.status) ); await map(requestEnrichersToRestart, async (requestEnricher) => { await retryRequestEnricher(client, requestEnricher.id); @@ -116,11 +116,11 @@ export async function bulkRetryEnrichers({ total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; // Log completion time @@ -130,7 +130,7 @@ export async function bulkRetryEnrichers({ requests.length } requests and ${totalRestarted} enrichers in "${ totalTime / 1000 - }" seconds.`, - ), + }" seconds.` + ) ); } diff --git a/src/lib/requests/cancelPrivacyRequests.ts b/src/lib/requests/cancelPrivacyRequests.ts index 8f5f96f6..e3b43a6b 100644 --- a/src/lib/requests/cancelPrivacyRequests.ts +++ b/src/lib/requests/cancelPrivacyRequests.ts @@ -1,9 +1,9 @@ -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, CANCEL_PRIVACY_REQUEST, @@ -12,7 +12,7 @@ import { makeGraphQLRequest, Template, UPDATE_PRIVACY_REQUEST, -} from '../graphql'; +} from "../graphql"; /** * Cancel a set of privacy requests @@ -66,11 +66,11 @@ export async function cancelPrivacyRequests({ const client = buildTranscendGraphQLClient(transcendUrl, auth); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Grab the template with that title @@ -78,14 +78,14 @@ export async function cancelPrivacyRequests({ if (cancellationTitle) { const matchingTemplates = await fetchAllTemplates( client, - cancellationTitle, + cancellationTitle ); const exactTitleMatch = matchingTemplates.find( - (template) => template.title === cancellationTitle, + (template) => template.title === cancellationTitle ); if (!exactTitleMatch) { throw new Error( - `Failed to find a template with title: "${cancellationTitle}"`, + `Failed to find a template with title: "${cancellationTitle}"` ); } cancelationTemplate = exactTitleMatch; @@ -106,9 +106,9 @@ export async function cancelPrivacyRequests({ `Canceling "${allRequests.length}" requests${ cancelationTemplate ? ` Using template: ${cancelationTemplate.title}` - : '' - }.`, - ), + : "" + }.` + ) ); let total = 0; @@ -146,19 +146,19 @@ export async function cancelPrivacyRequests({ total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully canceled ${total} requests in "${ totalTime / 1000 - }" seconds!`, - ), + }" seconds!` + ) ); return allRequests.length; } diff --git a/src/lib/requests/constants.ts b/src/lib/requests/constants.ts index 716296da..3e785dd8 100644 --- a/src/lib/requests/constants.ts +++ b/src/lib/requests/constants.ts @@ -1,48 +1,48 @@ -import { LanguageKey } from '@transcend-io/internationalization'; +import { LanguageKey } from "@transcend-io/internationalization"; import { CompletedRequestStatus, IsoCountryCode, IsoCountrySubdivisionCode, RequestAction, -} from '@transcend-io/privacy-types'; -import { applyEnum, valuesOf } from '@transcend-io/type-utils'; -import * as t from 'io-ts'; +} from "@transcend-io/privacy-types"; +import { applyEnum, valuesOf } from "@transcend-io/type-utils"; +import * as t from "io-ts"; -export const NONE = '[NONE]' as const; -export const BULK_APPLY = '[APPLY VALUE TO ALL ROWS]' as const; -export const BLANK = '' as const; +export const NONE = "[NONE]" as const; +export const BULK_APPLY = "[APPLY VALUE TO ALL ROWS]" as const; +export const BLANK = "" as const; /** These are uploaded at the top level of the request */ -export const IDENTIFIER_BLOCK_LIST = ['email', 'coreIdentifier']; +export const IDENTIFIER_BLOCK_LIST = ["email", "coreIdentifier"]; /** * Column names to map */ export enum ColumnName { /** The title of the email column */ - Email = 'email', + Email = "email", /** The title of the core identifier column */ - CoreIdentifier = 'coreIdentifier', + CoreIdentifier = "coreIdentifier", /** The title of the requestType column */ - RequestType = 'requestType', + RequestType = "requestType", /** The title of the subjectType column */ - SubjectType = 'subjectType', + SubjectType = "subjectType", /** The title of the locale column */ - Locale = 'locale', + Locale = "locale", /** The country */ - Country = 'country', + Country = "country", /** The country sub division */ - CountrySubDivision = 'countrySubDivision', + CountrySubDivision = "countrySubDivision", /** The title of the requestStatus column */ - RequestStatus = 'requestStatus', + RequestStatus = "requestStatus", /** The title of the createdAt column */ - CreatedAt = 'createdAt', + CreatedAt = "createdAt", /** The title of the dataSiloIds column */ - DataSiloIds = 'dataSiloIds', + DataSiloIds = "dataSiloIds", } /** These parameters are required in the Transcend DSR API */ -export const IS_REQUIRED: { [k in ColumnName]: boolean } = { +export const IS_REQUIRED: Record = { [ColumnName.Email]: false, [ColumnName.CoreIdentifier]: true, [ColumnName.RequestType]: true, @@ -56,7 +56,7 @@ export const IS_REQUIRED: { [k in ColumnName]: boolean } = { }; /** These parameters can be specified for the entire CSV set if needed */ -export const CAN_APPLY_IN_BULK: { [k in ColumnName]?: boolean } = { +export const CAN_APPLY_IN_BULK: Partial> = { [ColumnName.RequestType]: true, [ColumnName.SubjectType]: true, }; @@ -78,17 +78,17 @@ export const CachedFileState = t.type({ /** Mapping between region and country code */ regionToCountry: t.record( t.string, - valuesOf({ ...IsoCountryCode, [NONE]: NONE }), + valuesOf({ ...IsoCountryCode, [NONE]: NONE }) ), /** Mapping between region and country sub division code */ regionToCountrySubDivision: t.record( t.string, - valuesOf({ ...IsoCountrySubdivisionCode, [NONE]: NONE }), + valuesOf({ ...IsoCountrySubdivisionCode, [NONE]: NONE }) ), /** Mapping between request status in import to Transcend request status */ statusToRequestStatus: t.record( t.string, - valuesOf({ ...CompletedRequestStatus, [NONE]: NONE }), + valuesOf({ ...CompletedRequestStatus, [NONE]: NONE }) ), }); @@ -121,7 +121,7 @@ export const CachedRequestState = t.type({ rowIndex: t.number, coreIdentifier: t.string, attemptedAt: t.string, - }), + }) ), }); diff --git a/src/lib/requests/downloadPrivacyRequestFiles.ts b/src/lib/requests/downloadPrivacyRequestFiles.ts index 2bd300bf..e5771a85 100644 --- a/src/lib/requests/downloadPrivacyRequestFiles.ts +++ b/src/lib/requests/downloadPrivacyRequestFiles.ts @@ -1,20 +1,20 @@ -import { existsSync, mkdirSync, writeFileSync } from 'fs'; -import { dirname, join } from 'path'; -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { APPROVE_PRIVACY_REQUEST, buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequests, makeGraphQLRequest, -} from '../graphql'; -import { getFileMetadataForPrivacyRequests } from './getFileMetadataForPrivacyRequests'; -import { streamPrivacyRequestFiles } from './streamPrivacyRequestFiles'; +} from "../graphql"; +import { getFileMetadataForPrivacyRequests } from "./getFileMetadataForPrivacyRequests"; +import { streamPrivacyRequestFiles } from "./streamPrivacyRequestFiles"; /** * Download a set of privacy requests to disk @@ -81,14 +81,14 @@ export async function downloadPrivacyRequestFiles({ { sombra, concurrency, - }, + } ); // Start timer for download process - const t0 = new Date().getTime(); + const t0 = Date.now(); const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); let total = 0; let totalApproved = 0; @@ -134,23 +134,23 @@ export async function downloadPrivacyRequestFiles({ total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully downloaded ${total} requests in "${ totalTime / 1000 - }" seconds!`, - ), + }" seconds!` + ) ); if (totalApproved > 0) { logger.info( - colors.green(`Approved ${totalApproved} requests in Transcend.`), + colors.green(`Approved ${totalApproved} requests in Transcend.`) ); } return allRequests.length; diff --git a/src/lib/requests/extractClientError.ts b/src/lib/requests/extractClientError.ts index 8f7b85b3..41eb2e58 100644 --- a/src/lib/requests/extractClientError.ts +++ b/src/lib/requests/extractClientError.ts @@ -6,6 +6,6 @@ const CLIENT_ERROR = /{\\"message\\":\\"(.+?)\\",/; * @param err - Error message * @returns Client error or null */ -export function extractClientError(err: string): string | null { - return CLIENT_ERROR.test(err) ? CLIENT_ERROR.exec(err)![1] : null; +export function extractClientError(error: string): string | null { + return CLIENT_ERROR.test(error) ? CLIENT_ERROR.exec(error)![1] : null; } diff --git a/src/lib/requests/filterRows.ts b/src/lib/requests/filterRows.ts index a68f68a4..9cb62f30 100644 --- a/src/lib/requests/filterRows.ts +++ b/src/lib/requests/filterRows.ts @@ -1,10 +1,10 @@ -import { ObjByString } from '@transcend-io/type-utils'; -import colors from 'colors'; -import inquirer from 'inquirer'; -import { uniq } from 'lodash-es'; -import { logger } from '../../logger'; -import { NONE } from './constants'; -import { getUniqueValuesForColumn } from './getUniqueValuesForColumn'; +import { ObjByString } from "@transcend-io/type-utils"; +import colors from "colors"; +import inquirer from "inquirer"; +import { uniq } from "lodash-es"; +import { logger } from "../../logger"; +import { NONE } from "./constants"; +import { getUniqueValuesForColumn } from "./getUniqueValuesForColumn"; /** * Filter a list of CSV rows by column values @@ -15,7 +15,7 @@ import { getUniqueValuesForColumn } from './getUniqueValuesForColumn'; */ export async function filterRows(rows: ObjByString[]): Promise { // Determine set of column names - const columnNames = uniq(rows.map((x) => Object.keys(x)).flat()); + const columnNames = uniq(rows.flatMap((x) => Object.keys(x))); // update these variables recursively let filteredRows = rows; @@ -30,10 +30,10 @@ export async function filterRows(rows: ObjByString[]): Promise { filterColumnName: string; }>([ { - name: 'filterColumnName', - // eslint-disable-next-line max-len + name: "filterColumnName", + message: `If you need to filter the list of requests to import, choose the column to filter on. Currently ${filteredRows.length} rows.`, - type: 'list', + type: "list", default: columnNames, choices: [NONE, ...columnNames], }, @@ -49,16 +49,16 @@ export async function filterRows(rows: ObjByString[]): Promise { valuesToKeep: string[]; }>([ { - name: 'valuesToKeep', - message: 'Keep rows matching this value', - type: 'checkbox', + name: "valuesToKeep", + message: "Keep rows matching this value", + type: "checkbox", default: columnNames, choices: options, }, ]); filteredRows = filteredRows.filter((request) => - valuesToKeep.includes(request[filterColumnName]), + valuesToKeep.includes(request[filterColumnName]) ); } } diff --git a/src/lib/requests/getFileMetadataForPrivacyRequests.ts b/src/lib/requests/getFileMetadataForPrivacyRequests.ts index c795e0d8..c74a98d2 100644 --- a/src/lib/requests/getFileMetadataForPrivacyRequests.ts +++ b/src/lib/requests/getFileMetadataForPrivacyRequests.ts @@ -1,12 +1,12 @@ -import { TableEncryptionType } from '@transcend-io/privacy-types'; -import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import type { Got } from 'got'; -import * as t from 'io-ts'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; -import { PrivacyRequest } from '../graphql'; +import { TableEncryptionType } from "@transcend-io/privacy-types"; +import { decodeCodec, valuesOf } from "@transcend-io/type-utils"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import type { Got } from "got"; +import * as t from "io-ts"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; +import { PrivacyRequest } from "../graphql"; export const IntlMessage = t.type({ /** The message key */ @@ -93,7 +93,7 @@ export type RequestFileMetadataResponse = t.TypeOf< * @returns The number of requests canceled */ export async function getFileMetadataForPrivacyRequests( - requests: Pick[], + requests: Pick[], { sombra, concurrency = 5, @@ -105,18 +105,18 @@ export async function getFileMetadataForPrivacyRequests( limit?: number; /** Concurrency limit for approving */ concurrency?: number; - }, -): Promise<[Pick, RequestFileMetadata[]][]> { + } +): Promise<[Pick, RequestFileMetadata[]][]> { logger.info( - colors.magenta(`Pulling file metadata for ${requests.length} requests`), + colors.magenta(`Pulling file metadata for ${requests.length} requests`) ); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Start timer @@ -127,9 +127,9 @@ export async function getFileMetadataForPrivacyRequests( const results = await map( requests, async ( - requestToDownload, + requestToDownload ): Promise< - [Pick, RequestFileMetadata[]] + [Pick, RequestFileMetadata[]] > => { const localResults: RequestFileMetadata[] = []; @@ -149,7 +149,7 @@ export async function getFileMetadataForPrivacyRequests( limit, offset, }, - }, + } ) .json(); response = decodeCodec(RequestFileMetadataResponse, rawResponse); @@ -158,13 +158,12 @@ export async function getFileMetadataForPrivacyRequests( // Increase offset and break if no more pages offset += limit; shouldContinue = - // eslint-disable-next-line no-underscore-dangle !!response._links.next && response.nodes.length === limit; - } catch (err) { + } catch (error) { throw new Error( `Received an error from server: ${ - err?.response?.body || err?.message - }`, + error?.response?.body || error?.message + }` ); } } @@ -173,19 +172,19 @@ export async function getFileMetadataForPrivacyRequests( progressBar.update(total); return [requestToDownload, localResults]; }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully downloaded file metadata ${requests.length} requests in "${ totalTime / 1000 - }" seconds!`, - ), + }" seconds!` + ) ); return results; diff --git a/src/lib/requests/getUniqueValuesForColumn.ts b/src/lib/requests/getUniqueValuesForColumn.ts index 731210ca..fb4f56ee 100644 --- a/src/lib/requests/getUniqueValuesForColumn.ts +++ b/src/lib/requests/getUniqueValuesForColumn.ts @@ -1,5 +1,5 @@ -import { ObjByString } from '@transcend-io/type-utils'; -import { uniq } from 'lodash-es'; +import { ObjByString } from "@transcend-io/type-utils"; +import { uniq } from "lodash-es"; /** * Return the unique set of values for a column in a CSV @@ -10,7 +10,7 @@ import { uniq } from 'lodash-es'; */ export function getUniqueValuesForColumn( rows: ObjByString[], - columnName: string, + columnName: string ): string[] { - return uniq(rows.map((row) => row[columnName] || '').flat()); + return uniq(rows.flatMap((row) => row[columnName] || "")); } diff --git a/src/lib/requests/mapColumnsToAttributes.ts b/src/lib/requests/mapColumnsToAttributes.ts index c54293f3..3697d0da 100644 --- a/src/lib/requests/mapColumnsToAttributes.ts +++ b/src/lib/requests/mapColumnsToAttributes.ts @@ -1,16 +1,14 @@ -import type { PersistedState } from '@transcend-io/persisted-state'; -import type { GraphQLClient } from 'graphql-request'; -import inquirer from 'inquirer'; -import { AttributeKey } from '../graphql'; -import { CachedFileState } from './constants'; -import { fuzzyMatchColumns } from './fuzzyMatchColumns'; +import type { PersistedState } from "@transcend-io/persisted-state"; +import type { GraphQLClient } from "graphql-request"; +import inquirer from "inquirer"; +import { AttributeKey } from "../graphql"; +import { CachedFileState } from "./constants"; +import { fuzzyMatchColumns } from "./fuzzyMatchColumns"; /** * Mapping from attribute name to request input parameter */ -export type AttributeNameMap = { - [k in string]: string; -}; +export type AttributeNameMap = Record; /** * Create a mapping from the attributes names that can be included @@ -27,11 +25,11 @@ export async function mapColumnsToAttributes( client: GraphQLClient, columnNames: string[], state: PersistedState, - requestAttributeKeys: AttributeKey[], + requestAttributeKeys: AttributeKey[] ): Promise { // Determine the columns that should be mapped const columnQuestions = requestAttributeKeys.filter( - ({ name }) => !state.getValue('attributeNames', name), + ({ name }) => !state.getValue("attributeNames", name) ); // Skip mapping when everything is mapped @@ -39,28 +37,26 @@ export async function mapColumnsToAttributes( columnQuestions.length === 0 ? {} : // prompt questions to map columns - await inquirer.prompt<{ - [k in string]: string; - }>( + await inquirer.prompt>( columnQuestions.map(({ name }) => { const matches = fuzzyMatchColumns(columnNames, name, false); return { name, message: `Choose the column that will be used to map in the attribute: ${name}`, - type: 'list', + type: "list", default: matches[0], choices: matches, }; - }), + }) ); await Promise.all( Object.entries(attributeNameMap).map(([k, v]) => - state.setValue(v, 'attributeNames', k), - ), + state.setValue(v, "attributeNames", k) + ) ); return { - ...state.getValue('attributeNames'), + ...state.getValue("attributeNames"), ...attributeNameMap, }; } diff --git a/src/lib/requests/mapColumnsToIdentifiers.ts b/src/lib/requests/mapColumnsToIdentifiers.ts index e65114ad..214cc888 100644 --- a/src/lib/requests/mapColumnsToIdentifiers.ts +++ b/src/lib/requests/mapColumnsToIdentifiers.ts @@ -1,16 +1,14 @@ -import type { PersistedState } from '@transcend-io/persisted-state'; -import type { GraphQLClient } from 'graphql-request'; -import inquirer from 'inquirer'; -import { INITIALIZER, Initializer, makeGraphQLRequest } from '../graphql'; -import { CachedFileState, IDENTIFIER_BLOCK_LIST } from './constants'; -import { fuzzyMatchColumns } from './fuzzyMatchColumns'; +import type { PersistedState } from "@transcend-io/persisted-state"; +import type { GraphQLClient } from "graphql-request"; +import inquirer from "inquirer"; +import { INITIALIZER, Initializer, makeGraphQLRequest } from "../graphql"; +import { CachedFileState, IDENTIFIER_BLOCK_LIST } from "./constants"; +import { fuzzyMatchColumns } from "./fuzzyMatchColumns"; /** * Mapping from identifier name to request input parameter */ -export type IdentifierNameMap = { - [k in string]: string; -}; +export type IdentifierNameMap = Record; /** * Create a mapping from the identifier names that can be included @@ -25,7 +23,7 @@ export type IdentifierNameMap = { export async function mapColumnsToIdentifiers( client: GraphQLClient, columnNames: string[], - state: PersistedState, + state: PersistedState ): Promise { // Grab the initializer const { initializer } = await makeGraphQLRequest<{ @@ -36,8 +34,8 @@ export async function mapColumnsToIdentifiers( // Determine the columns that should be mapped const columnQuestions = initializer.identifiers.filter( ({ name }) => - !state.getValue('identifierNames', name) && - !IDENTIFIER_BLOCK_LIST.includes(name), + !state.getValue("identifierNames", name) && + !IDENTIFIER_BLOCK_LIST.includes(name) ); // Skip mapping when everything is mapped @@ -45,28 +43,26 @@ export async function mapColumnsToIdentifiers( columnQuestions.length === 0 ? {} : // prompt questions to map columns - await inquirer.prompt<{ - [k in string]: string; - }>( + await inquirer.prompt>( columnQuestions.map(({ name }) => { const matches = fuzzyMatchColumns(columnNames, name, false); return { name, message: `Choose the column that will be used to map in the identifier: ${name}`, - type: 'list', + type: "list", default: matches[0], choices: matches, }; - }), + }) ); await Promise.all( Object.entries(identifierNameMap).map(([k, v]) => - state.setValue(v, 'identifierNames', k), - ), + state.setValue(v, "identifierNames", k) + ) ); return { - ...state.getValue('identifierNames'), + ...state.getValue("identifierNames"), ...identifierNameMap, }; } diff --git a/src/lib/requests/mapCsvColumnsToApi.ts b/src/lib/requests/mapCsvColumnsToApi.ts index 54f0e794..76c1b0d1 100644 --- a/src/lib/requests/mapCsvColumnsToApi.ts +++ b/src/lib/requests/mapCsvColumnsToApi.ts @@ -1,21 +1,19 @@ -import type { PersistedState } from '@transcend-io/persisted-state'; -import { getEntries, getValues } from '@transcend-io/type-utils'; -import inquirer from 'inquirer'; -import { startCase } from 'lodash-es'; +import type { PersistedState } from "@transcend-io/persisted-state"; +import { getEntries, getValues } from "@transcend-io/type-utils"; +import inquirer from "inquirer"; +import { startCase } from "lodash-es"; import { CachedFileState, CAN_APPLY_IN_BULK, ColumnName, IS_REQUIRED, -} from './constants'; -import { fuzzyMatchColumns } from './fuzzyMatchColumns'; +} from "./constants"; +import { fuzzyMatchColumns } from "./fuzzyMatchColumns"; /** * Mapping from column name to request input parameter */ -export type ColumnNameMap = { - [k in ColumnName]?: string; -}; +export type ColumnNameMap = Partial>; /** * Determine the mapping between columns in CSV @@ -26,11 +24,11 @@ export type ColumnNameMap = { */ export async function mapCsvColumnsToApi( columnNames: string[], - state: PersistedState, + state: PersistedState ): Promise { // Determine the columns that should be mapped const columnQuestions = getValues(ColumnName).filter( - (name) => !state.getValue('columnNames', name), + (name) => !state.getValue("columnNames", name) ); // Skip mapping when everything is mapped @@ -38,31 +36,29 @@ export async function mapCsvColumnsToApi( columnQuestions.length === 0 ? {} : // prompt questions to map columns - await inquirer.prompt<{ - [k in ColumnName]?: string; - }>( + await inquirer.prompt>>( columnQuestions.map((name) => { - const field = startCase(name.replace('ColumnName', '')); + const field = startCase(name.replace("ColumnName", "")); const matches = fuzzyMatchColumns( columnNames, field, IS_REQUIRED[name], - !!CAN_APPLY_IN_BULK[name], + !!CAN_APPLY_IN_BULK[name] ); return { name, message: `Choose the column that will be used to map in the field: ${field}`, - type: 'list', + type: "list", default: matches[0], choices: matches, }; - }), + }) ); await Promise.all( getEntries(columnNameMap).map(([k, v]) => - state.setValue(v, 'columnNames', k), - ), + state.setValue(v, "columnNames", k) + ) ); return columnNameMap; } diff --git a/src/lib/requests/mapCsvRowsToRequestInputs.ts b/src/lib/requests/mapCsvRowsToRequestInputs.ts index d014b40e..125ad974 100644 --- a/src/lib/requests/mapCsvRowsToRequestInputs.ts +++ b/src/lib/requests/mapCsvRowsToRequestInputs.ts @@ -1,5 +1,5 @@ -import { LanguageKey } from '@transcend-io/internationalization'; -import type { PersistedState } from '@transcend-io/persisted-state'; +import { LanguageKey } from "@transcend-io/internationalization"; +import type { PersistedState } from "@transcend-io/persisted-state"; import { CompletedRequestStatus, IdentifierType, @@ -7,23 +7,23 @@ import { IsoCountrySubdivisionCode, NORMALIZE_PHONE_NUMBER, RequestAction, -} from '@transcend-io/privacy-types'; -import { ObjByString, valuesOf } from '@transcend-io/type-utils'; -import * as t from 'io-ts'; -import { DateFromISOString } from 'io-ts-types'; -import { AttributeKey } from '../graphql'; +} from "@transcend-io/privacy-types"; +import { ObjByString, valuesOf } from "@transcend-io/type-utils"; +import * as t from "io-ts"; +import { DateFromISOString } from "io-ts-types"; +import { AttributeKey } from "../graphql"; import { BLANK, BULK_APPLY, CachedFileState, ColumnName, NONE, -} from './constants'; -import { AttributeNameMap } from './mapColumnsToAttributes'; -import { IdentifierNameMap } from './mapColumnsToIdentifiers'; -import { ColumnNameMap } from './mapCsvColumnsToApi'; -import { ParsedAttributeInput } from './parseAttributesFromString'; -import { splitCsvToList } from './splitCsvToList'; +} from "./constants"; +import { AttributeNameMap } from "./mapColumnsToAttributes"; +import { IdentifierNameMap } from "./mapColumnsToIdentifiers"; +import { ColumnNameMap } from "./mapCsvColumnsToApi"; +import { ParsedAttributeInput } from "./parseAttributesFromString"; +import { splitCsvToList } from "./splitCsvToList"; /** * Shape of additional identifiers @@ -42,8 +42,8 @@ export const AttestedExtraIdentifiers = t.record( /** Name of identifier - option for non-custom identifier types */ name: t.string, }), - ]), - ), + ]) + ) ); /** Type override */ @@ -96,7 +96,7 @@ export type PrivacyRequestInput = t.TypeOf; export function normalizeIdentifierValue( identifierValue: string, identifierType: IdentifierType, - defaultPhoneCountryCode: string, + defaultPhoneCountryCode: string ): string { // Lowercase email if (identifierType === IdentifierType.Email) { @@ -106,17 +106,17 @@ export function normalizeIdentifierValue( // Normalize phone number if (identifierType === IdentifierType.Phone) { const normalized = identifierValue - .replace(NORMALIZE_PHONE_NUMBER, '') - .replace(/[()]/g, '') - .replace(/[–]/g, '') - .replace(/[:]/g, '') - .replace(/[‭‬]/g, '') - .replace(/[A-Za-z]/g, ''); - return !normalized - ? '' - : normalized.startsWith('+') + .replace(NORMALIZE_PHONE_NUMBER, "") + .replaceAll(/[()]/g, "") + .replaceAll(/[–]/g, "") + .replaceAll(/[:]/g, "") + .replaceAll(/[‭‬]/g, "") + .replaceAll(/[A-Za-z]/g, ""); + return normalized + ? normalized.startsWith("+") ? normalized - : `+${defaultPhoneCountryCode}${normalized}`; + : `+${defaultPhoneCountryCode}${normalized}` + : ""; } return identifierValue; } @@ -139,7 +139,7 @@ export function mapCsvRowsToRequestInputs( identifierNameMap, attributeNameMap, requestAttributeKeys, - defaultPhoneCountryCode = '1', // US + defaultPhoneCountryCode = "1", // US }: { /** Default country code */ defaultPhoneCountryCode?: string; @@ -151,70 +151,71 @@ export function mapCsvRowsToRequestInputs( attributeNameMap: AttributeNameMap; /** Request attribute keys */ requestAttributeKeys: AttributeKey[]; - }, + } ): [Record, PrivacyRequestInput][] { // map the CSV to request input const getMappedName = (attribute: ColumnName): string => - state.getValue('columnNames', attribute) || columnNameMap[attribute]!; + state.getValue("columnNames", attribute) || columnNameMap[attribute]!; return requestInputs.map( (input): [Record, PrivacyRequestInput] => { // The extra identifiers to upload for this request const attestedExtraIdentifiers: AttestedExtraIdentifiers = {}; - Object.entries(identifierNameMap) + for (const [identifierName, columnName] of Object.entries( + identifierNameMap + ) // filter out skipped identifiers - .filter(([, columnName]) => columnName !== NONE) - .forEach(([identifierName, columnName]) => { - // Determine the identifier type being specified - const identifierType = Object.values(IdentifierType).includes( - identifierName as any, // eslint-disable-line @typescript-eslint/no-explicit-any - ) - ? (identifierName as IdentifierType) - : IdentifierType.Custom; - - // Only add the identifier if the value exists - const identifierValue = input[columnName]; - if (identifierValue) { - const normalized = normalizeIdentifierValue( - identifierValue, - identifierType, - defaultPhoneCountryCode, - ); - if (normalized) { - // Initialize - if (!attestedExtraIdentifiers[identifierType]) { - attestedExtraIdentifiers[identifierType] = []; - } + .filter(([, columnName]) => columnName !== NONE)) { + // Determine the identifier type being specified + const identifierType = Object.values(IdentifierType).includes( + identifierName as any // eslint-disable-line @typescript-eslint/no-explicit-any + ) + ? (identifierName as IdentifierType) + : IdentifierType.Custom; - // Add the identifier - attestedExtraIdentifiers[identifierType]!.push({ - value: normalized, - name: identifierName, - }); + // Only add the identifier if the value exists + const identifierValue = input[columnName]; + if (identifierValue) { + const normalized = normalizeIdentifierValue( + identifierValue, + identifierType, + defaultPhoneCountryCode + ); + if (normalized) { + // Initialize + if (!attestedExtraIdentifiers[identifierType]) { + attestedExtraIdentifiers[identifierType] = []; } + + // Add the identifier + attestedExtraIdentifiers[identifierType].push({ + value: normalized, + name: identifierName, + }); } - }); + } + } // The extra attributes to upload for this request const attributes: ParsedAttributeInput[] = []; - Object.entries(attributeNameMap) + for (const [attributeName, columnName] of Object.entries(attributeNameMap) // filter out skipped attributes - .filter(([, columnName]) => columnName !== NONE) - .forEach(([attributeName, columnName]) => { - // Only add the identifier if the value exists - const attributeValueString = input[columnName]; - if (attributeValueString) { - // Add the attribute - const isMulti = - requestAttributeKeys.find((attr) => attr.name === attributeName) - ?.type === 'MULTI_SELECT'; - attributes.push({ - values: isMulti - ? splitCsvToList(attributeValueString) - : attributeValueString, - key: attributeName, - }); - } - }); + .filter(([, columnName]) => columnName !== NONE)) { + // Only add the identifier if the value exists + const attributeValueString = input[columnName]; + if (attributeValueString) { + // Add the attribute + const isMulti = + requestAttributeKeys.find( + (attribute) => attribute.name === attributeName + )?.type === "MULTI_SELECT"; + attributes.push({ + values: isMulti + ? splitCsvToList(attributeValueString) + : attributeValueString, + key: attributeName, + }); + } + } const requestTypeColumn = getMappedName(ColumnName.RequestType); const dataSubjectTypeColumn = getMappedName(ColumnName.SubjectType); @@ -227,24 +228,24 @@ export function mapCsvRowsToRequestInputs( coreIdentifier: input[getMappedName(ColumnName.CoreIdentifier)], requestType: requestTypeColumn === BULK_APPLY - ? state.getValue('requestTypeToRequestAction', BLANK) + ? state.getValue("requestTypeToRequestAction", BLANK) : state.getValue( - 'requestTypeToRequestAction', - input[requestTypeColumn], + "requestTypeToRequestAction", + input[requestTypeColumn] ), subjectType: dataSubjectTypeColumn === BULK_APPLY - ? state.getValue('subjectTypeToSubjectName', BLANK) + ? state.getValue("subjectTypeToSubjectName", BLANK) : state.getValue( - 'subjectTypeToSubjectName', - input[dataSubjectTypeColumn], + "subjectTypeToSubjectName", + input[dataSubjectTypeColumn] ), ...(getMappedName(ColumnName.Locale) !== NONE && input[getMappedName(ColumnName.Locale)] ? { locale: state.getValue( - 'languageToLocale', - input[getMappedName(ColumnName.Locale)], + "languageToLocale", + input[getMappedName(ColumnName.Locale)] ), } : {}), @@ -252,8 +253,8 @@ export function mapCsvRowsToRequestInputs( input[getMappedName(ColumnName.Country)] ? { country: state.getValue( - 'regionToCountry', - input[getMappedName(ColumnName.Country)], + "regionToCountry", + input[getMappedName(ColumnName.Country)] ) as IsoCountryCode, } : {}), @@ -261,21 +262,21 @@ export function mapCsvRowsToRequestInputs( input[getMappedName(ColumnName.CountrySubDivision)] ? { countrySubDivision: state.getValue( - 'regionToCountrySubDivision', - input[getMappedName(ColumnName.CountrySubDivision)], + "regionToCountrySubDivision", + input[getMappedName(ColumnName.CountrySubDivision)] ) as IsoCountrySubdivisionCode, } : {}), ...(getMappedName(ColumnName.RequestStatus) !== NONE && state.getValue( - 'statusToRequestStatus', - input[getMappedName(ColumnName.RequestStatus)], + "statusToRequestStatus", + input[getMappedName(ColumnName.RequestStatus)] ) !== NONE && input[getMappedName(ColumnName.RequestStatus)] ? { status: state.getValue( - 'statusToRequestStatus', - input[getMappedName(ColumnName.RequestStatus)], + "statusToRequestStatus", + input[getMappedName(ColumnName.RequestStatus)] ) as CompletedRequestStatus, } : {}), @@ -289,12 +290,12 @@ export function mapCsvRowsToRequestInputs( input[getMappedName(ColumnName.DataSiloIds)] ? { dataSiloIds: splitCsvToList( - input[getMappedName(ColumnName.DataSiloIds)], + input[getMappedName(ColumnName.DataSiloIds)] ), } : {}), }, ]; - }, + } ); } diff --git a/src/lib/requests/mapEnumValues.ts b/src/lib/requests/mapEnumValues.ts index fe85322a..4535a8c1 100644 --- a/src/lib/requests/mapEnumValues.ts +++ b/src/lib/requests/mapEnumValues.ts @@ -1,7 +1,7 @@ -import { apply, ObjByString } from '@transcend-io/type-utils'; -import inquirer from 'inquirer'; -import autoCompletePrompt from 'inquirer-autocomplete-prompt'; -import { fuzzySearch } from './fuzzyMatchColumns'; +import { apply, ObjByString } from "@transcend-io/type-utils"; +import inquirer from "inquirer"; +import autoCompletePrompt from "inquirer-autocomplete-prompt"; +import { fuzzySearch } from "./fuzzyMatchColumns"; /** * Map a set of inputs to a set of outputs @@ -14,34 +14,34 @@ import { fuzzySearch } from './fuzzyMatchColumns'; export async function mapEnumValues( csvInputs: string[], expectedOutputs: TValue[], - cache: { [k in string]: TValue }, -): Promise<{ [k in string]: TValue }> { - inquirer.registerPrompt('autocomplete', autoCompletePrompt); + cache: Record +): Promise> { + inquirer.registerPrompt("autocomplete", autoCompletePrompt); const inputs = csvInputs - .map((item) => item || '') + .map((item) => item || "") .filter((value) => !cache[value]); if (inputs.length === 0) { return cache; } - const result = await inquirer.prompt<{ [k in string]: TValue }>( + const result = await inquirer.prompt>( inputs.map((value) => ({ name: value, message: `Map value of: ${value}`, - type: 'autocomplete', + type: "autocomplete", default: expectedOutputs.find((x) => fuzzySearch(value, x)), source: (answersSoFar: ObjByString, input: string) => - !input - ? expectedOutputs - : expectedOutputs.filter( - (x) => typeof x === 'string' && fuzzySearch(input, x), - ), - })), + input + ? expectedOutputs.filter( + (x) => typeof x === "string" && fuzzySearch(input, x) + ) + : expectedOutputs, + })) ); return { ...cache, ...apply(result, (r) => - typeof r === 'string' ? (r as TValue) : (Object.values(r)[0] as TValue), + typeof r === "string" ? r : (Object.values(r)[0] as TValue) ), }; } diff --git a/src/lib/requests/mapRequestEnumValues.ts b/src/lib/requests/mapRequestEnumValues.ts index 6d047eeb..db5e93f9 100644 --- a/src/lib/requests/mapRequestEnumValues.ts +++ b/src/lib/requests/mapRequestEnumValues.ts @@ -1,20 +1,20 @@ -import { LanguageKey } from '@transcend-io/internationalization'; -import type { PersistedState } from '@transcend-io/persisted-state'; +import { LanguageKey } from "@transcend-io/internationalization"; +import type { PersistedState } from "@transcend-io/persisted-state"; import { CompletedRequestStatus, IsoCountryCode, IsoCountrySubdivisionCode, RequestAction, -} from '@transcend-io/privacy-types'; -import { ObjByString } from '@transcend-io/type-utils'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { logger } from '../../logger'; -import { DATA_SUBJECTS, DataSubject, makeGraphQLRequest } from '../graphql'; -import { CachedFileState, ColumnName, NONE } from './constants'; -import { getUniqueValuesForColumn } from './getUniqueValuesForColumn'; -import { ColumnNameMap } from './mapCsvColumnsToApi'; -import { mapEnumValues } from './mapEnumValues'; +} from "@transcend-io/privacy-types"; +import { ObjByString } from "@transcend-io/type-utils"; +import colors from "colors"; +import { GraphQLClient } from "graphql-request"; +import { logger } from "../../logger"; +import { DATA_SUBJECTS, DataSubject, makeGraphQLRequest } from "../graphql"; +import { CachedFileState, ColumnName, NONE } from "./constants"; +import { getUniqueValuesForColumn } from "./getUniqueValuesForColumn"; +import { ColumnNameMap } from "./mapCsvColumnsToApi"; +import { mapEnumValues } from "./mapEnumValues"; /** * Map the values in a CSV to the enum values in Transcend @@ -34,11 +34,11 @@ export async function mapRequestEnumValues( state: PersistedState; /** Mapping of column names */ columnNameMap: ColumnNameMap; - }, + } ): Promise { // Get mapped value const getMappedName = (attribute: ColumnName): string => - state.getValue('columnNames', attribute) || columnNameMap[attribute]!; + state.getValue("columnNames", attribute) || columnNameMap[attribute]!; // Fetch all data subjects in the organization const { internalSubjects } = await makeGraphQLRequest<{ @@ -48,90 +48,89 @@ export async function mapRequestEnumValues( // Map RequestAction logger.info( - colors.magenta('Determining mapping of columns for request action'), + colors.magenta("Determining mapping of columns for request action") ); - const requestTypeToRequestAction: { [k in string]: RequestAction } = + const requestTypeToRequestAction: Record = await mapEnumValues( getUniqueValuesForColumn(requests, getMappedName(ColumnName.RequestType)), Object.values(RequestAction), - state.getValue('requestTypeToRequestAction'), + state.getValue("requestTypeToRequestAction") ); await state.setValue( requestTypeToRequestAction, - 'requestTypeToRequestAction', + "requestTypeToRequestAction" ); // Map data subject type - logger.info(colors.magenta('Determining mapping of columns for subject')); - const subjectTypeToSubjectName: { [k in string]: string } = - await mapEnumValues( - getUniqueValuesForColumn(requests, getMappedName(ColumnName.SubjectType)), - internalSubjects.map(({ type }) => type), - state.getValue('subjectTypeToSubjectName'), - ); - await state.setValue(subjectTypeToSubjectName, 'subjectTypeToSubjectName'); + logger.info(colors.magenta("Determining mapping of columns for subject")); + const subjectTypeToSubjectName: Record = await mapEnumValues( + getUniqueValuesForColumn(requests, getMappedName(ColumnName.SubjectType)), + internalSubjects.map(({ type }) => type), + state.getValue("subjectTypeToSubjectName") + ); + await state.setValue(subjectTypeToSubjectName, "subjectTypeToSubjectName"); // Map locale - logger.info(colors.magenta('Determining mapping of columns for locale')); - const languageToLocale: { [k in string]: LanguageKey } = await mapEnumValues( + logger.info(colors.magenta("Determining mapping of columns for locale")); + const languageToLocale: Record = await mapEnumValues( getUniqueValuesForColumn(requests, getMappedName(ColumnName.Locale)), Object.values(LanguageKey), - state.getValue('languageToLocale'), + state.getValue("languageToLocale") ); - await state.setValue(languageToLocale, 'languageToLocale'); + await state.setValue(languageToLocale, "languageToLocale"); logger.info( - colors.magenta('Determining mapping of columns for request status'), + colors.magenta("Determining mapping of columns for request status") ); // Map request status logger.info( - colors.magenta('Determining mapping of columns for request status'), + colors.magenta("Determining mapping of columns for request status") ); const requestStatusColumn = getMappedName(ColumnName.RequestStatus); - const statusToRequestStatus: { - [k in string]: CompletedRequestStatus | typeof NONE; - } = + const statusToRequestStatus: Record< + string, + CompletedRequestStatus | typeof NONE + > = requestStatusColumn === NONE ? {} : await mapEnumValues( getUniqueValuesForColumn(requests, requestStatusColumn), [...Object.values(CompletedRequestStatus), NONE], - state.getValue('statusToRequestStatus'), + state.getValue("statusToRequestStatus") ); - await state.setValue(statusToRequestStatus, 'statusToRequestStatus'); + await state.setValue(statusToRequestStatus, "statusToRequestStatus"); // Map country - logger.info(colors.magenta('Determining mapping of columns for country')); + logger.info(colors.magenta("Determining mapping of columns for country")); const countryColumn = getMappedName(ColumnName.Country); - const regionToCountry: { - [k in string]: IsoCountryCode | typeof NONE; - } = + const regionToCountry: Record = countryColumn === NONE ? {} : await mapEnumValues( getUniqueValuesForColumn(requests, countryColumn), [...Object.values(IsoCountryCode), NONE], - state.getValue('regionToCountry'), + state.getValue("regionToCountry") ); - await state.setValue(regionToCountry, 'regionToCountry'); + await state.setValue(regionToCountry, "regionToCountry"); // Map country sub division logger.info( - colors.magenta('Determining mapping of columns for country sub division'), + colors.magenta("Determining mapping of columns for country sub division") ); const countrySubDivisionColumn = getMappedName(ColumnName.CountrySubDivision); - const regionToCountrySubDivision: { - [k in string]: IsoCountrySubdivisionCode | typeof NONE; - } = + const regionToCountrySubDivision: Record< + string, + IsoCountrySubdivisionCode | typeof NONE + > = countrySubDivisionColumn === NONE ? {} : await mapEnumValues( getUniqueValuesForColumn(requests, countrySubDivisionColumn), [...Object.values(IsoCountrySubdivisionCode), NONE], - state.getValue('regionToCountrySubDivision'), + state.getValue("regionToCountrySubDivision") ); await state.setValue( regionToCountrySubDivision, - 'regionToCountrySubDivision', + "regionToCountrySubDivision" ); } diff --git a/src/lib/requests/markSilentPrivacyRequests.ts b/src/lib/requests/markSilentPrivacyRequests.ts index 7c42193c..4a215125 100644 --- a/src/lib/requests/markSilentPrivacyRequests.ts +++ b/src/lib/requests/markSilentPrivacyRequests.ts @@ -1,15 +1,15 @@ -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, fetchAllRequests, makeGraphQLRequest, UPDATE_PRIVACY_REQUEST, -} from '../graphql'; +} from "../graphql"; /** * Mark a set of privacy requests to be in silent mode @@ -57,11 +57,11 @@ export async function markSilentPrivacyRequests({ const client = buildTranscendGraphQLClient(transcendUrl, auth); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Pull in the requests @@ -76,7 +76,7 @@ export async function markSilentPrivacyRequests({ // Notify Transcend logger.info( - colors.magenta(`Marking "${allRequests.length}" as silent mode.`), + colors.magenta(`Marking "${allRequests.length}" as silent mode.`) ); let total = 0; @@ -94,19 +94,19 @@ export async function markSilentPrivacyRequests({ total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully marked ${total} requests as silent mode in "${ totalTime / 1000 - }" seconds!`, - ), + }" seconds!` + ) ); return allRequests.length; } diff --git a/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts b/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts index 2bb4f4db..2fc43ef3 100644 --- a/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts +++ b/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts @@ -1,16 +1,16 @@ -import { RequestAction } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { RequestAction } from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, fetchAllRequests, fetchAllTemplates, makeGraphQLRequest, NOTIFY_ADDITIONAL_TIME, -} from '../graphql'; +} from "../graphql"; /** * Mark a set of privacy requests to be in silent mode. @@ -27,7 +27,7 @@ export async function notifyPrivacyRequestsAdditionalTime({ days = 45, daysLeft = 10, createdAtAfter, - emailTemplate = 'Additional Time Needed', + emailTemplate = "Additional Time Needed", concurrency = 100, transcendUrl = DEFAULT_TRANSCEND_API, }: { @@ -59,17 +59,17 @@ export async function notifyPrivacyRequestsAdditionalTime({ const client = buildTranscendGraphQLClient(transcendUrl, auth); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Grab the template with that title const matchingTemplates = await fetchAllTemplates(client, emailTemplate); const exactTemplateMatch = matchingTemplates.find( - (template) => template.title === emailTemplate, + (template) => template.title === emailTemplate ); if (!exactTemplateMatch) { throw new Error(`Failed to find a template with title: "${emailTemplate}"`); @@ -88,15 +88,15 @@ export async function notifyPrivacyRequestsAdditionalTime({ // Filter requests by daysLeft allRequests = allRequests.filter( (request) => - typeof request.daysRemaining === 'number' && - request.daysRemaining < daysLeft, + typeof request.daysRemaining === "number" && + request.daysRemaining < daysLeft ); // Notify Transcend logger.info( colors.magenta( - `Notifying "${allRequests.length}" that more time is needed.`, - ), + `Notifying "${allRequests.length}" that more time is needed.` + ) ); let total = 0; @@ -116,19 +116,19 @@ export async function notifyPrivacyRequestsAdditionalTime({ total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully marked ${total} requests as silent mode in "${ totalTime / 1000 - }" seconds!`, - ), + }" seconds!` + ) ); return allRequests.length; } diff --git a/src/lib/requests/pullPrivacyRequests.ts b/src/lib/requests/pullPrivacyRequests.ts index 81a9a2b2..9cf43a17 100644 --- a/src/lib/requests/pullPrivacyRequests.ts +++ b/src/lib/requests/pullPrivacyRequests.ts @@ -1,9 +1,9 @@ -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; -import colors from 'colors'; -import { groupBy } from 'lodash-es'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; +import colors from "colors"; +import { groupBy } from "lodash-es"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, createSombraGotInstance, @@ -11,7 +11,7 @@ import { fetchAllRequests, PrivacyRequest, RequestIdentifier, -} from '../graphql'; +} from "../graphql"; export interface ExportedPrivacyRequest extends PrivacyRequest { /** Request identifiers */ @@ -60,22 +60,20 @@ export async function pullPrivacyRequests({ /** All request information with attached identifiers */ requestsWithRequestIdentifiers: ExportedPrivacyRequest[]; /** Requests that are formatted for CSV */ - requestsFormattedForCsv: { - [k in string]: string | null | number | boolean; - }[]; + requestsFormattedForCsv: Record[]; }> { // Find all requests made before createdAt that are in a removing data state const client = buildTranscendGraphQLClient(transcendUrl, auth); const sombra = await createSombraGotInstance(transcendUrl, auth, sombraAuth); // Log date range - let dateRange = ''; + let dateRange = ""; if (createdAtBefore) { dateRange += ` before ${createdAtBefore.toISOString()}`; } if (createdAtAfter) { dateRange += `${ - dateRange ? ', and' : '' + dateRange ? ", and" : "" } after ${createdAtAfter.toISOString()}`; } @@ -85,9 +83,9 @@ export async function pullPrivacyRequests({ `${ actions.length > 0 ? `Pulling requests of type "${actions.join('" , "')}"` - : 'Pulling all requests' - }${dateRange}`, - ), + : "Pulling all requests" + }${dateRange}` + ) ); // fetch the requests @@ -109,7 +107,7 @@ export async function pullPrivacyRequests({ sombra, { requestId: request.id, - }, + } ); return { ...request, @@ -118,11 +116,11 @@ export async function pullPrivacyRequests({ }, { concurrency: pageLimit, - }, + } ); logger.info( - colors.magenta(`Pulled ${requestsWithRequestIdentifiers.length} requests`), + colors.magenta(`Pulled ${requestsWithRequestIdentifiers.length} requests`) ); // Write out to CSV @@ -146,36 +144,32 @@ export async function pullPrivacyRequests({ coreIdentifier, ...request }) => ({ - 'Request ID': id, - 'Created At': createdAt, + "Request ID": id, + "Created At": createdAt, Email: email, - 'Core Identifier': coreIdentifier, - 'Request Type': type, - 'Data Subject Type': subjectType, + "Core Identifier": coreIdentifier, + "Request Type": type, + "Data Subject Type": subjectType, Status: status, Country: country, - 'Country Sub Division': countrySubDivision, + "Country Sub Division": countrySubDivision, Details: details, Origin: origin, - 'Silent Mode': isSilent, - 'Is Test Request': isTest, + "Silent Mode": isSilent, + "Is Test Request": isTest, Language: locale, ...request, - ...Object.entries(groupBy(attributeValues, 'attributeKey.name')).reduce( - (acc, [name, values]) => - Object.assign(acc, { - [name]: values.map(({ name }) => name).join(','), - }), - {}, + ...Object.fromEntries( + Object.entries(groupBy(attributeValues, "attributeKey.name")).map( + ([name, values]) => [name, values.map(({ name }) => name).join(",")] + ) ), - ...Object.entries(groupBy(requestIdentifiers, 'name')).reduce( - (acc, [name, values]) => - Object.assign(acc, { - [name]: values.map(({ value }) => value).join(','), - }), - {}, + ...Object.fromEntries( + Object.entries(groupBy(requestIdentifiers, "name")).map( + ([name, values]) => [name, values.map(({ value }) => value).join(",")] + ) ), - }), + }) ); return { requestsWithRequestIdentifiers, requestsFormattedForCsv: data }; diff --git a/src/lib/requests/readCsv.ts b/src/lib/requests/readCsv.ts index df050570..fed0f1cd 100644 --- a/src/lib/requests/readCsv.ts +++ b/src/lib/requests/readCsv.ts @@ -1,8 +1,8 @@ -import { readFileSync } from 'fs'; -import { decodeCodec } from '@transcend-io/type-utils'; -import type { Options } from 'csv-parse'; -import { parse } from 'csv-parse/sync'; -import * as t from 'io-ts'; +import { readFileSync } from "node:fs"; +import { decodeCodec } from "@transcend-io/type-utils"; +import type { Options } from "csv-parse"; +import { parse } from "csv-parse/sync"; +import * as t from "io-ts"; /** * Read in a CSV and validate its shape @@ -15,10 +15,10 @@ import * as t from 'io-ts'; export function readCsv( pathToFile: string, codec: T, - options: Options = { columns: true }, + options: Options = { columns: true } ): t.TypeOf[] { // read file contents and parse - const fileContent = parse(readFileSync(pathToFile, 'utf-8'), options); + const fileContent = parse(readFileSync(pathToFile, "utf-8"), options); // validate codec const data = decodeCodec(t.array(codec), fileContent); @@ -26,12 +26,12 @@ export function readCsv( // remove any special characters from object keys const parsed = data.map((datum) => Object.entries(datum).reduce( - (acc, [key, value]) => - Object.assign(acc, { - [key.replace(/[^a-z_.+\-A-Z -~]/g, '')]: value, + (accumulator, [key, value]) => + Object.assign(accumulator, { + [key.replaceAll(/[^a-z_.+\-A-Z -~]/g, "")]: value, }), - {} as T, - ), + {} as T + ) ); return parsed; } diff --git a/src/lib/requests/removeUnverifiedRequestIdentifiers.ts b/src/lib/requests/removeUnverifiedRequestIdentifiers.ts index a93bedb8..2b8b02e3 100644 --- a/src/lib/requests/removeUnverifiedRequestIdentifiers.ts +++ b/src/lib/requests/removeUnverifiedRequestIdentifiers.ts @@ -1,16 +1,16 @@ -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, fetchAllRequestIdentifierMetadata, fetchAllRequests, makeGraphQLRequest, REMOVE_REQUEST_IDENTIFIERS, -} from '../graphql'; +} from "../graphql"; /** * Remove a set of unverified request identifier @@ -40,11 +40,11 @@ export async function removeUnverifiedRequestIdentifiers({ const client = buildTranscendGraphQLClient(transcendUrl, auth); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Pull in the requests @@ -54,7 +54,7 @@ export async function removeUnverifiedRequestIdentifiers({ }); // Notify Transcend - logger.info(colors.magenta('Fetched requests in preflight/enriching state.')); + logger.info(colors.magenta("Fetched requests in preflight/enriching state.")); let total = 0; let processed = 0; @@ -64,12 +64,12 @@ export async function removeUnverifiedRequestIdentifiers({ async (requestToRestart) => { const requestIdentifiers = await fetchAllRequestIdentifierMetadata( client, - { requestId: requestToRestart.id }, + { requestId: requestToRestart.id } ); const clearOut = requestIdentifiers .filter( ({ isVerifiedAtLeastOnce, name }) => - isVerifiedAtLeastOnce === false && identifierNames.includes(name), + !isVerifiedAtLeastOnce && identifierNames.includes(name) ) .map(({ id }) => id); @@ -89,19 +89,19 @@ export async function removeUnverifiedRequestIdentifiers({ total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully cleared out unverified identifiers "${ totalTime / 1000 - }" seconds for ${total} requests, ${processed} identifiers were cleared out!`, - ), + }" seconds for ${total} requests, ${processed} identifiers were cleared out!` + ) ); return allRequests.length; } diff --git a/src/lib/requests/retryRequestDataSilos.ts b/src/lib/requests/retryRequestDataSilos.ts index 48a5f2d2..478841b5 100644 --- a/src/lib/requests/retryRequestDataSilos.ts +++ b/src/lib/requests/retryRequestDataSilos.ts @@ -1,16 +1,16 @@ -import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, fetchAllRequests, fetchRequestDataSilo, makeGraphQLRequest, RETRY_REQUEST_DATA_SILO, -} from '../graphql'; +} from "../graphql"; /** * Retry a set of RequestDataSilos @@ -40,11 +40,11 @@ export async function retryRequestDataSilos({ const client = buildTranscendGraphQLClient(transcendUrl, auth); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Pull in the requests @@ -56,8 +56,8 @@ export async function retryRequestDataSilos({ // Notify Transcend logger.info( colors.magenta( - `Retrying requests for Data Silo: "${dataSiloId}", restarting "${allRequests.length}" requests.`, - ), + `Retrying requests for Data Silo: "${dataSiloId}", restarting "${allRequests.length}" requests.` + ) ); let total = 0; @@ -78,10 +78,10 @@ export async function retryRequestDataSilos({ }>(client, RETRY_REQUEST_DATA_SILO, { requestDataSiloId: requestDataSilo.id, }); - } catch (err) { + } catch (error) { // some requests may not have this data silo connected - if (!err.message.includes('Failed to find RequestDataSilo')) { - throw err; + if (!error.message.includes("Failed to find RequestDataSilo")) { + throw error; } skipped += 1; } @@ -89,19 +89,19 @@ export async function retryRequestDataSilos({ total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully notified Transcend in "${ totalTime / 1000 - }" seconds for ${total} requests, ${skipped} requests were skipped because data silo was not attached to the request!`, - ), + }" seconds for ${total} requests, ${skipped} requests were skipped because data silo was not attached to the request!` + ) ); return allRequests.length; } diff --git a/src/lib/requests/skipPreflightJobs.ts b/src/lib/requests/skipPreflightJobs.ts index 9d903d08..48922c98 100644 --- a/src/lib/requests/skipPreflightJobs.ts +++ b/src/lib/requests/skipPreflightJobs.ts @@ -1,19 +1,19 @@ import { RequestEnricherStatus, RequestStatus, -} from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map, mapSeries } from '../bluebird-replace'; +} from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map, mapSeries } from "../bluebird-replace"; import { buildTranscendGraphQLClient, fetchAllRequestEnrichers, fetchAllRequests, makeGraphQLRequest, SKIP_REQUEST_ENRICHER, -} from '../graphql'; +} from "../graphql"; /** * Given an enricher ID, mark all open request enrichers as skipped @@ -42,7 +42,7 @@ export async function skipPreflightJobs({ const client = buildTranscendGraphQLClient(transcendUrl, auth); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // fetch all RequestDataSilos that are open const requests = await fetchAllRequests(client, { @@ -52,16 +52,16 @@ export async function skipPreflightJobs({ // Notify Transcend logger.info( colors.magenta( - `Processing enricher: "${enricherIds.join(',')}" fetched "${ + `Processing enricher: "${enricherIds.join(",")}" fetched "${ requests.length - }" in enriching status.`, - ), + }" in enriching status.` + ) ); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); let total = 0; @@ -81,7 +81,7 @@ export async function skipPreflightJobs({ RequestEnricherStatus.Resolved, RequestEnricherStatus.Skipped, // eslint-disable-next-line @typescript-eslint/no-explicit-any - ].includes(enricher.status as any), + ].includes(enricher.status as any) ); // FIXME @@ -95,13 +95,13 @@ export async function skipPreflightJobs({ requestEnricherId: requestEnricher.id, }); totalSkipped += 1; - } catch (err) { + } catch (error) { if ( - !err.message.includes( - 'Client error: Cannot skip Request enricher because it has already completed', + !error.message.includes( + "Client error: Cannot skip Request enricher because it has already completed" ) ) { - throw err; + throw error; } } }); @@ -109,19 +109,19 @@ export async function skipPreflightJobs({ total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully skipped "${totalSkipped}" for "${ requests.length - }" requests in "${totalTime / 1000}" seconds!`, - ), + }" requests in "${totalTime / 1000}" seconds!` + ) ); return requests.length; } diff --git a/src/lib/requests/skipRequestDataSilos.ts b/src/lib/requests/skipRequestDataSilos.ts index d9c7b21c..89d0154c 100644 --- a/src/lib/requests/skipRequestDataSilos.ts +++ b/src/lib/requests/skipRequestDataSilos.ts @@ -1,15 +1,15 @@ -import { RequestStatus } from '@transcend-io/privacy-types'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { RequestStatus } from "@transcend-io/privacy-types"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, CHANGE_REQUEST_DATA_SILO_STATUS, fetchRequestDataSilos, makeGraphQLRequest, -} from '../graphql'; +} from "../graphql"; /** * Given a data silo ID, mark all open request data silos as skipped @@ -21,7 +21,7 @@ export async function skipRequestDataSilos({ dataSiloId, auth, concurrency = 100, - status = 'SKIPPED', + status = "SKIPPED", transcendUrl = DEFAULT_TRANSCEND_API, requestStatuses = [RequestStatus.Compiling, RequestStatus.Secondary], }: { @@ -30,7 +30,7 @@ export async function skipRequestDataSilos({ /** Data Silo ID to pull down jobs for */ dataSiloId: string; /** Status to set */ - status?: 'SKIPPED' | 'RESOLVED'; + status?: "SKIPPED" | "RESOLVED"; /** Upload concurrency */ concurrency?: number; /** API URL for Transcend backend */ @@ -42,7 +42,7 @@ export async function skipRequestDataSilos({ const client = buildTranscendGraphQLClient(transcendUrl, auth); // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // fetch all RequestDataSilos that are open const requestDataSilos = await fetchRequestDataSilos(client, { @@ -53,14 +53,14 @@ export async function skipRequestDataSilos({ // Notify Transcend logger.info( colors.magenta( - `Processing data silo: "${dataSiloId}" marking "${requestDataSilos.length}" requests as skipped.`, - ), + `Processing data silo: "${dataSiloId}" marking "${requestDataSilos.length}" requests as skipped.` + ) ); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); let total = 0; @@ -76,28 +76,28 @@ export async function skipRequestDataSilos({ requestDataSiloId: requestDataSilo.id, status, }); - } catch (err) { - if (!err.message.includes('Client error: Request must be active:')) { - throw err; + } catch (error) { + if (!error.message.includes("Client error: Request must be active:")) { + throw error; } } total += 1; progressBar.update(total); }, - { concurrency }, + { concurrency } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; logger.info( colors.green( `Successfully skipped "${requestDataSilos.length}" requests in "${ totalTime / 1000 - }" seconds!`, - ), + }" seconds!` + ) ); return requestDataSilos.length; } diff --git a/src/lib/requests/splitCsvToList.ts b/src/lib/requests/splitCsvToList.ts index adefd331..7daa7a9c 100644 --- a/src/lib/requests/splitCsvToList.ts +++ b/src/lib/requests/splitCsvToList.ts @@ -10,7 +10,7 @@ */ export function splitCsvToList(value: string): string[] { return value - .split(',') + .split(",") .map((x) => x.trim()) - .filter((x) => x); + .filter(Boolean); } diff --git a/src/lib/requests/streamPrivacyRequestFiles.ts b/src/lib/requests/streamPrivacyRequestFiles.ts index 98311bdf..76bb964c 100644 --- a/src/lib/requests/streamPrivacyRequestFiles.ts +++ b/src/lib/requests/streamPrivacyRequestFiles.ts @@ -1,8 +1,8 @@ -import colors from 'colors'; -import type { Got } from 'got'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; -import { RequestFileMetadata } from './getFileMetadataForPrivacyRequests'; +import colors from "colors"; +import type { Got } from "got"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; +import { RequestFileMetadata } from "./getFileMetadataForPrivacyRequests"; /** * This function will take in a set of file metadata for privacy requests @@ -28,7 +28,7 @@ export async function streamPrivacyRequestFiles( onFileDownloaded: (metadata: RequestFileMetadata, stream: Buffer) => void; /** Concurrent downloads at once */ concurrency?: number; - }, + } ): Promise { // Loop over each file await map( @@ -37,34 +37,36 @@ export async function streamPrivacyRequestFiles( try { // Construct the stream await sombra - .get('v1/files', { + .get("v1/files", { searchParams: { downloadKey: metadata.downloadKey, }, }) .buffer() - .then((fileResponse) => onFileDownloaded(metadata, fileResponse)); - } catch (err) { - if (err?.response?.body?.includes('fileMetadata#verify')) { + .then((fileResponse) => { + onFileDownloaded(metadata, fileResponse); + }); + } catch (error) { + if (error?.response?.body?.includes("fileMetadata#verify")) { logger.error( colors.red( `Failed to pull file for: ${metadata.fileName} (request:${requestId}) - JWT expired. ` + - 'This likely means that the file is no longer available. ' + - 'Try restarting the request from scratch in Transcend Admin Dashboard. ' + - 'Skipping the download of this file.', - ), + "This likely means that the file is no longer available. " + + "Try restarting the request from scratch in Transcend Admin Dashboard. " + + "Skipping the download of this file." + ) ); return; } throw new Error( `Received an error from server: ${ - err?.response?.body || err?.message - }`, + error?.response?.body || error?.message + }` ); } }, { concurrency, - }, + } ); } diff --git a/src/lib/requests/submitPrivacyRequest.ts b/src/lib/requests/submitPrivacyRequest.ts index 4f887d97..f6d875ae 100644 --- a/src/lib/requests/submitPrivacyRequest.ts +++ b/src/lib/requests/submitPrivacyRequest.ts @@ -3,13 +3,13 @@ import { IsoCountrySubdivisionCode, RequestAction, RequestStatus, -} from '@transcend-io/privacy-types'; -import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; -import type { Got } from 'got'; -import * as t from 'io-ts'; -import { uniq } from 'lodash-es'; -import { PrivacyRequestInput } from './mapCsvRowsToRequestInputs'; -import { ParsedAttributeInput } from './parseAttributesFromString'; +} from "@transcend-io/privacy-types"; +import { decodeCodec, valuesOf } from "@transcend-io/type-utils"; +import type { Got } from "got"; +import * as t from "io-ts"; +import { uniq } from "lodash-es"; +import { PrivacyRequestInput } from "./mapCsvRowsToRequestInputs"; +import { ParsedAttributeInput } from "./parseAttributesFromString"; export const PrivacyRequestResponse = t.type({ id: t.string, @@ -27,7 +27,7 @@ export const PrivacyRequestResponse = t.type({ t.type({ attributeKey: t.type({ name: t.string }), name: t.string, - }), + }) ), }); @@ -46,7 +46,7 @@ export async function submitPrivacyRequest( sombra: Got, input: PrivacyRequestInput, { - details = '', + details = "", isTest = false, emailIsVerified = true, skipSendingReceipt = false, @@ -65,14 +65,14 @@ export async function submitPrivacyRequest( details?: string; /** Additional attributes to tag the requests with */ additionalAttributes?: ParsedAttributeInput[]; - } = {}, + } = {} ): Promise { // Merge the per-request attributes with the // global attributes const mergedAttributes = [...additionalAttributes]; - (input.attributes || []).forEach((attribute) => { + for (const attribute of input.attributes || []) { const existing = mergedAttributes.find( - (attr) => attr.key === attribute.key, + (attribute_) => attribute_.key === attribute.key ); if (existing) { existing.values.push(...attribute.values); @@ -80,13 +80,13 @@ export async function submitPrivacyRequest( } else { mergedAttributes.push(attribute); } - }); + } // Make the GraphQL request let response: unknown; try { response = await sombra - .post('v1/data-subject-request', { + .post("v1/data-subject-request", { json: { type: input.requestType, subject: { @@ -110,8 +110,8 @@ export async function submitPrivacyRequest( country: input.country, } : input.countrySubDivision - ? { country: input.countrySubDivision.split('-')[0] } - : {}), + ? { country: input.countrySubDivision.split("-")[0] } + : {}), ...(input.countrySubDivision ? { countrySubDivision: input.countrySubDivision } : {}), @@ -124,9 +124,11 @@ export async function submitPrivacyRequest( }, }) .json(); - } catch (err) { + } catch (error) { throw new Error( - `Received an error from server: ${err?.response?.body || err?.message}`, + `Received an error from server: ${ + error?.response?.body || error?.message + }` ); } @@ -134,7 +136,7 @@ export async function submitPrivacyRequest( t.type({ request: PrivacyRequestResponse, }), - response, + response ); return requestResponse; } diff --git a/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts b/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts index d0916c3a..4f4f4163 100644 --- a/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts +++ b/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts @@ -1,4 +1,4 @@ -import { describe } from 'vitest'; +import { describe } from "vitest"; // TODO: https://transcend.height.app/T-10772 - mapCsvRowsToRequestInputs test -describe.skip('mapCsvRowsToRequestInputs', () => undefined); +describe.skip("mapCsvRowsToRequestInputs", () => {}); diff --git a/src/lib/requests/tests/readCsv.test.ts b/src/lib/requests/tests/readCsv.test.ts index b8c49127..6a8feef5 100644 --- a/src/lib/requests/tests/readCsv.test.ts +++ b/src/lib/requests/tests/readCsv.test.ts @@ -1,54 +1,54 @@ -import { join } from 'path'; -import * as t from 'io-ts'; -import { describe, expect, it } from 'vitest'; -import { readCsv } from '../index'; +import { join } from "node:path"; +import * as t from "io-ts"; +import { describe, expect, it } from "vitest"; +import { readCsv } from "../index"; -describe('readCsv', () => { - it('should successfully parse a csv', () => { +describe("readCsv", () => { + it("should successfully parse a csv", () => { expect( - readCsv(join(__dirname, 'sample.csv'), t.record(t.string, t.string)), + readCsv(join(__dirname, "sample.csv"), t.record(t.string, t.string)) ).to.deep.equal([ { - CASL_STATUS: 'Undefined', - CONTACT_ID: '949858', - 'Data Subject': 'Customer', - FIRST_NAME: 'Mike', - GDPR_STATUS: 'LBI Notice Not Sent', - LAST_NAME: 'Farrell', - LINKEDIN_HANDLE: 'michaelfarrell', - MOBILE_PHONE: '(860) 906-6012', - OTHER_URL: '', - PERSONAL_EMAIL: 'mike@transcend.io', - PHONE: '', - PREFERRED_EMAIL: 'mike@transcend.io', - 'Primary Company': 'Transcend', - 'Request Type': 'Opt In', - SEARCH_ID: '10749', - SEARCH_NAME: 'Transcend - Chief Technology Officer', - SKYPE_HANDLE: '', - STAGE_START_DATE: '2022-11-22', - TITLE: 'Chief Technology Officer', - WORK_EMAIL: '', - WORK_PHONE: '', + CASL_STATUS: "Undefined", + CONTACT_ID: "949858", + "Data Subject": "Customer", + FIRST_NAME: "Mike", + GDPR_STATUS: "LBI Notice Not Sent", + LAST_NAME: "Farrell", + LINKEDIN_HANDLE: "michaelfarrell", + MOBILE_PHONE: "(860) 906-6012", + OTHER_URL: "", + PERSONAL_EMAIL: "mike@transcend.io", + PHONE: "", + PREFERRED_EMAIL: "mike@transcend.io", + "Primary Company": "Transcend", + "Request Type": "Opt In", + SEARCH_ID: "10749", + SEARCH_NAME: "Transcend - Chief Technology Officer", + SKYPE_HANDLE: "", + STAGE_START_DATE: "2022-11-22", + TITLE: "Chief Technology Officer", + WORK_EMAIL: "", + WORK_PHONE: "", }, ]); }); - it('throw an error for invalid file', () => { + it("throw an error for invalid file", () => { expect(() => - readCsv(join(__dirname, 'sample.csv'), t.type({ notValid: t.string })), - ).to.throw('Failed to decode codec'); + readCsv(join(__dirname, "sample.csv"), t.type({ notValid: t.string })) + ).to.throw("Failed to decode codec"); }); - it('throw an error for invalid codec', () => { + it("throw an error for invalid codec", () => { expect(() => - readCsv(join(__dirname, 'sample.csvs'), t.record(t.string, t.string)), - ).to.throw('ENOENT: no such file or directory, open'); + readCsv(join(__dirname, "sample.csvs"), t.record(t.string, t.string)) + ).to.throw("ENOENT: no such file or directory, open"); }); - it('throw an error for invalid format', () => { + it("throw an error for invalid format", () => { expect(() => - readCsv(join(__dirname, 'readCsv.test.ts'), t.record(t.string, t.string)), + readCsv(join(__dirname, "readCsv.test.ts"), t.record(t.string, t.string)) ).to.throw(); }); }); diff --git a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts index e5b7216e..b897697d 100644 --- a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts +++ b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts @@ -1,29 +1,28 @@ -/* eslint-disable max-lines */ -import { join } from 'path'; -import { PersistedState } from '@transcend-io/persisted-state'; -import cliProgress from 'cli-progress'; -import colors from 'colors'; -import * as t from 'io-ts'; -import { uniq } from 'lodash-es'; -import { DEFAULT_TRANSCEND_API } from '../../constants'; -import { logger } from '../../logger'; -import { map } from '../bluebird-replace'; +import { join } from "node:path"; +import { PersistedState } from "@transcend-io/persisted-state"; +import cliProgress from "cli-progress"; +import colors from "colors"; +import * as t from "io-ts"; +import { uniq } from "lodash-es"; +import { DEFAULT_TRANSCEND_API } from "../../constants"; +import { logger } from "../../logger"; +import { map } from "../bluebird-replace"; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequestAttributeKeys, -} from '../graphql'; -import { CachedFileState, CachedRequestState } from './constants'; -import { extractClientError } from './extractClientError'; -import { filterRows } from './filterRows'; -import { mapColumnsToAttributes } from './mapColumnsToAttributes'; -import { mapColumnsToIdentifiers } from './mapColumnsToIdentifiers'; -import { mapCsvColumnsToApi } from './mapCsvColumnsToApi'; -import { mapCsvRowsToRequestInputs } from './mapCsvRowsToRequestInputs'; -import { mapRequestEnumValues } from './mapRequestEnumValues'; -import { parseAttributesFromString } from './parseAttributesFromString'; -import { readCsv } from './readCsv'; -import { submitPrivacyRequest } from './submitPrivacyRequest'; +} from "../graphql"; +import { CachedFileState, CachedRequestState } from "./constants"; +import { extractClientError } from "./extractClientError"; +import { filterRows } from "./filterRows"; +import { mapColumnsToAttributes } from "./mapColumnsToAttributes"; +import { mapColumnsToIdentifiers } from "./mapColumnsToIdentifiers"; +import { mapCsvColumnsToApi } from "./mapCsvColumnsToApi"; +import { mapCsvRowsToRequestInputs } from "./mapCsvRowsToRequestInputs"; +import { mapRequestEnumValues } from "./mapRequestEnumValues"; +import { parseAttributesFromString } from "./parseAttributesFromString"; +import { readCsv } from "./readCsv"; +import { submitPrivacyRequest } from "./submitPrivacyRequest"; /** * Upload a set of privacy requests from CSV @@ -37,7 +36,7 @@ export async function uploadPrivacyRequestsFromCsv({ auth, sombraAuth, concurrency = 100, - defaultPhoneCountryCode = '1', // USA + defaultPhoneCountryCode = "1", // USA transcendUrl = DEFAULT_TRANSCEND_API, attributes = [], emailIsVerified = true, @@ -82,11 +81,11 @@ export async function uploadPrivacyRequestsFromCsv({ dryRun?: boolean; }): Promise { // Time duration - const t0 = new Date().getTime(); + const t0 = Date.now(); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic, + cliProgress.Presets.shades_classic ); // Parse out the extra attributes to apply to all requests uploaded @@ -110,8 +109,8 @@ export async function uploadPrivacyRequestsFromCsv({ const requestCacheFile = join( requestReceiptFolder, `tr-request-upload-${new Date().toISOString()}-${file - .split('/') - .pop()}`.replace('.csv', '.json'), + .split("/") + .pop()}`.replace(".csv", ".json") ); const requestState = new PersistedState( requestCacheFile, @@ -120,7 +119,7 @@ export async function uploadPrivacyRequestsFromCsv({ successfulRequests: [], duplicateRequests: [], failingRequests: [], - }, + } ); // Create sombra instance to communicate with @@ -128,18 +127,18 @@ export async function uploadPrivacyRequestsFromCsv({ // Read in the list of integration requests const requestsList = readCsv(file, t.record(t.string, t.string)); - const columnNames = uniq(requestsList.map((x) => Object.keys(x)).flat()); + const columnNames = uniq(requestsList.flatMap((x) => Object.keys(x))); // Log out an example request if (requestsList.length === 0) { throw new Error( - 'No Requests found in list! Ensure the first row of the CSV is a header and the rest are requests.', + "No Requests found in list! Ensure the first row of the CSV is a header and the rest are requests." ); } if (debug) { const firstRequest = requestsList[0]; logger.info( - colors.magenta(`First request: ${JSON.stringify(firstRequest, null, 2)}`), + colors.magenta(`First request: ${JSON.stringify(firstRequest, null, 2)}`) ); } // Determine what rows in the CSV should be imported @@ -157,13 +156,13 @@ export async function uploadPrivacyRequestsFromCsv({ const identifierNameMap = await mapColumnsToIdentifiers( client, columnNames, - state, + state ); const attributeNameMap = await mapColumnsToAttributes( client, columnNames, state, - requestAttributeKeys, + requestAttributeKeys ); await mapRequestEnumValues(client, filteredRequestList, { state, @@ -199,16 +198,16 @@ export async function uploadPrivacyRequestsFromCsv({ `[${ind + 1}/${requestInputs.length}] Importing: ${JSON.stringify( requestInput, null, - 2, - )}`, - ), + 2 + )}` + ) ); } // Skip on dry run if (dryRun) { logger.info( - colors.magenta('Bailing out on dry run because dryRun is set'), + colors.magenta("Bailing out on dry run because dryRun is set") ); return; } @@ -222,14 +221,14 @@ export async function uploadPrivacyRequestsFromCsv({ details: `Uploaded by Transcend Cli: "tr-request-upload" : ${JSON.stringify( rawRow, null, - 2, + 2 )}`, isTest, emailIsVerified, skipSendingReceipt, isSilent, additionalAttributes: parsedAttributes, - }, + } ); // Log success @@ -238,20 +237,20 @@ export async function uploadPrivacyRequestsFromCsv({ colors.green( `[${ind + 1}/${ requestInputs.length - }] Successfully submitted the test data subject request: "${requestLogId}"`, - ), + }] Successfully submitted the test data subject request: "${requestLogId}"` + ) ); logger.info( colors.green( `[${ind + 1}/${requestInputs.length}] View it at: "${ requestResponse.link - }"`, - ), + }"` + ) ); } // Cache successful upload - const successfulRequests = requestState.getValue('successfulRequests'); + const successfulRequests = requestState.getValue("successfulRequests"); successfulRequests.push({ id: requestResponse.id, link: requestResponse.link, @@ -259,51 +258,51 @@ export async function uploadPrivacyRequestsFromCsv({ coreIdentifier: requestResponse.coreIdentifier, attemptedAt: new Date().toISOString(), }); - await requestState.setValue(successfulRequests, 'successfulRequests'); - } catch (err) { - const msg = `${err.message} - ${JSON.stringify( - err.response?.body, + await requestState.setValue(successfulRequests, "successfulRequests"); + } catch (error) { + const message = `${error.message} - ${JSON.stringify( + error.response?.body, null, - 2, + 2 )}`; - const clientError = extractClientError(msg); + const clientError = extractClientError(message); if ( - clientError === 'Client error: You have already made this request.' + clientError === "Client error: You have already made this request." ) { if (debug) { logger.info( colors.yellow( `[${ind + 1}/${ requestInputs.length - }] Skipping request as it is a duplicate`, - ), + }] Skipping request as it is a duplicate` + ) ); } - const duplicateRequests = requestState.getValue('duplicateRequests'); + const duplicateRequests = requestState.getValue("duplicateRequests"); duplicateRequests.push({ coreIdentifier: requestInput.coreIdentifier, rowIndex: ind, attemptedAt: new Date().toISOString(), }); - await requestState.setValue(duplicateRequests, 'duplicateRequests'); + await requestState.setValue(duplicateRequests, "duplicateRequests"); } else { - const failingRequests = requestState.getValue('failingRequests'); + const failingRequests = requestState.getValue("failingRequests"); failingRequests.push({ ...requestInput, rowIndex: ind, - error: clientError || msg, + error: clientError || message, attemptedAt: new Date().toISOString(), }); - await requestState.setValue(failingRequests, 'failingRequests'); + await requestState.setValue(failingRequests, "failingRequests"); if (debug) { - logger.error(colors.red(clientError || msg)); + logger.error(colors.red(clientError || message)); logger.error( colors.red( `[${ind + 1}/${ requestInputs.length - }] Failed to submit request for: "${requestLogId}"`, - ), + }] Failed to submit request for: "${requestLogId}"` + ) ); } } @@ -316,41 +315,40 @@ export async function uploadPrivacyRequestsFromCsv({ }, { concurrency, - }, + } ); progressBar.stop(); - const t1 = new Date().getTime(); + const t1 = Date.now(); const totalTime = t1 - t0; // Log completion time logger.info( - colors.green(`Completed upload in "${totalTime / 1000}" seconds.`), + colors.green(`Completed upload in "${totalTime / 1000}" seconds.`) ); // Log duplicates - if (requestState.getValue('duplicateRequests').length > 0) { + if (requestState.getValue("duplicateRequests").length > 0) { logger.info( colors.yellow( `Encountered "${ - requestState.getValue('duplicateRequests').length + requestState.getValue("duplicateRequests").length }" duplicate requests. ` + - `See "${requestCacheFile}" to review the core identifiers for these requests.`, - ), + `See "${requestCacheFile}" to review the core identifiers for these requests.` + ) ); } // Log errors - if (requestState.getValue('failingRequests').length > 0) { + if (requestState.getValue("failingRequests").length > 0) { logger.error( colors.red( `Encountered "${ - requestState.getValue('failingRequests').length + requestState.getValue("failingRequests").length }" errors. ` + - `See "${requestCacheFile}" to review the error messages and inputs.`, - ), + `See "${requestCacheFile}" to review the error messages and inputs.` + ) ); process.exit(1); } } -/* eslint-enable max-lines */ diff --git a/src/lib/tests/findCodePackagesInFolder.test.ts b/src/lib/tests/findCodePackagesInFolder.test.ts index 59b05dcc..0cb4d54f 100644 --- a/src/lib/tests/findCodePackagesInFolder.test.ts +++ b/src/lib/tests/findCodePackagesInFolder.test.ts @@ -1,700 +1,699 @@ -/* eslint-disable max-lines */ -import { join } from 'path'; -import { describe, expect, it } from 'vitest'; -import type { CodePackageInput } from '../../codecs'; -import { findCodePackagesInFolder } from '../code-scanning/findCodePackagesInFolder'; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import type { CodePackageInput } from "../../codecs"; +import { findCodePackagesInFolder } from "../code-scanning/findCodePackagesInFolder"; const expected: CodePackageInput[] = [ { - name: 'YourAppTargetName', - type: 'COCOA_PODS', + name: "YourAppTargetName", + type: "COCOA_PODS", softwareDevelopmentKits: [ { - name: 'Braze-iOS-SDK', + name: "Braze-iOS-SDK", version: undefined, }, { - name: 'Branch', + name: "Branch", version: undefined, }, { - name: 'Firebase/Analytics', + name: "Firebase/Analytics", version: undefined, }, { - name: 'Mixpanel', + name: "Mixpanel", version: undefined, }, { - name: 'Amplitude-iOS', - version: '8.0', + name: "Amplitude-iOS", + version: "8.0", }, { - name: 'Google-Mobile-Ads-SDK', + name: "Google-Mobile-Ads-SDK", version: undefined, }, { - name: 'FacebookAdsSDK', + name: "FacebookAdsSDK", version: undefined, }, { - name: 'MoPub-SDK', + name: "MoPub-SDK", version: undefined, }, { - name: 'Alamofire', - version: '5.2', + name: "Alamofire", + version: "5.2", }, { - name: 'SDWebImage', + name: "SDWebImage", version: undefined, }, { - name: 'AppsFlyerFramework', + name: "AppsFlyerFramework", version: undefined, }, { - name: 'Adjust', + name: "Adjust", version: undefined, }, { - name: 'Flurry-iOS-SDK/FlurrySDK', + name: "Flurry-iOS-SDK/FlurrySDK", version: undefined, }, ], - relativePath: 'test-cocoa-pods/Podfile', - repositoryName: 'transcend-io/cli', + relativePath: "test-cocoa-pods/Podfile", + repositoryName: "transcend-io/cli", }, { - name: 'ExampleBootstrap', - type: 'COCOA_PODS', + name: "ExampleBootstrap", + type: "COCOA_PODS", softwareDevelopmentKits: [ { - name: 'ExampleLib', + name: "ExampleLib", version: undefined, }, { - name: 'AppsFlyerFramework', + name: "AppsFlyerFramework", version: undefined, }, { - name: 'Adjust', + name: "Adjust", version: undefined, }, { - name: 'Flurry-iOS-SDK/FlurrySDK', + name: "Flurry-iOS-SDK/FlurrySDK", version: undefined, }, ], - relativePath: 'test-requirements-txt/nested-cocoapods/Podfile', - repositoryName: 'transcend-io/cli', + relativePath: "test-requirements-txt/nested-cocoapods/Podfile", + repositoryName: "transcend-io/cli", }, { - name: 'ExampleBootstrapTests', - type: 'COCOA_PODS', + name: "ExampleBootstrapTests", + type: "COCOA_PODS", softwareDevelopmentKits: [ { - name: 'ExampleLib', + name: "ExampleLib", version: undefined, }, { - name: 'Braze-iOS-SDK', + name: "Braze-iOS-SDK", version: undefined, }, { - name: 'Branch', + name: "Branch", version: undefined, }, { - name: 'Firebase/Analytics', + name: "Firebase/Analytics", version: undefined, }, { - name: 'Mixpanel', + name: "Mixpanel", version: undefined, }, { - name: 'Amplitude-iOS', - version: '8.0', + name: "Amplitude-iOS", + version: "8.0", }, ], - relativePath: 'test-requirements-txt/nested-cocoapods/Podfile', - repositoryName: 'transcend-io/cli', + relativePath: "test-requirements-txt/nested-cocoapods/Podfile", + repositoryName: "transcend-io/cli", }, { - name: 'Acme', - relativePath: 'test-requirements-txt/nested-cocoapods-2/Podfile', - repositoryName: 'transcend-io/cli', + name: "Acme", + relativePath: "test-requirements-txt/nested-cocoapods-2/Podfile", + repositoryName: "transcend-io/cli", softwareDevelopmentKits: [ { - name: 'RZVinyl', + name: "RZVinyl", version: undefined, }, { - name: 'RZTransitions', + name: "RZTransitions", version: undefined, }, { - name: 'SDWebImage', + name: "SDWebImage", version: undefined, }, { - name: 'SwiftLint', + name: "SwiftLint", version: undefined, }, ], - type: 'COCOA_PODS', + type: "COCOA_PODS", }, { - name: 'AcmeTests', - type: 'COCOA_PODS', + name: "AcmeTests", + type: "COCOA_PODS", softwareDevelopmentKits: [ { - name: 'RZVinyl', + name: "RZVinyl", version: undefined, }, { - name: 'iOSSnapshotTestCase', + name: "iOSSnapshotTestCase", version: undefined, }, { - name: 'SnapshotTesting', - version: '1.8.1', + name: "SnapshotTesting", + version: "1.8.1", }, ], - relativePath: 'test-requirements-txt/nested-cocoapods-2/Podfile', - repositoryName: 'transcend-io/cli', + relativePath: "test-requirements-txt/nested-cocoapods-2/Podfile", + repositoryName: "transcend-io/cli", }, { - name: 'NotificationServiceExtension', - relativePath: 'test-requirements-txt/nested-cocoapods-2/Podfile', - repositoryName: 'transcend-io/cli', + name: "NotificationServiceExtension", + relativePath: "test-requirements-txt/nested-cocoapods-2/Podfile", + repositoryName: "transcend-io/cli", softwareDevelopmentKits: [], - type: 'COCOA_PODS', + type: "COCOA_PODS", }, { - name: 'com.yourcompany.yourapp', + name: "com.yourcompany.yourapp", softwareDevelopmentKits: [ { - name: 'androidx.appcompat', - version: '1.2.0', + name: "androidx.appcompat", + version: "1.2.0", }, { - name: 'androidx.constraintlayout', - version: '2.0.4', + name: "androidx.constraintlayout", + version: "2.0.4", }, { - name: 'com.appboy', - version: '14.0.0', + name: "com.appboy", + version: "14.0.0", }, { - name: 'io.branch.sdk.android', - version: '5.0.1', + name: "io.branch.sdk.android", + version: "5.0.1", }, { - name: 'com.google.firebase', - version: '18.0.0', + name: "com.google.firebase", + version: "18.0.0", }, { - name: 'com.google.android.gms', - version: '19.7.0', + name: "com.google.android.gms", + version: "19.7.0", }, { - name: 'com.facebook.android', - version: '6.2.0', + name: "com.facebook.android", + version: "6.2.0", }, { - name: 'com.mixpanel.android', - version: '5.8.7', + name: "com.mixpanel.android", + version: "5.8.7", }, { - name: 'com.amplitude', - version: '2.30.0', + name: "com.amplitude", + version: "2.30.0", }, { - name: 'com.squareup.retrofit2', - version: '2.9.0', + name: "com.squareup.retrofit2", + version: "2.9.0", }, { - name: 'com.squareup.okhttp3', - version: '4.9.0', + name: "com.squareup.okhttp3", + version: "4.9.0", }, { - name: 'com.squareup.picasso', - version: '2.71828', + name: "com.squareup.picasso", + version: "2.71828", }, { - name: 'org.eclipse.jdt.core', - version: '3.28.0', + name: "org.eclipse.jdt.core", + version: "3.28.0", }, { - name: 'com.android.application', + name: "com.android.application", version: undefined, }, { - name: 'com.google.gms.google-services', + name: "com.google.gms.google-services", version: undefined, }, ], - relativePath: 'test-gradle/build.gradle', - type: 'GRADLE', - repositoryName: 'transcend-io/cli', + relativePath: "test-gradle/build.gradle", + type: "GRADLE", + repositoryName: "transcend-io/cli", }, { - name: '@test-example/test', - description: 'Example npm package.', + name: "@test-example/test", + description: "Example npm package.", softwareDevelopmentKits: [ { - name: 'dd-trace', - version: '2.45.1', + name: "dd-trace", + version: "2.45.1", }, { - name: 'fast-csv', - version: '^4.3.6', + name: "fast-csv", + version: "^4.3.6", }, { - name: 'sequelize', - version: '^6.37.3', + name: "sequelize", + version: "^6.37.3", }, { - name: 'sequelize-mock', - version: '^0.10.2', + name: "sequelize-mock", + version: "^0.10.2", }, { isDevDependency: true, - name: '@types/sequelize', - version: '^4.28.20', + name: "@types/sequelize", + version: "^4.28.20", }, { - name: 'typescript', - version: '^5.0.4', + name: "typescript", + version: "^5.0.4", isDevDependency: true, }, ], - relativePath: 'test-package-json/package.json', - type: 'PACKAGE_JSON', - repositoryName: 'transcend-io/cli', + relativePath: "test-package-json/package.json", + type: "PACKAGE_JSON", + repositoryName: "transcend-io/cli", }, { - name: '@test-example/nested-test', - description: 'Example npm nested package.', + name: "@test-example/nested-test", + description: "Example npm nested package.", softwareDevelopmentKits: [ { - name: 'dd-trace', - version: '2.45.1', + name: "dd-trace", + version: "2.45.1", }, { - name: 'fast-csv', - version: '^4.3.6', + name: "fast-csv", + version: "^4.3.6", }, { - name: 'typescript', - version: '^5.0.4', + name: "typescript", + version: "^5.0.4", isDevDependency: true, }, ], - relativePath: 'test-gradle/test-nested-package-json/package.json', - type: 'PACKAGE_JSON', - repositoryName: 'transcend-io/cli', + relativePath: "test-gradle/test-nested-package-json/package.json", + type: "PACKAGE_JSON", + repositoryName: "transcend-io/cli", }, { - name: 'test_requirements_txt', - type: 'REQUIREMENTS_TXT', - description: 'A sample Python package', + name: "test_requirements_txt", + type: "REQUIREMENTS_TXT", + description: "A sample Python package", softwareDevelopmentKits: [ { - name: 'pyarrow', - version: '14.0.1', + name: "pyarrow", + version: "14.0.1", }, { - name: 'cryptography', - version: '41.0.6', + name: "cryptography", + version: "41.0.6", }, { - name: 'Flask', - version: '2.2.5', + name: "Flask", + version: "2.2.5", }, { - name: 'cachetools', - version: '5.3.0', + name: "cachetools", + version: "5.3.0", }, ], - relativePath: 'test-requirements-txt/requirements.txt', - repositoryName: 'transcend-io/cli', + relativePath: "test-requirements-txt/requirements.txt", + repositoryName: "transcend-io/cli", }, { - name: 'test-nested-requirements-txt', - type: 'REQUIREMENTS_TXT', + name: "test-nested-requirements-txt", + type: "REQUIREMENTS_TXT", description: undefined, softwareDevelopmentKits: [ { - name: 'pyarrow', - version: '14.0.1', + name: "pyarrow", + version: "14.0.1", }, { - name: 'pandas', - version: '2.0.3', + name: "pandas", + version: "2.0.3", }, ], relativePath: - 'test-package-json/test-nested-requirements-txt/requirements.txt', - repositoryName: 'transcend-io/cli', + "test-package-json/test-nested-requirements-txt/requirements.txt", + repositoryName: "transcend-io/cli", }, { - name: 'test-gemfile', - type: 'GEMFILE', + name: "test-gemfile", + type: "GEMFILE", description: undefined, softwareDevelopmentKits: [ { - name: 'rails', - version: '~> 6.1.4', + name: "rails", + version: "~> 6.1.4", }, { - name: 'ahoy_matey', + name: "ahoy_matey", version: undefined, }, { - name: 'rack-tracker', + name: "rack-tracker", version: undefined, }, { - name: 'adroll', + name: "adroll", version: undefined, }, { - name: 'google-ads-googleads', + name: "google-ads-googleads", version: undefined, }, { - name: 'facebookads', + name: "facebookads", version: undefined, }, { - name: 'devise', + name: "devise", version: undefined, }, { - name: 'impressionist', + name: "impressionist", version: undefined, }, { - name: 'sidekiq', + name: "sidekiq", version: undefined, }, { - name: 'sidekiq-cron', - version: '~> 1.2', + name: "sidekiq-cron", + version: "~> 1.2", }, { - name: 'byebug', + name: "byebug", version: undefined, }, { - name: 'listen', - version: '~> 3.3', + name: "listen", + version: "~> 3.3", }, { - name: 'capybara', - version: '>= 2.15', + name: "capybara", + version: ">= 2.15", }, { - name: 'selenium-webdriver', + name: "selenium-webdriver", version: undefined, }, { - name: 'webdrivers', + name: "webdrivers", version: undefined, }, { - name: 'bundler-audit', + name: "bundler-audit", version: undefined, }, ], - relativePath: 'test-gemfile/Gemfile', - repositoryName: 'transcend-io/cli', + relativePath: "test-gemfile/Gemfile", + repositoryName: "transcend-io/cli", }, { - name: 'test-nested-gemfile', - type: 'GEMFILE', + name: "test-nested-gemfile", + type: "GEMFILE", softwareDevelopmentKits: [ { - name: 'rails', - version: '~> 6.1.4', + name: "rails", + version: "~> 6.1.4", }, { - name: 'ahoy_matey', + name: "ahoy_matey", version: undefined, }, { - name: 'rack-tracker', + name: "rack-tracker", version: undefined, }, { - name: 'adroll', + name: "adroll", version: undefined, }, { - name: 'google-ads-googleads', + name: "google-ads-googleads", version: undefined, }, { - name: 'facebookads', + name: "facebookads", version: undefined, }, { - name: 'byebug', + name: "byebug", version: undefined, }, { - name: 'listen', - version: '~> 3.3', + name: "listen", + version: "~> 3.3", }, { - name: 'capybara', - version: '>= 2.15', + name: "capybara", + version: ">= 2.15", }, { - name: 'selenium-webdriver', + name: "selenium-webdriver", version: undefined, }, { - name: 'webdrivers', + name: "webdrivers", version: undefined, }, { - name: 'bundler-audit', + name: "bundler-audit", version: undefined, }, ], description: undefined, - relativePath: 'test-gradle/test-nested-gemfile/Gemfile', - repositoryName: 'transcend-io/cli', + relativePath: "test-gradle/test-nested-gemfile/Gemfile", + repositoryName: "transcend-io/cli", }, { - name: 'example', - description: 'test example app', - type: 'PUBSPEC', + name: "example", + description: "test example app", + type: "PUBSPEC", softwareDevelopmentKits: [ { - name: 'flutter', - version: 'flutter', + name: "flutter", + version: "flutter", }, { - name: 'flutter_localizations', - version: 'flutter', + name: "flutter_localizations", + version: "flutter", }, { - name: 'firebase_core', - version: '2.16.0', + name: "firebase_core", + version: "2.16.0", }, { - name: 'firebase_analytics', - version: '10.5.0', + name: "firebase_analytics", + version: "10.5.0", }, { - name: 'firebase_crashlytics', - version: '3.3.6', + name: "firebase_crashlytics", + version: "3.3.6", }, { - name: 'video_player', - version: '2.6.1', + name: "video_player", + version: "2.6.1", }, { - name: 'appsflyer_sdk', - version: '6.12.2', + name: "appsflyer_sdk", + version: "6.12.2", }, { - name: 'isolate', - version: '2.1.1', + name: "isolate", + version: "2.1.1", }, { - name: 'custom_platform_device_id', - version: '1.0.8', + name: "custom_platform_device_id", + version: "1.0.8", }, { - name: 'image_editor', - version: '1.3.0', + name: "image_editor", + version: "1.3.0", }, { - name: 'firebase_remote_config', - version: '4.2.6', + name: "firebase_remote_config", + version: "4.2.6", }, { - name: 'intercom_flutter', - version: '7.8.4', + name: "intercom_flutter", + version: "7.8.4", }, { - name: 'dismissible_page', - version: '1.0.2', + name: "dismissible_page", + version: "1.0.2", }, { - name: 'extended_text', - version: '11.1.0', + name: "extended_text", + version: "11.1.0", }, { - name: 'recaptcha_enterprise_flutter', - version: '18.3.0', + name: "recaptcha_enterprise_flutter", + version: "18.3.0", }, { - name: 'flutter_test', - version: 'flutter', + name: "flutter_test", + version: "flutter", isDevDependency: true, }, { - name: 'test', - version: '1.24.3', + name: "test", + version: "1.24.3", isDevDependency: true, }, { - name: 'lints', - version: '3.0.0', + name: "lints", + version: "3.0.0", isDevDependency: true, }, { - name: 'mocktail', - version: '1.0.1', + name: "mocktail", + version: "1.0.1", isDevDependency: true, }, ], - relativePath: 'test-pubspec/pubspec.yml', - repositoryName: 'transcend-io/cli', + relativePath: "test-pubspec/pubspec.yml", + repositoryName: "transcend-io/cli", }, { - name: 'composer/example', - description: 'Example app', + name: "composer/example", + description: "Example app", softwareDevelopmentKits: [ { - name: 'php', - version: '^7.2.5 || ^8.0', + name: "php", + version: "^7.2.5 || ^8.0", }, { - name: 'composer/ca-bundle', - version: '^1.5', + name: "composer/ca-bundle", + version: "^1.5", }, { - name: 'composer/class-map-generator', - version: '^1.3.3', + name: "composer/class-map-generator", + version: "^1.3.3", }, { - name: 'composer/metadata-minifier', - version: '^1.0', + name: "composer/metadata-minifier", + version: "^1.0", }, { - name: 'composer/semver', - version: '^3.3', + name: "composer/semver", + version: "^3.3", }, { - name: 'composer/spdx-licenses', - version: '^1.5.7', + name: "composer/spdx-licenses", + version: "^1.5.7", }, { - name: 'composer/xdebug-handler', - version: '^2.0.2 || ^3.0.3', + name: "composer/xdebug-handler", + version: "^2.0.2 || ^3.0.3", }, { - name: 'justinrainbow/json-schema', - version: '^5.3', + name: "justinrainbow/json-schema", + version: "^5.3", }, { - name: 'psr/log', - version: '^1.0 || ^2.0 || ^3.0', + name: "psr/log", + version: "^1.0 || ^2.0 || ^3.0", }, { - name: 'seld/jsonlint', - version: '^1.4', + name: "seld/jsonlint", + version: "^1.4", }, { - name: 'seld/phar-utils', - version: '^1.2', + name: "seld/phar-utils", + version: "^1.2", }, { - name: 'symfony/console', - version: '^5.4.35 || ^6.3.12 || ^7.0.3', + name: "symfony/console", + version: "^5.4.35 || ^6.3.12 || ^7.0.3", }, { - name: 'symfony/filesystem', - version: '^5.4.35 || ^6.3.12 || ^7.0.3', + name: "symfony/filesystem", + version: "^5.4.35 || ^6.3.12 || ^7.0.3", }, { - name: 'symfony/finder', - version: '^5.4.35 || ^6.3.12 || ^7.0.3', + name: "symfony/finder", + version: "^5.4.35 || ^6.3.12 || ^7.0.3", }, { - name: 'symfony/process', - version: '^5.4.35 || ^6.3.12 || ^7.0.3', + name: "symfony/process", + version: "^5.4.35 || ^6.3.12 || ^7.0.3", }, { - name: 'react/promise', - version: '^3.2', + name: "react/promise", + version: "^3.2", }, { - name: 'composer/pcre', - version: '^2.2 || ^3.2', + name: "composer/pcre", + version: "^2.2 || ^3.2", }, { - name: 'symfony/polyfill-php73', - version: '^1.24', + name: "symfony/polyfill-php73", + version: "^1.24", }, { - name: 'symfony/polyfill-php80', - version: '^1.24', + name: "symfony/polyfill-php80", + version: "^1.24", }, { - name: 'symfony/polyfill-php81', - version: '^1.24', + name: "symfony/polyfill-php81", + version: "^1.24", }, { - name: 'seld/signal-handler', - version: '^2.0', + name: "seld/signal-handler", + version: "^2.0", }, { - name: 'symfony/phpunit-bridge', - version: '^6.4.3 || ^7.0.1', + name: "symfony/phpunit-bridge", + version: "^6.4.3 || ^7.0.1", isDevDependency: true, }, { - name: 'phpstan/phpstan', - version: '^1.11.8', + name: "phpstan/phpstan", + version: "^1.11.8", isDevDependency: true, }, { - name: 'phpstan/phpstan-phpunit', - version: '^1.4.0', + name: "phpstan/phpstan-phpunit", + version: "^1.4.0", isDevDependency: true, }, { - name: 'phpstan/phpstan-deprecation-rules', - version: '^1.2.0', + name: "phpstan/phpstan-deprecation-rules", + version: "^1.2.0", isDevDependency: true, }, { - name: 'phpstan/phpstan-strict-rules', - version: '^1.6.0', + name: "phpstan/phpstan-strict-rules", + version: "^1.6.0", isDevDependency: true, }, { - name: 'phpstan/phpstan-symfony', - version: '^1.4.0', + name: "phpstan/phpstan-symfony", + version: "^1.4.0", isDevDependency: true, }, ], - relativePath: 'test-php/composer.json', - type: 'COMPOSER_JSON', - repositoryName: 'transcend-io/cli', + relativePath: "test-php/composer.json", + type: "COMPOSER_JSON", + repositoryName: "transcend-io/cli", }, { - name: 'test-swift', - type: 'SWIFT', - relativePath: 'test-swift/Package.resolved', - repositoryName: 'transcend-io/cli', + name: "test-swift", + type: "SWIFT", + relativePath: "test-swift/Package.resolved", + repositoryName: "transcend-io/cli", softwareDevelopmentKits: [ { - name: 'alamofire', - version: '5.8.1', + name: "alamofire", + version: "5.8.1", }, { - name: 'swift-numerics', - version: '1.0.2', + name: "swift-numerics", + version: "1.0.2", }, ], }, @@ -707,26 +706,25 @@ const expected: CodePackageInput[] = [ * @returns Sorted code packages */ function sortCodePackages( - codePackages: CodePackageInput[], + codePackages: CodePackageInput[] ): CodePackageInput[] { return codePackages .sort((a, b) => a.name.localeCompare(b.name)) .map((c) => ({ ...c, softwareDevelopmentKits: c.softwareDevelopmentKits?.sort((a, b) => - a.name.localeCompare(b.name), + a.name.localeCompare(b.name) ), })); } // not easy to test this but can uncomment to run against current commit -describe('findCodePackagesInFolder', () => { - it('should remove links', async () => { +describe("findCodePackagesInFolder", () => { + it("should remove links", async () => { const result = await findCodePackagesInFolder({ - repositoryName: 'transcend-io/cli', - scanPath: join(__dirname, '../../../examples/code-scanning'), + repositoryName: "transcend-io/cli", + scanPath: join(__dirname, "../../../examples/code-scanning"), }); expect(sortCodePackages(result)).to.deep.equal(sortCodePackages(expected)); }); }); -/* eslint-enable max-lines */ diff --git a/src/lib/tests/getGitFilesThatChanged.test.ts b/src/lib/tests/getGitFilesThatChanged.test.ts index 942e61de..a90c7116 100644 --- a/src/lib/tests/getGitFilesThatChanged.test.ts +++ b/src/lib/tests/getGitFilesThatChanged.test.ts @@ -1,22 +1,22 @@ -import { join } from 'path'; -import { describe, expect, it } from 'vitest'; -import { getGitFilesThatChanged } from '../ai/getGitFilesThatChanged'; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { getGitFilesThatChanged } from "../ai/getGitFilesThatChanged"; // not easy to test this but can uncomment to run against current commit -describe.skip('getGitFilesThatChanged', () => { - it('should remove links', () => { +describe.skip("getGitFilesThatChanged", () => { + it("should remove links", () => { expect( getGitFilesThatChanged({ - baseBranch: 'main', - githubRepo: 'https://github.com/transcend-io/cli.git', - rootDirectory: join(__dirname, '../../'), - }), + baseBranch: "main", + githubRepo: "https://github.com/transcend-io/cli.git", + rootDirectory: join(__dirname, "../../"), + }) ).to.deep.equal({ changedFiles: [ - 'package.json', - 'src/ai/TranscendAiPrompt.ts', - 'src/cli-analyze-pull-request.ts', - 'src/tests/TranscendAiPrompt.test.ts', + "package.json", + "src/ai/TranscendAiPrompt.ts", + "src/cli-analyze-pull-request.ts", + "src/tests/TranscendAiPrompt.test.ts", ], fileDiffs: {}, }); diff --git a/src/lib/tests/readTranscendYaml.test.ts b/src/lib/tests/readTranscendYaml.test.ts index 7de70005..2c632f87 100644 --- a/src/lib/tests/readTranscendYaml.test.ts +++ b/src/lib/tests/readTranscendYaml.test.ts @@ -1,18 +1,18 @@ -import { join } from 'path'; -import { describe, expect, it } from 'vitest'; -import { readTranscendYaml } from '../../index'; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { readTranscendYaml } from "../../index"; -const EXAMPLE_DIR = join(__dirname, '..', '..', '..', 'examples'); +const EXAMPLE_DIR = join(__dirname, "..", "..", "..", "examples"); -describe('readTranscendYaml', () => { - it('simple.yml should pass the codec validation for TranscendInput', () => { +describe("readTranscendYaml", () => { + it("simple.yml should pass the codec validation for TranscendInput", () => { expect(() => - readTranscendYaml(join(EXAMPLE_DIR, 'simple.yml')), + readTranscendYaml(join(EXAMPLE_DIR, "simple.yml")) ).to.not.throw(); }); - it('invalid.yml should fail the codec validation for TranscendInput', () => { - expect(() => readTranscendYaml(join(EXAMPLE_DIR, 'invalid.yml'))).to + it("invalid.yml should fail the codec validation for TranscendInput", () => { + expect(() => readTranscendYaml(join(EXAMPLE_DIR, "invalid.yml"))).to .throw(` ".enrichers.0.0.title expected type 'string'", ".enrichers.1.0.output-identifiers expected type 'Array'", ".data-silos.0.0.title expected type 'string'", @@ -22,26 +22,26 @@ describe('readTranscendYaml', () => { ".data-silos.1.1.disabled expected type 'boolean'"`); }); - it('multi-instance.yml should fail when no variables are provided', () => { + it("multi-instance.yml should fail when no variables are provided", () => { expect(() => - readTranscendYaml(join(EXAMPLE_DIR, 'multi-instance.yml')), - ).to.throw('Found variable that was not set: domain'); + readTranscendYaml(join(EXAMPLE_DIR, "multi-instance.yml")) + ).to.throw("Found variable that was not set: domain"); }); - it('multi-instance.yml should be successful when variables are provided', () => { - const result = readTranscendYaml(join(EXAMPLE_DIR, 'multi-instance.yml'), { - domain: 'acme.com', - stage: 'Staging', + it("multi-instance.yml should be successful when variables are provided", () => { + const result = readTranscendYaml(join(EXAMPLE_DIR, "multi-instance.yml"), { + domain: "acme.com", + stage: "Staging", }); - expect(result!.enrichers![0].url).to.equal( - 'https://example.acme.com/transcend-enrichment-webhook', + expect(result.enrichers![0].url).to.equal( + "https://example.acme.com/transcend-enrichment-webhook" ); - expect(result!.enrichers![1].url).to.equal( - 'https://example.acme.com/transcend-fraud-check', + expect(result.enrichers![1].url).to.equal( + "https://example.acme.com/transcend-fraud-check" ); - expect(result!['data-silos']![0].description).to.equal( - 'The mega-warehouse that contains a copy over all SQL backed databases - Staging', + expect(result["data-silos"]![0].description).to.equal( + "The mega-warehouse that contains a copy over all SQL backed databases - Staging" ); }); }); From 9ad407c2598624721b9f403c0530a8c911aab70c Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 20:53:13 -0400 Subject: [PATCH 3/4] eslint fix + format --- .eslintignore | 14 ---- README.md | 204 ++++++++++++++++++++++++++++------------------- eslint.config.js | 35 ++++---- 3 files changed, 138 insertions(+), 115 deletions(-) delete mode 100644 .eslintignore diff --git a/.eslintignore b/.eslintignore deleted file mode 100644 index 6bbe05c8..00000000 --- a/.eslintignore +++ /dev/null @@ -1,14 +0,0 @@ -.gitignore -.git/* -**/docs/* -**/package.json -**/node_modules/* -**/build/* -**/dist/* -**/builds/* -**/coverage/* -*.testjs -*.testts -examples/code-scanning/test-package-json/src/* -LICENSE -working/* \ No newline at end of file diff --git a/README.md b/README.md index 925906b0..4a243c5a 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,7 @@ data-silos: ## Usage + ### `transcend request approve` ```txt @@ -210,6 +211,7 @@ FLAGS [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] -h --help Print help information and exit ``` + ### `transcend request upload` ```txt @@ -242,6 +244,7 @@ FLAGS [--defaultPhoneCountryCode] When uploading phone numbers, if the phone number is missing a country code, assume this country code [default = 1] -h --help Print help information and exit ``` + ### `transcend request download-files` ```txt @@ -264,6 +267,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend request cancel` ```txt @@ -286,6 +290,7 @@ FLAGS [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] -h --help Print help information and exit ``` + ### `transcend request restart` ```txt @@ -314,6 +319,7 @@ FLAGS [--skipWaitingPeriod] Skip queued state of request and go straight to compiling [default = false] -h --help Print help information and exit ``` + ### `transcend request notify-additional-time` ```txt @@ -336,6 +342,7 @@ FLAGS [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] -h --help Print help information and exit ``` + ### `transcend request mark-silent` ```txt @@ -356,6 +363,7 @@ FLAGS [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] -h --help Print help information and exit ``` + ### `transcend request enricher-restart` ```txt @@ -380,6 +388,7 @@ FLAGS [--createdAtAfter] Restart requests that were submitted after this time -h --help Print help information and exit ``` + ### `transcend request reject-unverified-identifiers` ```txt @@ -396,6 +405,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend request export` ```txt @@ -419,6 +429,7 @@ FLAGS [--pageLimit] The page limit to use when pulling in pages of requests [default = 100] -h --help Print help information and exit ``` + ### `transcend request skip-preflight-jobs` ```txt @@ -434,6 +445,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend request system mark-request-data-silos-completed` ```txt @@ -451,6 +463,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend request system retry-request-data-silos` ```txt @@ -467,6 +480,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend request system skip-request-data-silos` ```txt @@ -484,6 +498,7 @@ FLAGS [--status] The status to set the request data silo job to [SKIPPED|RESOLVED, default = SKIPPED] -h --help Print help information and exit ``` + ### `transcend request preflight pull-identifiers` ```txt @@ -510,6 +525,7 @@ FLAGS [--concurrency] The concurrency to use when uploading requests in parallel [default = 100] -h --help Print help information and exit ``` + ### `transcend request preflight push-identifiers` ```txt @@ -537,6 +553,7 @@ FLAGS [--concurrency] The concurrency to use when uploading requests in parallel [default = 100] -h --help Print help information and exit ``` + ### `transcend request cron pull-identifiers` ```txt @@ -562,6 +579,7 @@ FLAGS [--chunkSize] Maximum number of rows per CSV file. For large datasets, the output will be automatically split into multiple files to avoid file system size limits. Each file will contain at most this many rows [default = 10000] -h --help Print help information and exit ``` + ### `transcend request cron mark-identifiers-completed` ```txt @@ -588,6 +606,7 @@ FLAGS [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra -h --help Print help information and exit ``` + ### `transcend consent build-xdi-sync-endpoint` ```txt @@ -607,6 +626,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend consent pull-consent-metrics` ```txt @@ -633,6 +653,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend consent pull-consent-preferences` ```txt @@ -663,7 +684,7 @@ Each row in the CSV will include: | timestamp | Timestamp for when consent was collected for that user | string - timestamp | N/A | true | | purposes | JSON map from consent purpose name -> boolean indicating whether user has opted in or out of that purpose | {[k in string]: boolean } | {} | true | | airgapVersion | Version of airgap where consent was collected | string | N/A | false | -| [purpose name] | Each consent purpose from `purposes` is also included in a column | boolean | N/A | false | +| [purpose name] | Each consent purpose from `purposes` is also included in a column | boolean | N/A | false | | tcf | IAB TCF string | string - TCF | N/A | false | | gpp | IAB GPP string | string - GPP | N/A | false | @@ -683,6 +704,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend consent upload-consent-preferences` ```txt @@ -715,6 +737,7 @@ Each row in the CSV must include: | gpp | IAB GPP string | string - GPP | N/A | false | An sample CSV can be found [here](./examples/preference-upload.csv). + ### `transcend consent upload-cookies-from-csv` ```txt @@ -737,6 +760,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend consent upload-data-flows-from-csv` ```txt @@ -760,6 +784,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend consent upload-preferences` ```txt @@ -812,6 +837,7 @@ Specifying the backend URL, needed for US hosted backend infrastructure: ```sh transcend consent upload-preferences --auth=$TRANSCEND_API_KEY --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 --consentUrl=https://consent.us.transcend.io ``` + ### `transcend inventory pull` ```txt @@ -848,39 +874,39 @@ FLAGS The API key permissions for this command vary based on the `resources` argument: -| Resource | Description | Scopes | Link | -| --- | --- | --- | --- | -| apiKeys | API Key definitions assigned to Data Silos. API keys cannot be created through the CLI, but you can map API key usage to Data Silos. | View API Keys | [Developer Tools -> API keys](https://app.transcend.io/infrastructure/api-keys) | -| customFields | Custom field definitions that define extra metadata for each table in the Admin Dashboard. | View Global Attributes | [Custom Fields](https://app.transcend.io/infrastructure/attributes) | -| templates | Email templates. Only template titles can be created and mapped to other resources. | View Email Templates | [DSR Automation -> Email Templates](https://app.transcend.io/privacy-requests/email-templates) | -| dataSilos | The Data Silo/Integration definitions. | View Data Map, View Data Subject Request Settings | [Data Inventory -> Data Silos](https://app.transcend.io/data-map/data-inventory/) and [Infrastucture -> Integrations](https://app.transcend.io/infrastructure/integrationsdata-silos) | -| enrichers | The Privacy Request enricher configurations. | View Identity Verification Settings | [DSR Automation -> Identifiers](https://app.transcend.io/privacy-requests/identifiers) | -| dataFlows | Consent Manager Data Flow definitions. | View Data Flows | [Consent Management -> Data Flows](https://app.transcend.io/consent-manager/data-flows/approved) | -| businessEntities | The business entities in the data inventory. | View Data Inventory | [Data Inventory -> Business Entities](https://app.transcend.io/data-map/data-inventory/business-entities) | -| actions | The Privacy Request action settings. | View Data Subject Request Settings | [DSR Automation -> Request Settings](https://app.transcend.io/privacy-requests/settings) | -| dataSubjects | The Privacy Request data subject settings. | View Data Subject Request Settings | [DSR Automation -> Request Settings](https://app.transcend.io/privacy-requests/settings) | -| identifiers | The Privacy Request identifier configurations. | View Identity Verification Settings | [DSR Automation -> Identifiers](https://app.transcend.io/privacy-requests/identifiers) | -| cookies | Consent Manager Cookie definitions. | View Data Flows | [Consent Management -> Cookies](https://app.transcend.io/consent-manager/cookies/approved) | -| consentManager | Consent Manager general settings, including domain list. | View Consent Manager | [Consent Management -> Developer Settings](https://app.transcend.io/consent-manager/developer-settings) | -| partitions | The partitions in the account (often representative of separate data controllers). | View Consent Manager | [Consent Management -> Developer Settings -> Advanced Settings](https://app.transcend.io/consent-manager/developer-settings/advanced-settings) | -| prompts | The Transcend AI prompts | View Prompts | [Prompt Manager -> Browse](https://app.transcend.io/prompts/browse) | -| promptPartials | The Transcend AI prompt partials | View Prompts | [Prompt Manager -> Partials](https://app.transcend.io/prompts/partialss) | -| promptGroups | The Transcend AI prompt groups | View Prompts | [Prompt Manager -> Groups](https://app.transcend.io/prompts/groups) | -| agents | The agents in the prompt manager. | View Prompts | [Prompt Manager -> Agents](https://app.transcend.io/prompts/agents) | -| agentFunctions | The agent functions in the prompt manager. | View Prompts | [Prompt Manager -> Agent Functions](https://app.transcend.io/prompts/agent-functions) | -| agentFiles | The agent files in the prompt manager. | View Prompts | [Prompt Manager -> Agent Files](https://app.transcend.io/prompts/agent-files) | -| vendors | The vendors in the data inventory. | View Data Inventory | [Data Inventory -> Vendors](https://app.transcend.io/data-map/data-inventory/vendors) | -| dataCategories | The data categories in the data inventory. | View Data Inventory | [Data Inventory -> Data Categories](https://app.transcend.io/data-map/data-inventory/data-categories) | -| processingPurposes | The processing purposes in the data inventory. | View Data Inventory | [Data Inventory -> Processing Purposes](https://app.transcend.io/data-map/data-inventory/purposes) | -| actionItems | Onboarding related action items | View All Action Items | [Action Items](https://app.transcend.io/action-items/all) | -| actionItemCollections | Onboarding related action item group names | View All Action Items | [Action Items](https://app.transcend.io/action-items/all) | -| teams | Team definitions of users and scope groupings | View Scopes | [Administration -> Teams](https://app.transcend.io/admin/teams) | -| privacyCenters | The privacy center configurations. | View Privacy Center Layout | [Privacy Center](https://app.transcend.io/privacy-center/general-settings) | -| policies | The privacy center policies. | View Policies | [Privacy Center -> Policies](https://app.transcend.io/privacy-center/policies) | -| messages | Message definitions used across consent, privacy center, email templates and more. | View Internationalization Messages | [Privacy Center -> Messages](https://app.transcend.io/privacy-center/messages-internationalization), [Consent Management -> Display Settings -> Messages](https://app.transcend.io/consent-manager/display-settings/messages) | -| assessments | Assessment responses. | View Assessments | [Assessments -> Assessments](https://app.transcend.io/assessments/groups) | -| assessmentTemplates | Assessment template configurations. | View Assessments | [Assessment -> Templates](https://app.transcend.io/assessments/form-templates) | -| purposes | Consent purposes and related preference management topics. | View Consent Manager, View Preference Store Settings | [Consent Management -> Regional Experiences -> Purposes](https://app.transcend.io/consent-manager/regional-experiences/purposes) | +| Resource | Description | Scopes | Link | +| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| apiKeys | API Key definitions assigned to Data Silos. API keys cannot be created through the CLI, but you can map API key usage to Data Silos. | View API Keys | [Developer Tools -> API keys](https://app.transcend.io/infrastructure/api-keys) | +| customFields | Custom field definitions that define extra metadata for each table in the Admin Dashboard. | View Global Attributes | [Custom Fields](https://app.transcend.io/infrastructure/attributes) | +| templates | Email templates. Only template titles can be created and mapped to other resources. | View Email Templates | [DSR Automation -> Email Templates](https://app.transcend.io/privacy-requests/email-templates) | +| dataSilos | The Data Silo/Integration definitions. | View Data Map, View Data Subject Request Settings | [Data Inventory -> Data Silos](https://app.transcend.io/data-map/data-inventory/) and [Infrastucture -> Integrations](https://app.transcend.io/infrastructure/integrationsdata-silos) | +| enrichers | The Privacy Request enricher configurations. | View Identity Verification Settings | [DSR Automation -> Identifiers](https://app.transcend.io/privacy-requests/identifiers) | +| dataFlows | Consent Manager Data Flow definitions. | View Data Flows | [Consent Management -> Data Flows](https://app.transcend.io/consent-manager/data-flows/approved) | +| businessEntities | The business entities in the data inventory. | View Data Inventory | [Data Inventory -> Business Entities](https://app.transcend.io/data-map/data-inventory/business-entities) | +| actions | The Privacy Request action settings. | View Data Subject Request Settings | [DSR Automation -> Request Settings](https://app.transcend.io/privacy-requests/settings) | +| dataSubjects | The Privacy Request data subject settings. | View Data Subject Request Settings | [DSR Automation -> Request Settings](https://app.transcend.io/privacy-requests/settings) | +| identifiers | The Privacy Request identifier configurations. | View Identity Verification Settings | [DSR Automation -> Identifiers](https://app.transcend.io/privacy-requests/identifiers) | +| cookies | Consent Manager Cookie definitions. | View Data Flows | [Consent Management -> Cookies](https://app.transcend.io/consent-manager/cookies/approved) | +| consentManager | Consent Manager general settings, including domain list. | View Consent Manager | [Consent Management -> Developer Settings](https://app.transcend.io/consent-manager/developer-settings) | +| partitions | The partitions in the account (often representative of separate data controllers). | View Consent Manager | [Consent Management -> Developer Settings -> Advanced Settings](https://app.transcend.io/consent-manager/developer-settings/advanced-settings) | +| prompts | The Transcend AI prompts | View Prompts | [Prompt Manager -> Browse](https://app.transcend.io/prompts/browse) | +| promptPartials | The Transcend AI prompt partials | View Prompts | [Prompt Manager -> Partials](https://app.transcend.io/prompts/partialss) | +| promptGroups | The Transcend AI prompt groups | View Prompts | [Prompt Manager -> Groups](https://app.transcend.io/prompts/groups) | +| agents | The agents in the prompt manager. | View Prompts | [Prompt Manager -> Agents](https://app.transcend.io/prompts/agents) | +| agentFunctions | The agent functions in the prompt manager. | View Prompts | [Prompt Manager -> Agent Functions](https://app.transcend.io/prompts/agent-functions) | +| agentFiles | The agent files in the prompt manager. | View Prompts | [Prompt Manager -> Agent Files](https://app.transcend.io/prompts/agent-files) | +| vendors | The vendors in the data inventory. | View Data Inventory | [Data Inventory -> Vendors](https://app.transcend.io/data-map/data-inventory/vendors) | +| dataCategories | The data categories in the data inventory. | View Data Inventory | [Data Inventory -> Data Categories](https://app.transcend.io/data-map/data-inventory/data-categories) | +| processingPurposes | The processing purposes in the data inventory. | View Data Inventory | [Data Inventory -> Processing Purposes](https://app.transcend.io/data-map/data-inventory/purposes) | +| actionItems | Onboarding related action items | View All Action Items | [Action Items](https://app.transcend.io/action-items/all) | +| actionItemCollections | Onboarding related action item group names | View All Action Items | [Action Items](https://app.transcend.io/action-items/all) | +| teams | Team definitions of users and scope groupings | View Scopes | [Administration -> Teams](https://app.transcend.io/admin/teams) | +| privacyCenters | The privacy center configurations. | View Privacy Center Layout | [Privacy Center](https://app.transcend.io/privacy-center/general-settings) | +| policies | The privacy center policies. | View Policies | [Privacy Center -> Policies](https://app.transcend.io/privacy-center/policies) | +| messages | Message definitions used across consent, privacy center, email templates and more. | View Internationalization Messages | [Privacy Center -> Messages](https://app.transcend.io/privacy-center/messages-internationalization), [Consent Management -> Display Settings -> Messages](https://app.transcend.io/consent-manager/display-settings/messages) | +| assessments | Assessment responses. | View Assessments | [Assessments -> Assessments](https://app.transcend.io/assessments/groups) | +| assessmentTemplates | Assessment template configurations. | View Assessments | [Assessment -> Templates](https://app.transcend.io/assessments/form-templates) | +| purposes | Consent purposes and related preference management topics. | View Consent Manager, View Preference Store Settings | [Consent Management -> Regional Experiences -> Purposes](https://app.transcend.io/consent-manager/regional-experiences/purposes) | #### Usage @@ -1107,32 +1133,32 @@ name: Transcend Data Map Syncing # See https://app.transcend.io/privacy-requests/connected-services on: - push: - branches: - - 'main' + push: + branches: + - 'main' jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 - - name: Setup Node.js - uses: actions/setup-node@v2 - with: - node-version: '16' + - name: Setup Node.js + uses: actions/setup-node@v2 + with: + node-version: '16' - - name: Install Transcend CLI - run: npm install --global @transcend-io/cli + - name: Install Transcend CLI + run: npm install --global @transcend-io/cli - # If you have a script that generates your transcend.yml file from - # an ORM or infrastructure configuration, add that step here - # Leave this step commented out if you want to manage your transcend.yml manually - # - name: Generate transcend.yml - # run: ./scripts/generate_transcend_yml.py + # If you have a script that generates your transcend.yml file from + # an ORM or infrastructure configuration, add that step here + # Leave this step commented out if you want to manage your transcend.yml manually + # - name: Generate transcend.yml + # run: ./scripts/generate_transcend_yml.py - - name: Push Transcend config - run: transcend inventory push --auth=${{ secrets.TRANSCEND_API_KEY }} + - name: Push Transcend config + run: transcend inventory push --auth=${{ secrets.TRANSCEND_API_KEY }} ``` #### Dynamic Variables @@ -1149,32 +1175,32 @@ This command could fill out multiple parameters in a YAML file like [./examples/ ```yml api-keys: - - title: Webhook Key + - title: Webhook Key enrichers: - - title: Basic Identity Enrichment - description: Enrich an email address to the userId and phone number - # The data silo webhook URL is the same in each environment, - # except for the base domain in the webhook URL. - url: https://example.<>/transcend-enrichment-webhook - input-identifier: email - output-identifiers: - - userId - - phone - - myUniqueIdentifier - - title: Fraud Check - description: Ensure the email address is not marked as fraudulent - url: https://example.<>/transcend-fraud-check - input-identifier: email - output-identifiers: - - email - privacy-actions: - - ERASURE + - title: Basic Identity Enrichment + description: Enrich an email address to the userId and phone number + # The data silo webhook URL is the same in each environment, + # except for the base domain in the webhook URL. + url: https://example.<>/transcend-enrichment-webhook + input-identifier: email + output-identifiers: + - userId + - phone + - myUniqueIdentifier + - title: Fraud Check + description: Ensure the email address is not marked as fraudulent + url: https://example.<>/transcend-fraud-check + input-identifier: email + output-identifiers: + - email + privacy-actions: + - ERASURE data-silos: - - title: Redshift Data Warehouse - integrationName: server - description: The mega-warehouse that contains a copy over all SQL backed databases - <> - url: https://example.<>/transcend-webhook - api-key-title: Webhook Key + - title: Redshift Data Warehouse + integrationName: server + description: The mega-warehouse that contains a copy over all SQL backed databases - <> + url: https://example.<>/transcend-webhook + api-key-title: Webhook Key ``` ### `transcend inventory scan-packages` @@ -1210,6 +1236,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend inventory discover-silos` ```txt @@ -1246,6 +1273,7 @@ transcend inventory discover-silos --scanPath=./examples/ --auth=$TRANSCEND_API_ ``` This call will look for all the package.json files that in the scan path `./myJavascriptProject`, parse each of the dependencies into their individual package names, and send it to our Transcend backend for classification. These classifications can then be viewed [here](https://app.transcend.io/data-map/data-inventory/silo-discovery/triage). The process is the same for scanning requirements.txt, podfiles and build.gradle files. + ### `transcend inventory pull-datapoints` ```txt @@ -1266,6 +1294,7 @@ FLAGS [--subCategories]... List of subcategories to filter by [separator = ,] -h --help Print help information and exit ``` + ### `transcend inventory pull-unstructured-discovery-files` ```txt @@ -1285,6 +1314,7 @@ FLAGS [--includeEncryptedSnippets] Whether to include encrypted snippets of the entries classified [default = false] -h --help Print help information and exit ``` + ### `transcend inventory derive-data-silos-from-data-flows` ```txt @@ -1302,6 +1332,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend inventory derive-data-silos-from-data-flows-cross-instance` ```txt @@ -1319,6 +1350,7 @@ FLAGS [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] -h --help Print help information and exit ``` + ### `transcend inventory consent-manager-service-json-to-yml` ```txt @@ -1339,6 +1371,7 @@ FLAGS [--output] Path to the output transcend.yml to write to [default = ./transcend.yml] -h --help Print help information and exit ``` + ### `transcend inventory consent-managers-to-business-entities` ```txt @@ -1353,6 +1386,7 @@ FLAGS [--output] Path to the output transcend.yml with business entity configuration [default = ./combined-business-entities.yml] -h --help Print help information and exit ``` + ### `transcend admin generate-api-keys` ```txt @@ -1398,12 +1432,12 @@ Filter for only a specific organization by ID, returning all child accounts asso ```gql query { - user { - organization { - id - parentOrganizationId - } - } + user { + organization { + id + parentOrganizationId + } + } } ``` @@ -1428,6 +1462,7 @@ transcend admin generate-api-keys --email=test@transcend.io --password=$TRANSCE --scopes="View Email Templates,View Data Map" --apiKeyTitle="CLI Usage Cross Instance Sync" -file=./working/auth.json \ --deleteExistingApiKey=false ``` + ### `transcend migration sync-ot` ```txt @@ -1456,6 +1491,7 @@ FLAGS [--debug] Whether to print detailed logs in case of error [default = false] -h --help Print help information and exit ``` + ## Prompt Manager diff --git a/eslint.config.js b/eslint.config.js index 3bd4cde5..f8bff766 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,7 +1,7 @@ // @ts-check -import eslint from '@eslint/js'; -import eslintPluginUnicorn from 'eslint-plugin-unicorn'; -import tseslint from 'typescript-eslint'; +import eslint from "@eslint/js"; +import eslintPluginUnicorn from "eslint-plugin-unicorn"; +import tseslint from "typescript-eslint"; /** * @type {import('typescript-eslint').Config} @@ -25,33 +25,34 @@ const eslintConfig = tseslint.config( }, { rules: { - 'unicorn/filename-case': [ - 'error', + "unicorn/filename-case": [ + "error", { - case: 'kebabCase', + case: "kebabCase", }, ], + "unicorn/no-nested-ternary": "off", }, }, { // CJS compatibility for src directory since it's also a library - files: ['src/**/*'], + files: ["src/**/*"], // The CLI does not require CJS compatibility - ignores: ['src/bin/**/*', 'src/commands/**/*'], + ignores: ["src/bin/**/*", "src/commands/**/*"], rules: { // Ban top-level await - 'unicorn/prefer-top-level-await': 'off', - 'no-restricted-syntax': [ - 'error', + "unicorn/prefer-top-level-await": "off", + "no-restricted-syntax": [ + "error", { // Matches await expressions at the top level (direct child of Program) - selector: 'Program > ExpressionStatement > AwaitExpression', - message: 'Top-level await is not allowed in CJS-compatible files.', + selector: "Program > ExpressionStatement > AwaitExpression", + message: "Top-level await is not allowed in CJS-compatible files.", }, ], // Ban default exports - 'no-restricted-exports': [ - 'error', + "no-restricted-exports": [ + "error", { restrictDefaultExports: { direct: true, @@ -65,8 +66,8 @@ const eslintConfig = tseslint.config( }, }, { - ignores: ['dist', 'examples'], - }, + ignores: ["dist", "examples"], + } ); export default eslintConfig; From 82450517093eb67d621d8fed3e6284bf27c63a92 Mon Sep 17 00:00:00 2001 From: bencmbrook <7354176+bencmbrook@users.noreply.github.com> Date: Wed, 16 Jul 2025 21:03:39 -0400 Subject: [PATCH 4/4] format --- README.md | 92 +-- eslint.config.js | 36 +- scripts/buildPathfinderJsonSchema.ts | 18 +- scripts/buildTranscendJsonSchema.ts | 20 +- src/codecs.ts | 244 +++---- src/commands/admin/generate-api-keys/impl.ts | 28 +- .../consent/build-xdi-sync-endpoint/impl.ts | 20 +- .../consent/pull-consent-metrics/impl.ts | 78 +-- .../consent/upload-preferences/impl.ts | 50 +- .../impl.ts | 30 +- .../impl.ts | 30 +- .../impl.ts | 74 +-- .../derive-data-silos-from-data-flows/impl.ts | 38 +- .../inventory/pull-datapoints/impl.ts | 52 +- .../pull-unstructured-discovery-files/impl.ts | 50 +- src/commands/inventory/pull/impl.ts | 62 +- src/commands/inventory/push/impl.ts | 86 +-- src/commands/inventory/scan-packages/impl.ts | 36 +- src/commands/migration/sync-ot/impl.ts | 44 +- .../request/cron/pull-identifiers/impl.ts | 30 +- .../request/cron/pull-profiles/impl.ts | 58 +- src/commands/request/export/impl.ts | 20 +- src/constants.ts | 78 +-- src/lib/ai/TranscendPromptManager.ts | 132 ++-- src/lib/ai/filterNullishValuesFromObject.ts | 10 +- src/lib/ai/getGitFilesThatChanged.ts | 28 +- src/lib/ai/removeLinks.ts | 2 +- .../api-keys/generateCrossAccountApiKeys.ts | 68 +- src/lib/api-keys/listDirectories.ts | 6 +- src/lib/api-keys/listFiles.ts | 10 +- src/lib/api-keys/validateTranscendAuth.ts | 18 +- src/lib/bluebird-replace.ts | 6 +- src/lib/code-scanning/constants.ts | 6 +- .../code-scanning/findCodePackagesInFolder.ts | 38 +- src/lib/code-scanning/findFilesToScan.ts | 26 +- .../code-scanning/integrations/cocoaPods.ts | 42 +- .../integrations/composerJson.ts | 26 +- src/lib/code-scanning/integrations/gemfile.ts | 42 +- src/lib/code-scanning/integrations/gradle.ts | 70 +- .../integrations/javascriptPackageJson.ts | 28 +- src/lib/code-scanning/integrations/pubspec.ts | 42 +- .../integrations/pythonRequirementsTxt.ts | 30 +- src/lib/code-scanning/integrations/swift.ts | 20 +- .../consent-manager/buildXdiSyncEndpoint.ts | 36 +- .../consentManagersToBusinessEntities.ts | 26 +- src/lib/consent-manager/createConsentToken.ts | 18 +- .../consent-manager/dataFlowsToDataSilos.ts | 38 +- src/lib/consent-manager/uploadConsents.ts | 72 +-- .../consent-manager/uploadCookiesFromCsv.ts | 46 +- .../consent-manager/uploadDataFlowsFromCsv.ts | 44 +- src/lib/cron/markCronIdentifierCompleted.ts | 12 +- .../cron/markRequestDataSiloIdsCompleted.ts | 28 +- ...ChunkedCustomSiloOutstandingIdentifiers.ts | 40 +- src/lib/cron/pullCronPageOfIdentifiers.ts | 16 +- .../pullCustomSiloOutstandingIdentifiers.ts | 40 +- src/lib/cron/pushCronIdentifiersFromCsv.ts | 50 +- src/lib/cron/writeCsv.ts | 32 +- src/lib/data-inventory/pullAllDatapoints.ts | 82 +-- ...UnstructuredSubDataPointRecommendations.ts | 38 +- src/lib/graphql/fetchAllAssessments.ts | 10 +- src/lib/graphql/fetchAllRequestIdentifiers.ts | 28 +- src/lib/graphql/fetchAllRequests.ts | 38 +- src/lib/graphql/fetchApiKeys.ts | 44 +- src/lib/graphql/fetchCatalogs.ts | 14 +- src/lib/graphql/fetchDataSubjects.ts | 42 +- src/lib/graphql/fetchIdentifiers.ts | 44 +- src/lib/graphql/fetchPrompts.ts | 12 +- src/lib/graphql/fetchRequestDataSilo.ts | 26 +- src/lib/graphql/formatAttributeValues.ts | 6 +- src/lib/graphql/gqls/consentManager.ts | 2 +- src/lib/graphql/loginUser.ts | 16 +- src/lib/graphql/makeGraphQLRequest.ts | 40 +- src/lib/graphql/pullTranscendConfiguration.ts | 438 ++++++------- src/lib/graphql/syncActionItemCollections.ts | 55 +- src/lib/graphql/syncActionItems.ts | 74 +-- src/lib/graphql/syncAgentFiles.ts | 44 +- src/lib/graphql/syncAgentFunctions.ts | 48 +- src/lib/graphql/syncAgents.ts | 42 +- src/lib/graphql/syncAttribute.ts | 32 +- src/lib/graphql/syncBusinessEntities.ts | 50 +- src/lib/graphql/syncCodePackages.ts | 86 +-- .../graphql/syncConfigurationToTranscend.ts | 206 +++--- src/lib/graphql/syncConsentManager.ts | 54 +- src/lib/graphql/syncCookies.ts | 28 +- src/lib/graphql/syncDataCategories.ts | 48 +- src/lib/graphql/syncDataFlows.ts | 50 +- src/lib/graphql/syncDataSilos.ts | 228 +++---- src/lib/graphql/syncDataSubject.ts | 12 +- src/lib/graphql/syncEnrichers.ts | 42 +- src/lib/graphql/syncIdentifier.ts | 12 +- src/lib/graphql/syncIntlMessages.ts | 30 +- src/lib/graphql/syncPartitions.ts | 30 +- src/lib/graphql/syncPolicies.ts | 36 +- src/lib/graphql/syncPrivacyCenter.ts | 22 +- src/lib/graphql/syncProcessingPurposes.ts | 58 +- src/lib/graphql/syncPromptGroups.ts | 62 +- src/lib/graphql/syncPromptPartials.ts | 58 +- src/lib/graphql/syncPrompts.ts | 38 +- src/lib/graphql/syncRepositories.ts | 52 +- .../graphql/syncSoftwareDevelopmentKits.ts | 74 +-- src/lib/graphql/syncTeams.ts | 50 +- src/lib/graphql/syncVendors.ts | 44 +- src/lib/helpers/inquirer.ts | 24 +- src/lib/helpers/parseVariablesFromString.ts | 8 +- .../manual-enrichment/enrichPrivacyRequest.ts | 40 +- .../pullManualEnrichmentIdentifiersToCsv.ts | 46 +- .../pushManualEnrichmentIdentifiersFromCsv.ts | 26 +- src/lib/mergeTranscendInputs.ts | 4 +- .../oneTrust/helpers/convertToEmptyStrings.ts | 8 +- .../helpers/parseCliSyncOtArguments.ts | 74 +-- .../helpers/syncOneTrustAssessmentToDisk.ts | 18 +- .../syncOneTrustAssessmentToTranscend.ts | 26 +- .../syncOneTrustAssessmentsFromFile.ts | 38 +- .../syncOneTrustAssessmentsFromOneTrust.ts | 66 +- .../tests/convertToEmptyStrings.test.ts | 50 +- .../checkIfPendingPreferenceUpdatesAreNoOp.ts | 18 +- ...IfPendingPreferenceUpdatesCauseConflict.ts | 18 +- .../getPreferenceUpdatesFromRow.ts | 44 +- .../getPreferencesForIdentifiers.ts | 42 +- .../parsePreferenceAndPurposeValuesFromCsv.ts | 66 +- .../parsePreferenceIdentifiersFromCsv.ts | 48 +- .../parsePreferenceManagementCsv.ts | 60 +- .../parsePreferenceTimestampsFromCsv.ts | 34 +- ...kIfPendingPreferenceUpdatesAreNoOp.test.ts | 240 +++---- ...dingPreferenceUpdatesCauseConflict.test.ts | 246 +++---- .../tests/getPreferenceUpdatesFromRow.test.ts | 510 +++++++-------- ...ferenceManagementPreferencesInteractive.ts | 96 +-- src/lib/readTranscendYaml.ts | 22 +- src/lib/requests/approvePrivacyRequests.ts | 24 +- src/lib/requests/bulkRestartRequests.ts | 74 +-- src/lib/requests/bulkRetryEnrichers.ts | 36 +- src/lib/requests/cancelPrivacyRequests.ts | 34 +- src/lib/requests/constants.ts | 44 +- .../requests/downloadPrivacyRequestFiles.ts | 34 +- src/lib/requests/filterRows.ts | 26 +- .../getFileMetadataForPrivacyRequests.ts | 42 +- src/lib/requests/getUniqueValuesForColumn.ts | 8 +- src/lib/requests/mapColumnsToAttributes.ts | 26 +- src/lib/requests/mapColumnsToIdentifiers.ts | 28 +- src/lib/requests/mapCsvColumnsToApi.ts | 28 +- src/lib/requests/mapCsvRowsToRequestInputs.ts | 100 +-- src/lib/requests/mapEnumValues.ts | 22 +- src/lib/requests/mapRequestEnumValues.ts | 66 +- src/lib/requests/markSilentPrivacyRequests.ts | 24 +- .../notifyPrivacyRequestsAdditionalTime.ts | 34 +- src/lib/requests/pullPrivacyRequests.ts | 63 +- src/lib/requests/readCsv.ts | 20 +- .../removeUnverifiedRequestIdentifiers.ts | 28 +- src/lib/requests/retryRequestDataSilos.ts | 28 +- src/lib/requests/skipPreflightJobs.ts | 32 +- src/lib/requests/skipRequestDataSilos.ts | 32 +- src/lib/requests/splitCsvToList.ts | 2 +- src/lib/requests/streamPrivacyRequestFiles.ts | 28 +- src/lib/requests/submitPrivacyRequest.ts | 32 +- .../tests/mapCsvRowsToRequestInputs.test.ts | 4 +- src/lib/requests/tests/readCsv.test.ts | 72 +-- .../requests/uploadPrivacyRequestsFromCsv.ts | 124 ++-- .../tests/findCodePackagesInFolder.test.ts | 604 +++++++++--------- src/lib/tests/getGitFilesThatChanged.test.ts | 26 +- src/lib/tests/readTranscendYaml.test.ts | 40 +- 160 files changed, 4318 insertions(+), 4306 deletions(-) diff --git a/README.md b/README.md index 4a243c5a..30de3733 100644 --- a/README.md +++ b/README.md @@ -10,52 +10,52 @@ - [Installation](#installation) - [transcend.yml](#transcendyml) - [Usage](#usage) - - [`transcend request approve`](#transcend-request-approve) - - [`transcend request upload`](#transcend-request-upload) - - [`transcend request download-files`](#transcend-request-download-files) - - [`transcend request cancel`](#transcend-request-cancel) - - [`transcend request restart`](#transcend-request-restart) - - [`transcend request notify-additional-time`](#transcend-request-notify-additional-time) - - [`transcend request mark-silent`](#transcend-request-mark-silent) - - [`transcend request enricher-restart`](#transcend-request-enricher-restart) - - [`transcend request reject-unverified-identifiers`](#transcend-request-reject-unverified-identifiers) - - [`transcend request export`](#transcend-request-export) - - [`transcend request skip-preflight-jobs`](#transcend-request-skip-preflight-jobs) - - [`transcend request system mark-request-data-silos-completed`](#transcend-request-system-mark-request-data-silos-completed) - - [`transcend request system retry-request-data-silos`](#transcend-request-system-retry-request-data-silos) - - [`transcend request system skip-request-data-silos`](#transcend-request-system-skip-request-data-silos) - - [`transcend request preflight pull-identifiers`](#transcend-request-preflight-pull-identifiers) - - [`transcend request preflight push-identifiers`](#transcend-request-preflight-push-identifiers) - - [`transcend request cron pull-identifiers`](#transcend-request-cron-pull-identifiers) - - [`transcend request cron mark-identifiers-completed`](#transcend-request-cron-mark-identifiers-completed) - - [`transcend consent build-xdi-sync-endpoint`](#transcend-consent-build-xdi-sync-endpoint) - - [`transcend consent pull-consent-metrics`](#transcend-consent-pull-consent-metrics) - - [`transcend consent pull-consent-preferences`](#transcend-consent-pull-consent-preferences) - - [`transcend consent update-consent-manager`](#transcend-consent-update-consent-manager) - - [`transcend consent upload-consent-preferences`](#transcend-consent-upload-consent-preferences) - - [`transcend consent upload-cookies-from-csv`](#transcend-consent-upload-cookies-from-csv) - - [`transcend consent upload-data-flows-from-csv`](#transcend-consent-upload-data-flows-from-csv) - - [`transcend consent upload-preferences`](#transcend-consent-upload-preferences) - - [`transcend inventory pull`](#transcend-inventory-pull) - - [Scopes](#scopes) - - [Usage](#usage-1) - - [`transcend inventory push`](#transcend-inventory-push) - - [Scopes](#scopes-1) - - [Usage](#usage-2) - - [CI Integration](#ci-integration) - - [Dynamic Variables](#dynamic-variables) - - [`transcend inventory scan-packages`](#transcend-inventory-scan-packages) - - [`transcend inventory discover-silos`](#transcend-inventory-discover-silos) - - [Usage](#usage-3) - - [`transcend inventory pull-datapoints`](#transcend-inventory-pull-datapoints) - - [`transcend inventory pull-unstructured-discovery-files`](#transcend-inventory-pull-unstructured-discovery-files) - - [`transcend inventory derive-data-silos-from-data-flows`](#transcend-inventory-derive-data-silos-from-data-flows) - - [`transcend inventory derive-data-silos-from-data-flows-cross-instance`](#transcend-inventory-derive-data-silos-from-data-flows-cross-instance) - - [`transcend inventory consent-manager-service-json-to-yml`](#transcend-inventory-consent-manager-service-json-to-yml) - - [`transcend inventory consent-managers-to-business-entities`](#transcend-inventory-consent-managers-to-business-entities) - - [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys) - - [Usage](#usage-4) - - [`transcend migration sync-ot`](#transcend-migration-sync-ot) + - [`transcend request approve`](#transcend-request-approve) + - [`transcend request upload`](#transcend-request-upload) + - [`transcend request download-files`](#transcend-request-download-files) + - [`transcend request cancel`](#transcend-request-cancel) + - [`transcend request restart`](#transcend-request-restart) + - [`transcend request notify-additional-time`](#transcend-request-notify-additional-time) + - [`transcend request mark-silent`](#transcend-request-mark-silent) + - [`transcend request enricher-restart`](#transcend-request-enricher-restart) + - [`transcend request reject-unverified-identifiers`](#transcend-request-reject-unverified-identifiers) + - [`transcend request export`](#transcend-request-export) + - [`transcend request skip-preflight-jobs`](#transcend-request-skip-preflight-jobs) + - [`transcend request system mark-request-data-silos-completed`](#transcend-request-system-mark-request-data-silos-completed) + - [`transcend request system retry-request-data-silos`](#transcend-request-system-retry-request-data-silos) + - [`transcend request system skip-request-data-silos`](#transcend-request-system-skip-request-data-silos) + - [`transcend request preflight pull-identifiers`](#transcend-request-preflight-pull-identifiers) + - [`transcend request preflight push-identifiers`](#transcend-request-preflight-push-identifiers) + - [`transcend request cron pull-identifiers`](#transcend-request-cron-pull-identifiers) + - [`transcend request cron mark-identifiers-completed`](#transcend-request-cron-mark-identifiers-completed) + - [`transcend consent build-xdi-sync-endpoint`](#transcend-consent-build-xdi-sync-endpoint) + - [`transcend consent pull-consent-metrics`](#transcend-consent-pull-consent-metrics) + - [`transcend consent pull-consent-preferences`](#transcend-consent-pull-consent-preferences) + - [`transcend consent update-consent-manager`](#transcend-consent-update-consent-manager) + - [`transcend consent upload-consent-preferences`](#transcend-consent-upload-consent-preferences) + - [`transcend consent upload-cookies-from-csv`](#transcend-consent-upload-cookies-from-csv) + - [`transcend consent upload-data-flows-from-csv`](#transcend-consent-upload-data-flows-from-csv) + - [`transcend consent upload-preferences`](#transcend-consent-upload-preferences) + - [`transcend inventory pull`](#transcend-inventory-pull) + - [Scopes](#scopes) + - [Usage](#usage-1) + - [`transcend inventory push`](#transcend-inventory-push) + - [Scopes](#scopes-1) + - [Usage](#usage-2) + - [CI Integration](#ci-integration) + - [Dynamic Variables](#dynamic-variables) + - [`transcend inventory scan-packages`](#transcend-inventory-scan-packages) + - [`transcend inventory discover-silos`](#transcend-inventory-discover-silos) + - [Usage](#usage-3) + - [`transcend inventory pull-datapoints`](#transcend-inventory-pull-datapoints) + - [`transcend inventory pull-unstructured-discovery-files`](#transcend-inventory-pull-unstructured-discovery-files) + - [`transcend inventory derive-data-silos-from-data-flows`](#transcend-inventory-derive-data-silos-from-data-flows) + - [`transcend inventory derive-data-silos-from-data-flows-cross-instance`](#transcend-inventory-derive-data-silos-from-data-flows-cross-instance) + - [`transcend inventory consent-manager-service-json-to-yml`](#transcend-inventory-consent-manager-service-json-to-yml) + - [`transcend inventory consent-managers-to-business-entities`](#transcend-inventory-consent-managers-to-business-entities) + - [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys) + - [Usage](#usage-4) + - [`transcend migration sync-ot`](#transcend-migration-sync-ot) - [Prompt Manager](#prompt-manager) - [Proxy usage](#proxy-usage) diff --git a/eslint.config.js b/eslint.config.js index f8bff766..af2ce018 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,7 +1,7 @@ // @ts-check -import eslint from "@eslint/js"; -import eslintPluginUnicorn from "eslint-plugin-unicorn"; -import tseslint from "typescript-eslint"; +import eslint from '@eslint/js'; +import eslintPluginUnicorn from 'eslint-plugin-unicorn'; +import tseslint from 'typescript-eslint'; /** * @type {import('typescript-eslint').Config} @@ -25,34 +25,34 @@ const eslintConfig = tseslint.config( }, { rules: { - "unicorn/filename-case": [ - "error", + 'unicorn/filename-case': [ + 'error', { - case: "kebabCase", + case: 'kebabCase', }, ], - "unicorn/no-nested-ternary": "off", + 'unicorn/no-nested-ternary': 'off', }, }, { // CJS compatibility for src directory since it's also a library - files: ["src/**/*"], + files: ['src/**/*'], // The CLI does not require CJS compatibility - ignores: ["src/bin/**/*", "src/commands/**/*"], + ignores: ['src/bin/**/*', 'src/commands/**/*'], rules: { // Ban top-level await - "unicorn/prefer-top-level-await": "off", - "no-restricted-syntax": [ - "error", + 'unicorn/prefer-top-level-await': 'off', + 'no-restricted-syntax': [ + 'error', { // Matches await expressions at the top level (direct child of Program) - selector: "Program > ExpressionStatement > AwaitExpression", - message: "Top-level await is not allowed in CJS-compatible files.", + selector: 'Program > ExpressionStatement > AwaitExpression', + message: 'Top-level await is not allowed in CJS-compatible files.', }, ], // Ban default exports - "no-restricted-exports": [ - "error", + 'no-restricted-exports': [ + 'error', { restrictDefaultExports: { direct: true, @@ -66,8 +66,8 @@ const eslintConfig = tseslint.config( }, }, { - ignores: ["dist", "examples"], - } + ignores: ['dist', 'examples'], + }, ); export default eslintConfig; diff --git a/scripts/buildPathfinderJsonSchema.ts b/scripts/buildPathfinderJsonSchema.ts index 5db31b12..ba22da89 100644 --- a/scripts/buildPathfinderJsonSchema.ts +++ b/scripts/buildPathfinderJsonSchema.ts @@ -12,16 +12,16 @@ * @see https://github.com/SchemaStore/schemastore */ -import { writeFileSync } from "node:fs"; -import { join } from "node:path"; -import { toJsonSchema } from "@transcend-io/type-utils"; -import { PathfinderPolicy } from "../src/codecs"; +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { toJsonSchema } from '@transcend-io/type-utils'; +import { PathfinderPolicy } from '../src/codecs'; const schemaDefaults = { - $schema: "http://json-schema.org/draft-07/schema#", - $id: "https://raw.githubusercontent.com/transcend-io/cli/main/pathfinder-policy-yml-schema.json", - title: "pathfinder.yml", - description: "Policies for the Transcend Pathfinder AI governance proxy.", + $schema: 'http://json-schema.org/draft-07/schema#', + $id: 'https://raw.githubusercontent.com/transcend-io/cli/main/pathfinder-policy-yml-schema.json', + title: 'pathfinder.yml', + description: 'Policies for the Transcend Pathfinder AI governance proxy.', }; // Build the JSON schema from io-ts codec @@ -30,6 +30,6 @@ const jsonSchema = { ...toJsonSchema(PathfinderPolicy, true), }; -const schemaFilePath = join(process.cwd(), "pathfinder-policy-yml-schema.json"); +const schemaFilePath = join(process.cwd(), 'pathfinder-policy-yml-schema.json'); writeFileSync(schemaFilePath, `${JSON.stringify(jsonSchema, null, 2)}\n`); diff --git a/scripts/buildTranscendJsonSchema.ts b/scripts/buildTranscendJsonSchema.ts index 3087a3fa..9c00ec6c 100644 --- a/scripts/buildTranscendJsonSchema.ts +++ b/scripts/buildTranscendJsonSchema.ts @@ -12,24 +12,24 @@ * @see https://github.com/SchemaStore/schemastore */ -import { writeFileSync } from "node:fs"; -import { join } from "node:path"; -import { toJsonSchema } from "@transcend-io/type-utils"; -import * as packageJson from "../package.json"; -import { TranscendInput } from "../src/codecs"; +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { toJsonSchema } from '@transcend-io/type-utils'; +import * as packageJson from '../package.json'; +import { TranscendInput } from '../src/codecs'; -const majorVersion = packageJson.version.split(".")[0]; +const majorVersion = packageJson.version.split('.')[0]; // Create a major version JSON schema definition, and update the latest JSON schema definition. -for (const key of [`v${majorVersion}`, "latest"]) { +for (const key of [`v${majorVersion}`, 'latest']) { const fileName = `transcend-yml-schema-${key}.json`; const schemaDefaults = { - $schema: "http://json-schema.org/draft-07/schema#", + $schema: 'http://json-schema.org/draft-07/schema#', $id: `https://raw.githubusercontent.com/transcend-io/cli/main/${fileName}`, - title: "transcend.yml", + title: 'transcend.yml', description: - "Define personal data schema and Transcend config as code with the Transcend CLI.", + 'Define personal data schema and Transcend config as code with the Transcend CLI.', }; // Build the JSON schema from io-ts codec diff --git a/src/codecs.ts b/src/codecs.ts index 7fcf8b4f..c1d8342c 100644 --- a/src/codecs.ts +++ b/src/codecs.ts @@ -5,8 +5,8 @@ import { InitialViewState, OnConsentExpiry, UserPrivacySignalEnum, -} from "@transcend-io/airgap.js-types"; -import { LanguageKey } from "@transcend-io/internationalization"; +} from '@transcend-io/airgap.js-types'; +import { LanguageKey } from '@transcend-io/internationalization'; import { ActionItemCode, ActionItemPriorityOverride, @@ -58,12 +58,12 @@ import { UnknownRequestPolicy, UnstructuredSubDataPointRecommendationStatus, UspapiOption, -} from "@transcend-io/privacy-types"; -import { applyEnum, valuesOf } from "@transcend-io/type-utils"; -import * as t from "io-ts"; -import { OpenAIRouteName, PathfinderPolicyName } from "./enums"; -import { buildAIIntegrationType } from "./lib/helpers/buildAIIntegrationType"; -import { buildEnabledRouteType } from "./lib/helpers/buildEnabledRouteType"; +} from '@transcend-io/privacy-types'; +import { applyEnum, valuesOf } from '@transcend-io/type-utils'; +import * as t from 'io-ts'; +import { OpenAIRouteName, PathfinderPolicyName } from './enums'; +import { buildAIIntegrationType } from './lib/helpers/buildAIIntegrationType'; +import { buildEnabledRouteType } from './lib/helpers/buildEnabledRouteType'; /** * Input to define email templates that can be used to communicate to end-users @@ -126,11 +126,11 @@ export const TeamInput = t.intersection([ }), t.partial({ /** SSO department for automated provisioning */ - "sso-department": t.string, + 'sso-department': t.string, /** SSO group name for automated provisioning */ - "sso-group": t.string, + 'sso-group': t.string, /** SSO title mapping for automated provisioning */ - "sso-title": t.string, + 'sso-title': t.string, /** List of user emails on the team */ users: t.array(t.string), /** List of scopes that the team should have */ @@ -162,7 +162,7 @@ export const EnricherInput = t.intersection([ * The names of the identifiers that can be resolved by this enricher. * i.e. email -> [userId, phone, advertisingId] */ - "output-identifiers": t.array(t.string), + 'output-identifiers': t.array(t.string), }), t.partial({ /** Internal description for why the enricher is needed */ @@ -176,7 +176,7 @@ export const EnricherInput = t.intersection([ * Whenever a privacy request contains this identifier, the webhook will * be called with the value of that identifier as input */ - "input-identifier": t.string, + 'input-identifier': t.string, /** * A regular expression that can be used to match on for cancelation */ @@ -199,16 +199,16 @@ export const EnricherInput = t.intersection([ phoneNumbers: t.array(t.string), /** The list of regions that should trigger the preflight check */ regionList: t.array( - valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }) + valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }), ), /** * Specify which data subjects the enricher should run for */ - "data-subjects": t.array(t.string), + 'data-subjects': t.array(t.string), /** Headers to include in the webhook */ headers: t.array(WebhookHeader), /** The privacy actions that the enricher should run against */ - "privacy-actions": t.array(valuesOf(RequestAction)), + 'privacy-actions': t.array(valuesOf(RequestAction)), }), ]); @@ -355,7 +355,7 @@ export const AgentInput = t.intersection([ /** Whether the agent has retrieval enabled */ retrievalEnabled: t.boolean, /** Large language model powering the agent */ - "large-language-model": t.type({ + 'large-language-model': t.type({ /** Name of the model */ name: t.string, /** Client of the model */ @@ -383,11 +383,11 @@ export const AgentInput = t.intersection([ /** * The names of the functions that the agent has access to */ - "agent-functions": t.array(t.string), + 'agent-functions': t.array(t.string), /** * The names of the files that the agent has access to for retrieval */ - "agent-files": t.array(t.string), + 'agent-files': t.array(t.string), }), ]); @@ -646,19 +646,19 @@ export const FieldInput = t.intersection([ * * @see https://github.com/transcend-io/privacy-types/blob/main/src/objects.ts */ - "guessed-categories": t.array(DataCategoryGuessInput), + 'guessed-categories': t.array(DataCategoryGuessInput), /** * When true, this subdatapoint should be revealed in a data access request. * When false, this field should be redacted */ - "access-request-visibility-enabled": t.boolean, + 'access-request-visibility-enabled': t.boolean, /** * When true, this subdatapoint should be redacted during an erasure request. * There normally is a choice of enabling hard deletion or redaction at the * datapoint level, but if redaction is enabled, this column can be used * to define which fields should be redacted. */ - "erasure-request-redaction-enabled": t.boolean, + 'erasure-request-redaction-enabled': t.boolean, /** Attributes tagged to subdatapoint */ attributes: t.array(AttributePreview), }), @@ -701,21 +701,21 @@ export const DatapointInput = t.intersection([ * * @see https://docs.transcend.io/docs/privacy-requests/connecting-data-silos/saas-tools#configuring-an-integration */ - "data-collection-tag": t.string, + 'data-collection-tag': t.string, /** * The SQL queries that should be run for that datapoint in a privacy request. * * @see https://github.com/transcend-io/privacy-types/blob/main/src/actions.ts */ - "privacy-action-queries": t.partial( - applyEnum(RequestActionObjectResolver, () => t.string) + 'privacy-action-queries': t.partial( + applyEnum(RequestActionObjectResolver, () => t.string), ), /** * The types of privacy actions that this datapoint can implement * * @see https://github.com/transcend-io/privacy-types/blob/main/src/actions.ts */ - "privacy-actions": t.array(valuesOf(RequestActionObjectResolver)), + 'privacy-actions': t.array(valuesOf(RequestActionObjectResolver)), /** * Provide field-level metadata for this datapoint. * This is often the column metadata @@ -742,30 +742,30 @@ export type DatapointInput = t.TypeOf; export const PromptAVendorEmailSettings = t.partial({ /** The email address of the user to notify when a promptAPerson integration */ - "notify-email-address": t.string, + 'notify-email-address': t.string, /** * The frequency with which we should be sending emails for this data silo, in milliseconds. */ - "send-frequency": t.number, + 'send-frequency': t.number, /** * The type of emails to send for this data silo, i.e. send an email for each DSR, across all open DSRs, * or per profile in a DSR. */ - "send-type": valuesOf(PromptAVendorEmailSendType), + 'send-type': valuesOf(PromptAVendorEmailSendType), /** * Indicates whether prompt-a-vendor emails should include a list of identifiers * in addition to a link to the bulk processing UI. */ - "include-identifiers-attachment": t.boolean, + 'include-identifiers-attachment': t.boolean, /** * Indicates what kind of link to generate as part of the emails sent out for this Prompt-a-Vendor silo. */ - "completion-link-type": valuesOf(PromptAVendorEmailCompletionLinkType), + 'completion-link-type': valuesOf(PromptAVendorEmailCompletionLinkType), /** * The frequency with which we should retry sending emails for this data silo, in milliseconds. * Needs to be a string because the number can be larger than the MAX_INT */ - "manual-work-retry-frequency": t.string, + 'manual-work-retry-frequency': t.string, }); /** Type override */ @@ -975,11 +975,11 @@ export const ActionInput = t.intersection([ regionDetectionMethod: valuesOf(RegionDetectionMethod), /** The list of regions to show in the form */ regionList: t.array( - valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }) + valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }), ), /** The list of regions NOT to show in the form */ regionBlockList: t.array( - valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }) + valuesOf({ ...IsoCountryCode, ...IsoCountrySubdivisionCode }), ), }), ]); @@ -1128,7 +1128,7 @@ export const ConsentManageExperienceInput = t.intersection([ t.partial({ countrySubDivision: valuesOf(IsoCountrySubdivisionCode), country: valuesOf(IsoCountryCode), - }) + }), ), /** How to handle consent expiry */ onConsentExpiry: valuesOf(OnConsentExpiry), @@ -1145,14 +1145,14 @@ export const ConsentManageExperienceInput = t.intersection([ t.type({ /** Slug of purpose */ trackingType: t.string, - }) + }), ), /** Purposes that are opted out by default in a particular experience */ optedOutPurposes: t.array( t.type({ /** Slug of purpose */ trackingType: t.string, - }) + }), ), /** * Browser languages that define this regional experience @@ -1344,13 +1344,13 @@ export const DataSiloInput = t.intersection([ }), t.partial({ /** For prompt a person or database integrations, the underlying integration name */ - "outer-type": t.string, + 'outer-type': t.string, /** A description for that data silo */ description: t.string, /** The webhook URL to notify for data privacy requests */ url: t.string, /** The title of the API key that will be used to respond to privacy requests */ - "api-key-title": t.string, + 'api-key-title': t.string, /** Custom headers to include in outbound webhook */ headers: t.array(WebhookHeader), /** @@ -1358,18 +1358,18 @@ export const DataSiloInput = t.intersection([ * This field can be omitted, and the default assumption will be that the system may potentially * contain PII for any potential data subject type. */ - "data-subjects": t.array(t.string), + 'data-subjects': t.array(t.string), /** * When this data silo implements a privacy request, these are the identifiers * that should be looked up within this system. */ - "identity-keys": t.array(t.string), + 'identity-keys': t.array(t.string), /** * When a data erasure request is being performed, this data silo should not be deleted from * until all of the following data silos were deleted first. This list can contain other internal * systems defined in this file, as well as any of the SaaS tools connected in your Transcend instance. */ - "deletion-dependencies": t.array(t.string), + 'deletion-dependencies': t.array(t.string), /** * The email addresses of the employees within your company that are the go-to individuals * for managing this data silo @@ -1396,7 +1396,7 @@ export const DataSiloInput = t.intersection([ /** * Configure email notification settings for privacy requests */ - "email-settings": PromptAVendorEmailSettings, + 'email-settings': PromptAVendorEmailSettings, /** Country of data silo hosting */ country: valuesOf(IsoCountryCode), /** Sub-division of data silo hosting */ @@ -1484,13 +1484,13 @@ export type ActionItemInput = t.TypeOf; export const AssessmentRuleInput = t.intersection([ t.type({ /** The reference id of the question whose answer is compared by this rule */ - "depends-on-question-reference-id": t.string, + 'depends-on-question-reference-id': t.string, /** The operator to use when comparing the question answer to the operands */ - "comparison-operator": valuesOf(ComparisonOperator), + 'comparison-operator': valuesOf(ComparisonOperator), }), t.partial({ /** The values to compare the question answer to */ - "comparison-operands": t.array(t.string), + 'comparison-operands': t.array(t.string), }), ]); @@ -1499,28 +1499,28 @@ export type AssessmentRuleInput = t.TypeOf; export interface AssessmentNestedRuleInput { /** The operator to use when comparing the nested rules */ - "logic-operator": LogicOperator; + 'logic-operator': LogicOperator; /** The rules to evaluate and be compared with to other using the LogicOperator */ rules?: AssessmentRuleInput[]; /** The nested rules to add one more level of nesting to the rules. They are also compared to each other. */ - "nested-rules"?: AssessmentNestedRuleInput[]; + 'nested-rules'?: AssessmentNestedRuleInput[]; } export const AssessmentNestedRuleInput: t.RecursiveType< t.Type -> = t.recursion("AssessmentNestedRuleInput", (self) => +> = t.recursion('AssessmentNestedRuleInput', (self) => t.intersection([ t.type({ /** The operator to use when comparing the nested rules */ - "logic-operator": valuesOf(LogicOperator), + 'logic-operator': valuesOf(LogicOperator), }), t.partial({ /** The rules to evaluate and be compared with to other using the LogicOperator */ rules: t.array(AssessmentRuleInput), /** The nested rules to add one more level of nesting to the rules. They are also compared to each other. */ - "nested-rules": t.array(self), + 'nested-rules': t.array(self), }), - ]) + ]), ); export const AssessmentDisplayLogicInput = t.intersection([ @@ -1532,7 +1532,7 @@ export const AssessmentDisplayLogicInput = t.intersection([ /** The rule to evaluate */ rule: AssessmentRuleInput, /** The nested rule to evaluate */ - "nested-rule": AssessmentNestedRuleInput, + 'nested-rule': AssessmentNestedRuleInput, }), ]); @@ -1543,11 +1543,11 @@ export type AssessmentDisplayLogicInput = t.TypeOf< export const RiskAssignmentInput = t.partial({ /** The risk level to assign to the question */ - "risk-level": t.string, + 'risk-level': t.string, /** The risk matrix column to assign to a question. */ - "risk-matrix-column": t.string, + 'risk-matrix-column': t.string, /** The risk matrix row to assign to a question. */ - "risk-matrix-row": t.string, + 'risk-matrix-row': t.string, }); /** Type override */ @@ -1556,17 +1556,17 @@ export type RiskAssignmentInput = t.TypeOf; export const RiskLogicInput = t.intersection([ t.type({ /** The values to compare */ - "comparison-operands": t.array(t.string), + 'comparison-operands': t.array(t.string), /** The operator */ - "comparison-operator": valuesOf(ComparisonOperator), + 'comparison-operator': valuesOf(ComparisonOperator), }), t.partial({ /** The risk level to assign to the question */ - "risk-level": t.string, + 'risk-level': t.string, /** The risk matrix column to assign to a question. */ - "risk-matrix-column": t.string, + 'risk-matrix-column': t.string, /** The risk matrix row to assign to a question. */ - "risk-matrix-row": t.string, + 'risk-matrix-row': t.string, }), ]); @@ -1592,41 +1592,41 @@ export const AssessmentSectionQuestionInput = t.intersection([ }), t.partial({ /** The sub-type of the assessment question */ - "sub-type": valuesOf(AssessmentQuestionSubType), + 'sub-type': valuesOf(AssessmentQuestionSubType), /** The question placeholder */ placeholder: t.string, /** The question description */ description: t.string, /** Whether an answer is required */ - "is-required": t.boolean, + 'is-required': t.boolean, /** Used to identify the question within a form or template so it can be referenced in conditional logic. */ - "reference-id": t.string, + 'reference-id': t.string, /** Display logic for the question */ - "display-logic": AssessmentDisplayLogicInput, + 'display-logic': AssessmentDisplayLogicInput, /** Risk logic for the question */ - "risk-logic": t.array(RiskLogicInput), + 'risk-logic': t.array(RiskLogicInput), /** Risk category titles for the question */ - "risk-categories": t.array(t.string), + 'risk-categories': t.array(t.string), /** Risk framework titles for the question */ - "risk-framework": t.string, + 'risk-framework': t.string, /** Answer options for the question */ - "answer-options": t.array(AssessmentAnswerOptionInput), + 'answer-options': t.array(AssessmentAnswerOptionInput), /** The selected answers to the assessments */ - "selected-answers": t.array(t.string), + 'selected-answers': t.array(t.string), /** Allowed MIME types for the question */ - "allowed-mime-types": t.array(t.string), + 'allowed-mime-types': t.array(t.string), /** Allow selecting other options */ - "allow-select-other": t.boolean, + 'allow-select-other': t.boolean, /** Sync model for the question */ - "sync-model": valuesOf(AssessmentSyncModel), + 'sync-model': valuesOf(AssessmentSyncModel), /** Sync column for the question */ - "sync-column": valuesOf(AssessmentSyncColumn), + 'sync-column': valuesOf(AssessmentSyncColumn), /** Attribute key / custom field name for the question */ - "attribute-key": t.string, + 'attribute-key': t.string, /** Require risk evaluation for the question */ - "require-risk-evaluation": t.boolean, + 'require-risk-evaluation': t.boolean, /** Require risk matrix evaluation for the question */ - "require-risk-matrix-evaluation": t.boolean, + 'require-risk-matrix-evaluation': t.boolean, }), ]); @@ -1646,11 +1646,11 @@ export const AssessmentSectionInput = t.intersection([ /** Email address of those assigned */ assignees: t.array(t.string), /** Email address of those externally assigned */ - "external-assignees": t.array(t.string), + 'external-assignees': t.array(t.string), /** Status of section */ status: t.string, /** Whether assessment is reviewed */ - "is-reviewed": t.boolean, + 'is-reviewed': t.boolean, }), ]); @@ -1661,7 +1661,7 @@ export const AssessmentRetentionScheduleInput = t.type({ /** The retention schedule type */ type: valuesOf(RetentionScheduleType), /** The duration of the retention schedule in days */ - "duration-days": t.number, + 'duration-days': t.number, /** The operation to perform on the retention schedule */ operand: valuesOf(RetentionScheduleOperation), }); @@ -1690,15 +1690,15 @@ export const AssessmentTemplateInput = t.intersection([ /** Whether the template is in a locked status */ locked: t.boolean, /** ID of parent template this was cloned from */ - "parent-id": t.string, + 'parent-id': t.string, /** Whether the template is archived */ archived: t.boolean, /** The date that the assessment was created */ - "created-at": t.string, + 'created-at': t.string, /** The names of the custom fields associated to this assessment template */ - "attribute-keys": t.array(t.string), + 'attribute-keys': t.array(t.string), /** The retention schedule configuration */ - "retention-schedule": AssessmentRetentionScheduleInput, + 'retention-schedule': AssessmentRetentionScheduleInput, /** The titles of the email templates used in the assessment template */ templates: t.array(t.string), }), @@ -1736,7 +1736,7 @@ export const AssessmentInput = t.intersection([ /** The emails of the transcend users assigned to the assessment */ assignees: t.array(t.string), /** The emails of the external emails assigned to the assessment */ - "external-assignees": t.array(t.string), + 'external-assignees': t.array(t.string), /** The emails of the assessment reviewers */ reviewers: t.array(t.string), /** Whether the assessment is in a locked status */ @@ -1748,25 +1748,25 @@ export const AssessmentInput = t.intersection([ /** * Whether the form title is an internal label only, and the group title should be used in communications with assignees */ - "title-is-internal": t.boolean, + 'title-is-internal': t.boolean, /** The date that the assessment is due */ - "due-date": t.string, + 'due-date': t.string, /** The date that the assessment was created */ - "created-at": t.string, + 'created-at': t.string, /** The date that the assessment was assigned at */ - "assigned-at": t.string, + 'assigned-at': t.string, /** The date that the assessment was submitted at */ - "submitted-at": t.string, + 'submitted-at': t.string, /** The date that the assessment was approved at */ - "approved-at": t.string, + 'approved-at': t.string, /** The date that the assessment was rejected at */ - "rejected-at": t.string, + 'rejected-at': t.string, /** The linked data inventory resources */ resources: t.array(AssessmentResourceInput), /** The linked data inventory synced rows */ rows: t.array(AssessmentResourceInput), /** The assessment retention schedule */ - "retention-schedule": AssessmentRetentionScheduleInput, + 'retention-schedule': AssessmentRetentionScheduleInput, /** The assessment custom fields */ attributes: t.array(AttributePreview), }), @@ -1798,9 +1798,9 @@ export const ConsentPreferenceTopic = t.intersection([ }), t.partial({ /** Default value */ - "default-configuration": t.string, + 'default-configuration': t.string, /** Whether the preference topic is shown in privacy center */ - "show-in-privacy-center": t.boolean, + 'show-in-privacy-center': t.boolean, /** The options when type is single or multi select */ options: t.array(ConsentPreferenceTopicOptionValue), }), @@ -1822,23 +1822,23 @@ export const ConsentPurpose = t.intersection([ /** Description of purpose */ description: t.string, /** Whether purpose is active */ - "is-active": t.boolean, + 'is-active': t.boolean, /** Whether purpose is configurable */ configurable: t.boolean, /** Display order of purpose for privacy center */ - "display-order": t.number, + 'display-order': t.number, /** Whether purpose is shown in privacy center */ - "show-in-privacy-center": t.boolean, + 'show-in-privacy-center': t.boolean, /** Whether purpose is show in consent manger */ - "show-in-consent-manager": t.boolean, + 'show-in-consent-manager': t.boolean, /** The preference topics configured for the purpose */ - "preference-topics": t.array(ConsentPreferenceTopic), + 'preference-topics': t.array(ConsentPreferenceTopic), /** Authentication level for purpose on privacy center */ - "auth-level": valuesOf(PreferenceStoreAuthLevel), + 'auth-level': valuesOf(PreferenceStoreAuthLevel), /** Opt out signals that should instantly opt out of this purpose */ - "opt-out-signals": t.array(valuesOf(UserPrivacySignalEnum)), + 'opt-out-signals': t.array(valuesOf(UserPrivacySignalEnum)), /** Default consent value */ - "default-consent": valuesOf(DefaultConsentOption), + 'default-consent': valuesOf(DefaultConsentOption), }), ]); @@ -1849,15 +1849,15 @@ export const TranscendInput = t.partial({ /** * Action items */ - "action-items": t.array(ActionItemInput), + 'action-items': t.array(ActionItemInput), /** * Action item collections */ - "action-item-collections": t.array(ActionItemCollectionInput), + 'action-item-collections': t.array(ActionItemCollectionInput), /** * API key definitions */ - "api-keys": t.array(ApiKeyInput), + 'api-keys': t.array(ApiKeyInput), /** Team definitions */ teams: t.array(TeamInput), /** @@ -1875,7 +1875,7 @@ export const TranscendInput = t.partial({ /** * Business entity definitions */ - "business-entities": t.array(BusinessEntityInput), + 'business-entities': t.array(BusinessEntityInput), /** * Vendor definitions */ @@ -1883,15 +1883,15 @@ export const TranscendInput = t.partial({ /** * Data categories definitions */ - "data-categories": t.array(DataCategoryInput), + 'data-categories': t.array(DataCategoryInput), /** * Vendor definitions */ - "processing-purposes": t.array(ProcessingPurposeInput), + 'processing-purposes': t.array(ProcessingPurposeInput), /** * Data subject definitions */ - "data-subjects": t.array(DataSubjectInput), + 'data-subjects': t.array(DataSubjectInput), /** * Action definitions */ @@ -1903,11 +1903,11 @@ export const TranscendInput = t.partial({ /** * Data silo definitions */ - "data-silos": t.array(DataSiloInput), + 'data-silos': t.array(DataSiloInput), /** * Data flow definitions */ - "data-flows": t.array(DataFlowInput), + 'data-flows': t.array(DataFlowInput), /** * Cookie definitions */ @@ -1915,7 +1915,7 @@ export const TranscendInput = t.partial({ /** * Consent manager definition */ - "consent-manager": ConsentManagerInput, + 'consent-manager': ConsentManagerInput, /** * Prompt definitions */ @@ -1923,11 +1923,11 @@ export const TranscendInput = t.partial({ /** * Prompt partial definitions */ - "prompt-partials": t.array(PromptPartialInput), + 'prompt-partials': t.array(PromptPartialInput), /** * Prompt group definitions */ - "prompt-groups": t.array(PromptGroupInput), + 'prompt-groups': t.array(PromptGroupInput), /** * Agent definitions */ @@ -1935,15 +1935,15 @@ export const TranscendInput = t.partial({ /** * Agent function definitions */ - "agent-functions": t.array(AgentFunctionInput), + 'agent-functions': t.array(AgentFunctionInput), /** * Agent file definitions */ - "agent-files": t.array(AgentFileInput), + 'agent-files': t.array(AgentFileInput), /** * The privacy center configuration */ - "privacy-center": PrivacyCenterInput, + 'privacy-center': PrivacyCenterInput, /** * The policies configuration */ @@ -1957,7 +1957,7 @@ export const TranscendInput = t.partial({ /** * The full list of assessment templates */ - "assessment-templates": t.array(AssessmentTemplateInput), + 'assessment-templates': t.array(AssessmentTemplateInput), /** * The full list of assessment results */ @@ -1992,7 +1992,7 @@ export type StoredApiKey = t.TypeOf; export const DataFlowCsvInput = t.intersection([ t.type({ /** The value of the data flow (host or regex) */ - "Connections Made To": t.string, + 'Connections Made To': t.string, /** The type of the data flow */ Type: valuesOf(DataFlowScope), /** The CSV of purposes mapped to that data flow */ @@ -2058,7 +2058,7 @@ export const ConsentManagerServiceMetadata = t.type({ name: t.string, /** Allowed purposes */ trackingPurposes: t.array(t.string), - }) + }), ), /** Data Flows */ dataFlows: t.array( @@ -2069,7 +2069,7 @@ export const ConsentManagerServiceMetadata = t.type({ type: valuesOf(DataFlowScope), /** Allowed purposes */ trackingPurposes: t.array(t.string), - }) + }), ), }); diff --git a/src/commands/admin/generate-api-keys/impl.ts b/src/commands/admin/generate-api-keys/impl.ts index f9c0d278..0288e38e 100644 --- a/src/commands/admin/generate-api-keys/impl.ts +++ b/src/commands/admin/generate-api-keys/impl.ts @@ -1,17 +1,17 @@ -import { writeFileSync } from "node:fs"; -import { ScopeName, TRANSCEND_SCOPES } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { keyBy } from "lodash-es"; -import type { LocalContext } from "../../../context"; -import { generateCrossAccountApiKeys } from "../../../lib/api-keys"; -import { logger } from "../../../logger"; +import { writeFileSync } from 'node:fs'; +import { ScopeName, TRANSCEND_SCOPES } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { keyBy } from 'lodash-es'; +import type { LocalContext } from '../../../context'; +import { generateCrossAccountApiKeys } from '../../../lib/api-keys'; +import { logger } from '../../../logger'; const SCOPES_BY_TITLE = keyBy( Object.entries(TRANSCEND_SCOPES).map(([name, value]) => ({ ...value, name, })), - "title" + 'title', ); const SCOPE_TITLES = Object.keys(SCOPES_BY_TITLE); @@ -41,25 +41,25 @@ export async function generateApiKeys( createNewApiKey, parentOrganizationId, transcendUrl, - }: GenerateApiKeysCommandFlags + }: GenerateApiKeysCommandFlags, ): Promise { // Validate scopes const splitScopes = scopes.map((x) => x.trim()); const invalidScopes = splitScopes.filter( - (scopeTitle) => !SCOPES_BY_TITLE[scopeTitle] + (scopeTitle) => !SCOPES_BY_TITLE[scopeTitle], ); if (invalidScopes.length > 0) { logger.error( colors.red( - `Failed to parse scopes:"${invalidScopes.join(",")}".\n` + - `Expected one of: \n${SCOPE_TITLES.join("\n")}` - ) + `Failed to parse scopes:"${invalidScopes.join(',')}".\n` + + `Expected one of: \n${SCOPE_TITLES.join('\n')}`, + ), ); process.exit(1); } const scopeNames = splitScopes.map( - (scopeTitle) => SCOPES_BY_TITLE[scopeTitle].name as ScopeName + (scopeTitle) => SCOPES_BY_TITLE[scopeTitle].name as ScopeName, ); // Upload privacy requests diff --git a/src/commands/consent/build-xdi-sync-endpoint/impl.ts b/src/commands/consent/build-xdi-sync-endpoint/impl.ts index 2b98351a..b42f26bb 100644 --- a/src/commands/consent/build-xdi-sync-endpoint/impl.ts +++ b/src/commands/consent/build-xdi-sync-endpoint/impl.ts @@ -1,9 +1,9 @@ -import { writeFileSync } from "node:fs"; -import colors from "colors"; -import type { LocalContext } from "../../../context"; -import { validateTranscendAuth } from "../../../lib/api-keys"; -import { buildXdiSyncEndpoint as buildXdiSyncEndpointHelper } from "../../../lib/consent-manager"; -import { logger } from "../../../logger"; +import { writeFileSync } from 'node:fs'; +import colors from 'colors'; +import type { LocalContext } from '../../../context'; +import { validateTranscendAuth } from '../../../lib/api-keys'; +import { buildXdiSyncEndpoint as buildXdiSyncEndpointHelper } from '../../../lib/consent-manager'; +import { logger } from '../../../logger'; interface BuildXdiSyncEndpointCommandFlags { auth: string; @@ -25,7 +25,7 @@ export async function buildXdiSyncEndpoint( domainBlockList, xdiAllowedCommands, transcendUrl, - }: BuildXdiSyncEndpointCommandFlags + }: BuildXdiSyncEndpointCommandFlags, ): Promise { // Parse authentication as API key or path to list of API keys const apiKeyOrList = await validateTranscendAuth(auth); @@ -45,9 +45,9 @@ export async function buildXdiSyncEndpoint( `Successfully constructed sync endpoint for sync groups: ${JSON.stringify( syncGroups, null, - 2 - )}` - ) + 2, + )}`, + ), ); // Write to disk diff --git a/src/commands/consent/pull-consent-metrics/impl.ts b/src/commands/consent/pull-consent-metrics/impl.ts index a7078c79..0b1c58b2 100644 --- a/src/commands/consent/pull-consent-metrics/impl.ts +++ b/src/commands/consent/pull-consent-metrics/impl.ts @@ -1,17 +1,17 @@ -import fs, { existsSync, mkdirSync } from "node:fs"; -import { join } from "node:path"; -import colors from "colors"; -import { ADMIN_DASH_INTEGRATIONS } from "../../../constants"; -import type { LocalContext } from "../../../context"; -import { validateTranscendAuth } from "../../../lib/api-keys"; -import { mapSeries } from "../../../lib/bluebird-replace"; -import { pullConsentManagerMetrics } from "../../../lib/consent-manager"; -import { writeCsv } from "../../../lib/cron"; +import fs, { existsSync, mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import colors from 'colors'; +import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { validateTranscendAuth } from '../../../lib/api-keys'; +import { mapSeries } from '../../../lib/bluebird-replace'; +import { pullConsentManagerMetrics } from '../../../lib/consent-manager'; +import { writeCsv } from '../../../lib/cron'; import { buildTranscendGraphQLClient, ConsentManagerMetricBin, -} from "../../../lib/graphql"; -import { logger } from "../../../logger"; +} from '../../../lib/graphql'; +import { logger } from '../../../logger'; interface PullConsentMetricsCommandFlags { auth: string; @@ -31,7 +31,7 @@ export async function pullConsentMetrics( folder, bin, transcendUrl, - }: PullConsentMetricsCommandFlags + }: PullConsentMetricsCommandFlags, ): Promise { // Parse authentication as API key or path to list of API keys const apiKeyOrList = await validateTranscendAuth(auth); @@ -40,8 +40,8 @@ export async function pullConsentMetrics( if (fs.existsSync(folder) && !fs.lstatSync(folder).isDirectory()) { logger.error( colors.red( - 'The provided argument "folder" was passed a file. expected: folder="./consent-metrics/"' - ) + 'The provided argument "folder" was passed a file. expected: folder="./consent-metrics/"', + ), ); process.exit(1); } @@ -53,9 +53,9 @@ export async function pullConsentMetrics( colors.red( `Failed to parse argument "bin" with value "${bin}"\n` + `Expected one of: \n${Object.values(ConsentManagerMetricBin).join( - "\n" - )}` - ) + '\n', + )}`, + ), ); process.exit(1); } @@ -66,16 +66,16 @@ export async function pullConsentMetrics( if (Number.isNaN(startDate.getTime())) { logger.error( colors.red( - `Start date provided is invalid date. Got --start="${start}" expected --start="01/01/2023"` - ) + `Start date provided is invalid date. Got --start="${start}" expected --start="01/01/2023"`, + ), ); process.exit(1); } if (Number.isNaN(endDate.getTime())) { logger.error( colors.red( - `End date provided is invalid date. Got --end="${end}" expected --end="01/01/2023"` - ) + `End date provided is invalid date. Got --end="${end}" expected --end="01/01/2023"`, + ), ); process.exit(1); } @@ -83,8 +83,8 @@ export async function pullConsentMetrics( logger.error( colors.red( `Got a start date "${startDate.toISOString()}" that was larger than the end date "${endDate.toISOString()}". ` + - "Start date must be before end date." - ) + 'Start date must be before end date.', + ), ); process.exit(1); } @@ -96,12 +96,12 @@ export async function pullConsentMetrics( logger.info( colors.magenta( - `Pulling consent metrics from start=${startDate.toString()} to end=${endDate.toISOString()} with bin size "${bin}"` - ) + `Pulling consent metrics from start=${startDate.toString()} to end=${endDate.toISOString()} with bin size "${bin}"`, + ), ); // Sync to Disk - if (typeof apiKeyOrList === "string") { + if (typeof apiKeyOrList === 'string') { try { // Create a GraphQL client const client = buildTranscendGraphQLClient(transcendUrl, apiKeyOrList); @@ -118,20 +118,20 @@ export async function pullConsentMetrics( for (const { points, name } of metrics) { const file = join(folder, `${metricName}_${name}.csv`); logger.info( - colors.magenta(`Writing configuration to file "${file}"...`) + colors.magenta(`Writing configuration to file "${file}"...`), ); writeCsv( file, points.map(({ key, value }) => ({ timestamp: key, value, - })) + })), ); } } } catch (error) { logger.error( - colors.red(`An error occurred syncing the schema: ${error.message}`) + colors.red(`An error occurred syncing the schema: ${error.message}`), ); process.exit(1); } @@ -139,8 +139,8 @@ export async function pullConsentMetrics( // Indicate success logger.info( colors.green( - `Successfully synced consent metrics to disk in folder "${folder}"! View at ${ADMIN_DASH_INTEGRATIONS}` - ) + `Successfully synced consent metrics to disk in folder "${folder}"! View at ${ADMIN_DASH_INTEGRATIONS}`, + ), ); } else { const encounteredErrors: string[] = []; @@ -150,8 +150,8 @@ export async function pullConsentMetrics( }] `; logger.info( colors.magenta( - `~~~\n\n${prefix}Attempting to pull consent metrics...\n\n~~~` - ) + `~~~\n\n${prefix}Attempting to pull consent metrics...\n\n~~~`, + ), ); // Create a GraphQL client @@ -175,20 +175,20 @@ export async function pullConsentMetrics( for (const { points, name } of metrics) { const file = join(subFolder, `${metricName}_${name}.csv`); logger.info( - colors.magenta(`Writing configuration to file "${file}"...`) + colors.magenta(`Writing configuration to file "${file}"...`), ); writeCsv( file, points.map(({ key, value }) => ({ timestamp: key, value, - })) + })), ); } } logger.info( - colors.green(`${prefix}Successfully pulled configuration!`) + colors.green(`${prefix}Successfully pulled configuration!`), ); } catch { logger.error(colors.red(`${prefix}Failed to sync configuration.`)); @@ -200,9 +200,9 @@ export async function pullConsentMetrics( logger.info( colors.red( `Sync encountered errors for "${encounteredErrors.join( - "," - )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}` - ) + ',', + )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}`, + ), ); process.exit(1); diff --git a/src/commands/consent/upload-preferences/impl.ts b/src/commands/consent/upload-preferences/impl.ts index 5d2f61c9..d8c64d03 100644 --- a/src/commands/consent/upload-preferences/impl.ts +++ b/src/commands/consent/upload-preferences/impl.ts @@ -1,11 +1,11 @@ -import { readdirSync } from "node:fs"; -import { basename, join } from "node:path"; -import colors from "colors"; -import type { LocalContext } from "../../../context"; -import { map } from "../../../lib/bluebird-replace"; -import { uploadPreferenceManagementPreferencesInteractive } from "../../../lib/preference-management"; -import { splitCsvToList } from "../../../lib/requests"; -import { logger } from "../../../logger"; +import { readdirSync } from 'node:fs'; +import { basename, join } from 'node:path'; +import colors from 'colors'; +import type { LocalContext } from '../../../context'; +import { map } from '../../../lib/bluebird-replace'; +import { uploadPreferenceManagementPreferencesInteractive } from '../../../lib/preference-management'; +import { splitCsvToList } from '../../../lib/requests'; +import { logger } from '../../../logger'; interface UploadPreferencesCommandFlags { auth: string; @@ -33,7 +33,7 @@ export async function uploadPreferences( partition, sombraAuth, consentUrl, - file = "", + file = '', directory, dryRun, skipExistingRecordCheck, @@ -44,13 +44,13 @@ export async function uploadPreferences( isSilent, attributes, concurrency, - }: UploadPreferencesCommandFlags + }: UploadPreferencesCommandFlags, ): Promise { if (!!directory && !!file) { logger.error( colors.red( - "Cannot provide both a directory and a file. Please provide only one." - ) + 'Cannot provide both a directory and a file. Please provide only one.', + ), ); process.exit(1); } @@ -58,8 +58,8 @@ export async function uploadPreferences( if (!file && !directory) { logger.error( colors.red( - "A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences" - ) + 'A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences', + ), ); process.exit(1); } @@ -69,11 +69,11 @@ export async function uploadPreferences( if (directory) { try { const filesInDirectory = readdirSync(directory); - const csvFiles = filesInDirectory.filter((file) => file.endsWith(".csv")); + const csvFiles = filesInDirectory.filter((file) => file.endsWith('.csv')); if (csvFiles.length === 0) { logger.error( - colors.red(`No CSV files found in directory: ${directory}`) + colors.red(`No CSV files found in directory: ${directory}`), ); process.exit(1); } @@ -88,8 +88,8 @@ export async function uploadPreferences( } else { try { // Verify file exists and is a CSV - if (!file.endsWith(".csv")) { - logger.error(colors.red("File must be a CSV file")); + if (!file.endsWith('.csv')) { + logger.error(colors.red('File must be a CSV file')); process.exit(1); } files.push(file); @@ -102,23 +102,23 @@ export async function uploadPreferences( logger.info( colors.green( - `Processing ${files.length} consent preferences files for partition: ${partition}` - ) + `Processing ${files.length} consent preferences files for partition: ${partition}`, + ), ); - logger.debug(`Files to process: ${files.join(", ")}`); + logger.debug(`Files to process: ${files.join(', ')}`); if (skipExistingRecordCheck) { logger.info( colors.bgYellow( - `Skipping existing record check: ${skipExistingRecordCheck}` - ) + `Skipping existing record check: ${skipExistingRecordCheck}`, + ), ); } await map( files, async (filePath) => { - const fileName = basename(filePath).replace(".csv", ""); + const fileName = basename(filePath).replace('.csv', ''); await uploadPreferenceManagementPreferencesInteractive({ receiptFilepath: join(receiptFileDir, `${fileName}-receipts.json`), auth, @@ -135,6 +135,6 @@ export async function uploadPreferences( forceTriggerWorkflows, }); }, - { concurrency } + { concurrency }, ); } diff --git a/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts b/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts index ad41b2e9..28ab8b64 100644 --- a/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts +++ b/src/commands/inventory/consent-manager-service-json-to-yml/impl.ts @@ -1,19 +1,19 @@ -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readFileSync } from 'node:fs'; import { ConsentTrackerStatus, DataFlowScope, -} from "@transcend-io/privacy-types"; -import { decodeCodec } from "@transcend-io/type-utils"; -import colors from "colors"; -import * as t from "io-ts"; +} from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import colors from 'colors'; +import * as t from 'io-ts'; import { ConsentManagerServiceMetadata, CookieInput, DataFlowInput, -} from "../../../codecs"; -import type { LocalContext } from "../../../context"; -import { writeTranscendYaml } from "../../../lib/readTranscendYaml"; -import { logger } from "../../../logger"; +} from '../../../codecs'; +import type { LocalContext } from '../../../context'; +import { writeTranscendYaml } from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; interface ConsentManagerServiceJsonToYmlCommandFlags { file: string; @@ -22,7 +22,7 @@ interface ConsentManagerServiceJsonToYmlCommandFlags { export function consentManagerServiceJsonToYml( this: LocalContext, - { file, output }: ConsentManagerServiceJsonToYmlCommandFlags + { file, output }: ConsentManagerServiceJsonToYmlCommandFlags, ): void { // Ensure files exist if (!existsSync(file)) { @@ -33,7 +33,7 @@ export function consentManagerServiceJsonToYml( // Read in each consent manager configuration const services = decodeCodec( t.array(ConsentManagerServiceMetadata), - readFileSync(file, "utf-8") + readFileSync(file, 'utf-8'), ); // Create data flows and cookie configurations @@ -41,7 +41,7 @@ export function consentManagerServiceJsonToYml( const cookies: CookieInput[] = []; for (const service of services) { for (const dataFlow of service.dataFlows.filter( - ({ type }) => type !== DataFlowScope.CSP + ({ type }) => type !== DataFlowScope.CSP, )) { dataFlows.push({ value: dataFlow.value, @@ -62,13 +62,13 @@ export function consentManagerServiceJsonToYml( // write to disk writeTranscendYaml(output, { - "data-flows": dataFlows, + 'data-flows': dataFlows, cookies, }); logger.info( colors.green( - `Successfully wrote ${dataFlows.length} data flows and ${cookies.length} cookies to file "${output}"` - ) + `Successfully wrote ${dataFlows.length} data flows and ${cookies.length} cookies to file "${output}"`, + ), ); } diff --git a/src/commands/inventory/consent-managers-to-business-entities/impl.ts b/src/commands/inventory/consent-managers-to-business-entities/impl.ts index a504d375..e379557a 100644 --- a/src/commands/inventory/consent-managers-to-business-entities/impl.ts +++ b/src/commands/inventory/consent-managers-to-business-entities/impl.ts @@ -1,14 +1,14 @@ -import { existsSync, lstatSync } from "node:fs"; -import { join } from "node:path"; -import colors from "colors"; -import type { LocalContext } from "../../../context"; -import { listFiles } from "../../../lib/api-keys"; -import { consentManagersToBusinessEntities as consentManagersToBusinessEntitiesHelper } from "../../../lib/consent-manager"; +import { existsSync, lstatSync } from 'node:fs'; +import { join } from 'node:path'; +import colors from 'colors'; +import type { LocalContext } from '../../../context'; +import { listFiles } from '../../../lib/api-keys'; +import { consentManagersToBusinessEntities as consentManagersToBusinessEntitiesHelper } from '../../../lib/consent-manager'; import { readTranscendYaml, writeTranscendYaml, -} from "../../../lib/readTranscendYaml"; -import { logger } from "../../../logger"; +} from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; interface ConsentManagersToBusinessEntitiesCommandFlags { consentManagerYmlFolder: string; @@ -20,7 +20,7 @@ export function consentManagersToBusinessEntities( { consentManagerYmlFolder, output, - }: ConsentManagersToBusinessEntitiesCommandFlags + }: ConsentManagersToBusinessEntitiesCommandFlags, ): void { // Ensure folder is passed if ( @@ -28,15 +28,15 @@ export function consentManagersToBusinessEntities( !lstatSync(consentManagerYmlFolder).isDirectory() ) { logger.error( - colors.red(`Folder does not exist: "${consentManagerYmlFolder}"`) + colors.red(`Folder does not exist: "${consentManagerYmlFolder}"`), ); process.exit(1); } // Read in each consent manager configuration const inputs = listFiles(consentManagerYmlFolder).map((directory) => { - const { "consent-manager": consentManager } = readTranscendYaml( - join(consentManagerYmlFolder, directory) + const { 'consent-manager': consentManager } = readTranscendYaml( + join(consentManagerYmlFolder, directory), ); return { name: directory, input: consentManager }; }); @@ -46,12 +46,12 @@ export function consentManagersToBusinessEntities( // write to disk writeTranscendYaml(output, { - "business-entities": businessEntities, + 'business-entities': businessEntities, }); logger.info( colors.green( - `Successfully wrote ${businessEntities.length} business entities to file "${output}"` - ) + `Successfully wrote ${businessEntities.length} business entities to file "${output}"`, + ), ); } diff --git a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts index 0002d8f0..9b29f0ba 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows-cross-instance/impl.ts @@ -1,20 +1,20 @@ -import { existsSync, lstatSync } from "node:fs"; -import { join } from "node:path"; -import colors from "colors"; -import { difference } from "lodash-es"; -import { DataFlowInput } from "../../../codecs"; -import type { LocalContext } from "../../../context"; -import { listFiles } from "../../../lib/api-keys"; -import { dataFlowsToDataSilos } from "../../../lib/consent-manager/dataFlowsToDataSilos"; +import { existsSync, lstatSync } from 'node:fs'; +import { join } from 'node:path'; +import colors from 'colors'; +import { difference } from 'lodash-es'; +import { DataFlowInput } from '../../../codecs'; +import type { LocalContext } from '../../../context'; +import { listFiles } from '../../../lib/api-keys'; +import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; import { buildTranscendGraphQLClient, fetchAndIndexCatalogs, -} from "../../../lib/graphql"; +} from '../../../lib/graphql'; import { readTranscendYaml, writeTranscendYaml, -} from "../../../lib/readTranscendYaml"; -import { logger } from "../../../logger"; +} from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; interface DeriveDataSilosFromDataFlowsCrossInstanceCommandFlags { auth: string; @@ -32,14 +32,14 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( output, ignoreYmls = [], transcendUrl, - }: DeriveDataSilosFromDataFlowsCrossInstanceCommandFlags + }: DeriveDataSilosFromDataFlowsCrossInstanceCommandFlags, ): Promise { // Ensure folder is passed to dataFlowsYmlFolder if (!dataFlowsYmlFolder) { logger.error( colors.red( - "Missing required arg: --dataFlowsYmlFolder=./working/data-flows/" - ) + 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', + ), ); process.exit(1); } @@ -54,13 +54,13 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( } // Ignore the data flows in these yml files - const instancesToIgnore = ignoreYmls.map((x) => x.split(".")[0]); + const instancesToIgnore = ignoreYmls.map((x) => x.split('.')[0]); // Map over each data flow yml file and convert to data silo configurations const dataSiloInputs = listFiles(dataFlowsYmlFolder).map((directory) => { // read in the data flows for a specific instance - const { "data-flows": dataFlows = [] } = readTranscendYaml( - join(dataFlowsYmlFolder, directory) + const { 'data-flows': dataFlows = [] } = readTranscendYaml( + join(dataFlowsYmlFolder, directory), ); // map the data flows to data silos @@ -69,13 +69,13 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( { serviceToSupportedIntegration, serviceToTitle, - } + }, ); return { adTechDataSilos, siteTechDataSilos, - organizationName: directory.split(".")[0], + organizationName: directory.split('.')[0], }; }); @@ -88,7 +88,7 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( } of dataSiloInputs) { const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos]; for (const dataSilo of allDataSilos) { - const service = dataSilo["outer-type"] || dataSilo.integrationName; + const service = dataSilo['outer-type'] || dataSilo.integrationName; // create mapping to instance if (!serviceToInstance[service]) { serviceToInstance[service] = []; @@ -103,9 +103,9 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( ...new Set( dataSiloInputs.flatMap(({ adTechDataSilos }) => adTechDataSilos.map( - (silo) => silo["outer-type"] || silo.integrationName - ) - ) + (silo) => silo['outer-type'] || silo.integrationName, + ), + ), ), ]; @@ -115,12 +115,12 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( ...new Set( dataSiloInputs.flatMap(({ siteTechDataSilos }) => siteTechDataSilos.map( - (silo) => silo["outer-type"] || silo.integrationName - ) - ) + (silo) => silo['outer-type'] || silo.integrationName, + ), + ), ), ], - adTechIntegrations + adTechIntegrations, ); // Mapping from service name to list of @@ -128,9 +128,9 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( for (const { adTechDataSilos, siteTechDataSilos } of dataSiloInputs) { const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos]; for (const dataSilo of allDataSilos) { - const service = dataSilo["outer-type"] || dataSilo.integrationName; + const service = dataSilo['outer-type'] || dataSilo.integrationName; const foundOnDomain = dataSilo.attributes?.find( - (attribute) => attribute.key === "Found On Domain" + (attribute) => attribute.key === 'Found On Domain', ); // create mapping to instance if (!serviceToFoundOnDomain[service]) { @@ -154,25 +154,25 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( title: serviceToTitle[service], ...(serviceToSupportedIntegration[service] ? { integrationName: service } - : { integrationName: "promptAPerson", "outer-type": service }), + : { integrationName: 'promptAPerson', 'outer-type': service }), attributes: [ { - key: "Tech Type", - values: ["Ad Tech"], + key: 'Tech Type', + values: ['Ad Tech'], }, { - key: "Business Units", + key: 'Business Units', values: difference( serviceToInstance[service] || [], - instancesToIgnore + instancesToIgnore, ), }, { - key: "Found On Domain", + key: 'Found On Domain', values: serviceToFoundOnDomain[service] || [], }, ], - }) + }), ); // Log output @@ -182,6 +182,6 @@ export async function deriveDataSilosFromDataFlowsCrossInstance( // Write to yaml writeTranscendYaml(output, { - "data-silos": dataSilos, + 'data-silos': dataSilos, }); } diff --git a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts index 5c6e642e..27bb7b51 100644 --- a/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts +++ b/src/commands/inventory/derive-data-silos-from-data-flows/impl.ts @@ -1,19 +1,19 @@ -import { existsSync, lstatSync } from "node:fs"; -import { join } from "node:path"; -import colors from "colors"; -import { DataFlowInput } from "../../../codecs"; -import type { LocalContext } from "../../../context"; -import { listFiles } from "../../../lib/api-keys"; -import { dataFlowsToDataSilos } from "../../../lib/consent-manager/dataFlowsToDataSilos"; +import { existsSync, lstatSync } from 'node:fs'; +import { join } from 'node:path'; +import colors from 'colors'; +import { DataFlowInput } from '../../../codecs'; +import type { LocalContext } from '../../../context'; +import { listFiles } from '../../../lib/api-keys'; +import { dataFlowsToDataSilos } from '../../../lib/consent-manager/dataFlowsToDataSilos'; import { buildTranscendGraphQLClient, fetchAndIndexCatalogs, -} from "../../../lib/graphql"; +} from '../../../lib/graphql'; import { readTranscendYaml, writeTranscendYaml, -} from "../../../lib/readTranscendYaml"; -import { logger } from "../../../logger"; +} from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; interface DeriveDataSilosFromDataFlowsCommandFlags { auth: string; @@ -31,14 +31,14 @@ export async function deriveDataSilosFromDataFlows( dataSilosYmlFolder, ignoreYmls = [], transcendUrl, - }: DeriveDataSilosFromDataFlowsCommandFlags + }: DeriveDataSilosFromDataFlowsCommandFlags, ): Promise { // Ensure folder is passed to dataFlowsYmlFolder if (!dataFlowsYmlFolder) { logger.error( colors.red( - "Missing required arg: --dataFlowsYmlFolder=./working/data-flows/" - ) + 'Missing required arg: --dataFlowsYmlFolder=./working/data-flows/', + ), ); process.exit(1); } @@ -56,8 +56,8 @@ export async function deriveDataSilosFromDataFlows( if (!dataSilosYmlFolder) { logger.error( colors.red( - "Missing required arg: --dataSilosYmlFolder=./working/data-silos/" - ) + 'Missing required arg: --dataSilosYmlFolder=./working/data-silos/', + ), ); process.exit(1); } @@ -79,8 +79,8 @@ export async function deriveDataSilosFromDataFlows( // List of each data flow yml file for (const directory of listFiles(dataFlowsYmlFolder)) { // read in the data flows for a specific instance - const { "data-flows": dataFlows = [] } = readTranscendYaml( - join(dataFlowsYmlFolder, directory) + const { 'data-flows': dataFlows = [] } = readTranscendYaml( + join(dataFlowsYmlFolder, directory), ); // map the data flows to data silos @@ -89,7 +89,7 @@ export async function deriveDataSilosFromDataFlows( { serviceToSupportedIntegration, serviceToTitle, - } + }, ); // combine and write to yml file @@ -98,7 +98,7 @@ export async function deriveDataSilosFromDataFlows( logger.log(`Ad Tech Services: ${adTechDataSilos.length}`); logger.log(`Site Tech Services: ${siteTechDataSilos.length}`); writeTranscendYaml(join(dataSilosYmlFolder, directory), { - "data-silos": ignoreYmls.includes(directory) ? [] : dataSilos, + 'data-silos': ignoreYmls.includes(directory) ? [] : dataSilos, }); } } diff --git a/src/commands/inventory/pull-datapoints/impl.ts b/src/commands/inventory/pull-datapoints/impl.ts index fa968a75..a76b656a 100644 --- a/src/commands/inventory/pull-datapoints/impl.ts +++ b/src/commands/inventory/pull-datapoints/impl.ts @@ -1,12 +1,12 @@ -import { DataCategoryType } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { groupBy, uniq } from "lodash-es"; -import { ADMIN_DASH_DATAPOINTS } from "../../../constants"; -import type { LocalContext } from "../../../context"; -import { writeCsv } from "../../../lib/cron"; -import { pullAllDatapoints } from "../../../lib/data-inventory"; -import { buildTranscendGraphQLClient } from "../../../lib/graphql"; -import { logger } from "../../../logger"; +import { DataCategoryType } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { groupBy, uniq } from 'lodash-es'; +import { ADMIN_DASH_DATAPOINTS } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { writeCsv } from '../../../lib/cron'; +import { pullAllDatapoints } from '../../../lib/data-inventory'; +import { buildTranscendGraphQLClient } from '../../../lib/graphql'; +import { logger } from '../../../logger'; interface PullDatapointsCommandFlags { auth: string; @@ -30,7 +30,7 @@ export async function pullDatapoints( includeGuessedCategories, parentCategories, subCategories = [], - }: PullDatapointsCommandFlags + }: PullDatapointsCommandFlags, ): Promise { try { // Create a GraphQL client @@ -48,28 +48,28 @@ export async function pullDatapoints( let headers: string[] = []; const inputs = dataPoints.map((point) => { const result = { - "Property ID": point.id, - "Data Silo": point.dataSilo.title, + 'Property ID': point.id, + 'Data Silo': point.dataSilo.title, Object: point.dataPoint.name, - "Object Path": point.dataPoint.path.join("."), + 'Object Path': point.dataPoint.path.join('.'), Property: point.name, - "Property Description": point.description, - "Data Categories": point.categories + 'Property Description': point.description, + 'Data Categories': point.categories .map((category) => `${category.category}:${category.name}`) - .join(", "), - "Guessed Category": point.pendingCategoryGuesses?.[0] + .join(', '), + 'Guessed Category': point.pendingCategoryGuesses?.[0] ? `${point.pendingCategoryGuesses[0].category.category}:${point.pendingCategoryGuesses[0].category.name}` - : "", - "Processing Purposes": point.purposes + : '', + 'Processing Purposes': point.purposes .map((purpose) => `${purpose.purpose}:${purpose.name}`) - .join(", "), + .join(', '), ...Object.entries( groupBy( point.attributeValues || [], - ({ attributeKey }) => attributeKey.name - ) + ({ attributeKey }) => attributeKey.name, + ), ).reduce>((accumulator, [key, values]) => { - accumulator[key] = values.map((value) => value.name).join(","); + accumulator[key] = values.map((value) => value.name).join(','); return accumulator; }, {}), }; @@ -79,7 +79,7 @@ export async function pullDatapoints( writeCsv(file, inputs, headers); } catch (error) { logger.error( - colors.red(`An error occurred syncing the datapoints: ${error.message}`) + colors.red(`An error occurred syncing the datapoints: ${error.message}`), ); process.exit(1); } @@ -87,7 +87,7 @@ export async function pullDatapoints( // Indicate success logger.info( colors.green( - `Successfully synced datapoints to disk at ${file}! View at ${ADMIN_DASH_DATAPOINTS}` - ) + `Successfully synced datapoints to disk at ${file}! View at ${ADMIN_DASH_DATAPOINTS}`, + ), ); } diff --git a/src/commands/inventory/pull-unstructured-discovery-files/impl.ts b/src/commands/inventory/pull-unstructured-discovery-files/impl.ts index 001ed4b0..9c6026f3 100644 --- a/src/commands/inventory/pull-unstructured-discovery-files/impl.ts +++ b/src/commands/inventory/pull-unstructured-discovery-files/impl.ts @@ -1,11 +1,11 @@ -import type { UnstructuredSubDataPointRecommendationStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { uniq } from "lodash-es"; -import type { LocalContext } from "../../../context"; -import { writeCsv } from "../../../lib/cron"; -import { pullUnstructuredSubDataPointRecommendations } from "../../../lib/data-inventory"; -import { buildTranscendGraphQLClient } from "../../../lib/graphql"; -import { logger } from "../../../logger"; +import type { UnstructuredSubDataPointRecommendationStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { uniq } from 'lodash-es'; +import type { LocalContext } from '../../../context'; +import { writeCsv } from '../../../lib/cron'; +import { pullUnstructuredSubDataPointRecommendations } from '../../../lib/data-inventory'; +import { buildTranscendGraphQLClient } from '../../../lib/graphql'; +import { logger } from '../../../logger'; interface PullUnstructuredDiscoveryFilesCommandFlags { auth: string; @@ -27,7 +27,7 @@ export async function pullUnstructuredDiscoveryFiles( subCategories, status, includeEncryptedSnippets, - }: PullUnstructuredDiscoveryFilesCommandFlags + }: PullUnstructuredDiscoveryFilesCommandFlags, ): Promise { try { // Create a GraphQL client @@ -42,24 +42,24 @@ export async function pullUnstructuredDiscoveryFiles( logger.info( colors.magenta( - `Writing unstructured discovery files to file "${file}"...` - ) + `Writing unstructured discovery files to file "${file}"...`, + ), ); let headers: string[] = []; const inputs = entries.map((entry) => { const result = { - "Entry ID": entry.id, - "Data Silo ID": entry.dataSiloId, - "Object Path ID": entry.scannedObjectPathId, - "Object ID": entry.scannedObjectId, + 'Entry ID': entry.id, + 'Data Silo ID': entry.dataSiloId, + 'Object Path ID': entry.scannedObjectPathId, + 'Object ID': entry.scannedObjectId, ...(includeEncryptedSnippets - ? { Entry: entry.name, "Context Snippet": entry.contextSnippet } + ? { Entry: entry.name, 'Context Snippet': entry.contextSnippet } : {}), - "Data Category": `${entry.dataSubCategory.category}:${entry.dataSubCategory.name}`, - "Classification Status": entry.status, - "Confidence Score": entry.confidence, - "Classification Method": entry.classificationMethod, - "Classifier Version": entry.classifierVersion, + 'Data Category': `${entry.dataSubCategory.category}:${entry.dataSubCategory.name}`, + 'Classification Status': entry.status, + 'Confidence Score': entry.confidence, + 'Classification Method': entry.classificationMethod, + 'Classifier Version': entry.classifierVersion, }; headers = uniq([...headers, ...Object.keys(result)]); return result; @@ -68,8 +68,8 @@ export async function pullUnstructuredDiscoveryFiles( } catch (error) { logger.error( colors.red( - `An error occurred syncing the unstructured discovery files: ${error.message}` - ) + `An error occurred syncing the unstructured discovery files: ${error.message}`, + ), ); process.exit(1); } @@ -77,7 +77,7 @@ export async function pullUnstructuredDiscoveryFiles( // Indicate success logger.info( colors.green( - `Successfully synced unstructured discovery files to disk at ${file}!` - ) + `Successfully synced unstructured discovery files to disk at ${file}!`, + ), ); } diff --git a/src/commands/inventory/pull/impl.ts b/src/commands/inventory/pull/impl.ts index 799d574b..a6802e91 100644 --- a/src/commands/inventory/pull/impl.ts +++ b/src/commands/inventory/pull/impl.ts @@ -1,26 +1,26 @@ -import fs from "node:fs"; -import { join } from "node:path"; -import { ConsentTrackerStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { ADMIN_DASH_INTEGRATIONS } from "../../../constants"; -import type { LocalContext } from "../../../context"; -import { TranscendPullResource } from "../../../enums"; -import { validateTranscendAuth } from "../../../lib/api-keys"; -import { mapSeries } from "../../../lib/bluebird-replace"; +import fs from 'node:fs'; +import { join } from 'node:path'; +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { TranscendPullResource } from '../../../enums'; +import { validateTranscendAuth } from '../../../lib/api-keys'; +import { mapSeries } from '../../../lib/bluebird-replace'; import { buildTranscendGraphQLClient, pullTranscendConfiguration, -} from "../../../lib/graphql"; -import { writeTranscendYaml } from "../../../lib/readTranscendYaml"; -import { logger } from "../../../logger"; +} from '../../../lib/graphql'; +import { writeTranscendYaml } from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; import { DEFAULT_CONSENT_TRACKER_STATUSES, DEFAULT_TRANSCEND_PULL_RESOURCES, -} from "./command"; +} from './command'; interface PullCommandFlags { auth: string; - resources?: (TranscendPullResource | "all")[]; + resources?: (TranscendPullResource | 'all')[]; file: string; transcendUrl: string; dataSiloIds?: string[]; @@ -48,17 +48,17 @@ export async function pull( skipSubDatapoints, includeGuessedCategories, debug, - }: PullCommandFlags + }: PullCommandFlags, ): Promise { // Parse authentication as API key or path to list of API keys const apiKeyOrList = await validateTranscendAuth(auth); - const resourcesToPull: TranscendPullResource[] = resources.includes("all") + const resourcesToPull: TranscendPullResource[] = resources.includes('all') ? Object.values(TranscendPullResource) : (resources as TranscendPullResource[]); // Sync to Disk - if (typeof apiKeyOrList === "string") { + if (typeof apiKeyOrList === 'string') { try { // Create a GraphQL client const client = buildTranscendGraphQLClient(transcendUrl, apiKeyOrList); @@ -82,8 +82,8 @@ export async function pull( colors.red( `An error occurred syncing the schema: ${ debug ? error.stack : error.message - }` - ) + }`, + ), ); process.exit(1); } @@ -91,13 +91,13 @@ export async function pull( // Indicate success logger.info( colors.green( - `Successfully synced yaml file to disk at ${file}! View at ${ADMIN_DASH_INTEGRATIONS}` - ) + `Successfully synced yaml file to disk at ${file}! View at ${ADMIN_DASH_INTEGRATIONS}`, + ), ); } else { if (!fs.lstatSync(file).isDirectory()) { throw new Error( - "File is expected to be a folder when passing in a list of API keys to pull from. e.g. --file=./working/" + 'File is expected to be a folder when passing in a list of API keys to pull from. e.g. --file=./working/', ); } @@ -108,8 +108,8 @@ export async function pull( }] `; logger.info( colors.magenta( - `~~~\n\n${prefix}Attempting to pull configuration...\n\n~~~` - ) + `~~~\n\n${prefix}Attempting to pull configuration...\n\n~~~`, + ), ); // Create a GraphQL client @@ -130,18 +130,18 @@ export async function pull( const filePath = join(file, `${apiKey.organizationName}.yml`); logger.info( - colors.magenta(`Writing configuration to file "${filePath}"...`) + colors.magenta(`Writing configuration to file "${filePath}"...`), ); writeTranscendYaml(filePath, configuration); logger.info( - colors.green(`${prefix}Successfully pulled configuration!`) + colors.green(`${prefix}Successfully pulled configuration!`), ); } catch (error) { logger.error( colors.red( - `${prefix}Failed to sync configuration. - ${error.message}` - ) + `${prefix}Failed to sync configuration. - ${error.message}`, + ), ); encounteredErrors.push(apiKey.organizationName); } @@ -151,9 +151,9 @@ export async function pull( logger.info( colors.red( `Sync encountered errors for "${encounteredErrors.join( - "," - )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}` - ) + ',', + )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}`, + ), ); process.exit(1); diff --git a/src/commands/inventory/push/impl.ts b/src/commands/inventory/push/impl.ts index d95b7633..3ba81b28 100644 --- a/src/commands/inventory/push/impl.ts +++ b/src/commands/inventory/push/impl.ts @@ -1,19 +1,19 @@ -import { existsSync, lstatSync } from "node:fs"; -import { join } from "node:path"; -import colors from "colors"; -import { TranscendInput } from "../../../codecs"; -import { ADMIN_DASH_INTEGRATIONS } from "../../../constants"; -import type { LocalContext } from "../../../context"; -import { listFiles, validateTranscendAuth } from "../../../lib/api-keys"; -import { mapSeries } from "../../../lib/bluebird-replace"; +import { existsSync, lstatSync } from 'node:fs'; +import { join } from 'node:path'; +import colors from 'colors'; +import { TranscendInput } from '../../../codecs'; +import { ADMIN_DASH_INTEGRATIONS } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { listFiles, validateTranscendAuth } from '../../../lib/api-keys'; +import { mapSeries } from '../../../lib/bluebird-replace'; import { buildTranscendGraphQLClient, syncConfigurationToTranscend, -} from "../../../lib/graphql"; -import { parseVariablesFromString } from "../../../lib/helpers/parseVariablesFromString"; -import { mergeTranscendInputs } from "../../../lib/mergeTranscendInputs"; -import { readTranscendYaml } from "../../../lib/readTranscendYaml"; -import { logger } from "../../../logger"; +} from '../../../lib/graphql'; +import { parseVariablesFromString } from '../../../lib/helpers/parseVariablesFromString'; +import { mergeTranscendInputs } from '../../../lib/mergeTranscendInputs'; +import { readTranscendYaml } from '../../../lib/readTranscendYaml'; +import { logger } from '../../../logger'; /** * Sync configuration to Transcend @@ -57,14 +57,14 @@ async function syncConfiguration({ publishToPrivacyCenter, classifyService, deleteExtraAttributeValues, - } + }, ); return !encounteredError; } catch (error) { logger.error( colors.red( - `An unexpected error occurred syncing the schema: ${error.message}` - ) + `An unexpected error occurred syncing the schema: ${error.message}`, + ), ); return false; } @@ -84,7 +84,7 @@ interface PushCommandFlags { export async function push( this: LocalContext, { - file = "./transcend.yml", + file = './transcend.yml', transcendUrl, auth, variables, @@ -92,7 +92,7 @@ export async function push( publishToPrivacyCenter, classifyService, deleteExtraAttributeValues, - }: PushCommandFlags + }: PushCommandFlags, ): Promise { // Parse authentication as API key or path to list of API keys const apiKeyOrList = await validateTranscendAuth(auth); @@ -105,11 +105,11 @@ export async function push( fileList = Array.isArray(apiKeyOrList) && lstatSync(file).isDirectory() ? listFiles(file).map((filePath) => join(file, filePath)) - : file.split(","); + : file.split(','); // Ensure at least one file is parsed if (fileList.length === 0) { - throw new Error("No file specified!"); + throw new Error('No file specified!'); } const transcendInputs = fileList.map((filePath) => { @@ -119,8 +119,8 @@ export async function push( } else { logger.error( colors.red( - `The file path does not exist on disk: ${filePath}. You can specify the filepath using --file=./examples/transcend.yml` - ) + `The file path does not exist on disk: ${filePath}. You can specify the filepath using --file=./examples/transcend.yml`, + ), ); process.exit(1); } @@ -131,20 +131,20 @@ export async function push( logger.info(colors.green(`Successfully read in "${filePath}"`)); return { content: newContents, - name: filePath.split("/").pop()!.replace(".yml", ""), + name: filePath.split('/').pop()!.replace('.yml', ''), }; } catch (error) { logger.error( colors.red( - `The shape of your yaml file is invalid with the following errors: ${error.message}` - ) + `The shape of your yaml file is invalid with the following errors: ${error.message}`, + ), ); process.exit(1); } }); // process a single API key - if (typeof apiKeyOrList === "string") { + if (typeof apiKeyOrList === 'string') { // if passed multiple inputs, merge them together const [base, ...rest] = transcendInputs.map(({ content }) => content); const contents = mergeTranscendInputs(base, ...rest); @@ -164,8 +164,8 @@ export async function push( if (!success) { logger.info( colors.red( - `Sync encountered errors. View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}` - ) + `Sync encountered errors. View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}`, + ), ); process.exit(1); @@ -177,12 +177,12 @@ export async function push( transcendInputs.length !== apiKeyOrList.length ) { throw new Error( - "Expected list of yml files to be equal to the list of API keys." + + 'Expected list of yml files to be equal to the list of API keys.' + `Got ${transcendInputs.length} YML file${ - transcendInputs.length === 1 ? "" : "s" + transcendInputs.length === 1 ? '' : 's' } and ${apiKeyOrList.length} API key${ - apiKeyOrList.length === 1 ? "" : "s" - }` + apiKeyOrList.length === 1 ? '' : 's' + }`, ); } @@ -193,8 +193,8 @@ export async function push( }] `; logger.info( colors.magenta( - `~~~\n\n${prefix}Attempting to push configuration...\n\n~~~` - ) + `~~~\n\n${prefix}Attempting to push configuration...\n\n~~~`, + ), ); // use the merged contents if 1 yml passed, else use the contents that map to that organization @@ -202,15 +202,15 @@ export async function push( transcendInputs.length === 1 ? transcendInputs[0].content : transcendInputs.find( - (input) => input.name === apiKey.organizationName + (input) => input.name === apiKey.organizationName, )?.content; // Throw error if cannot find a yml file matching that organization name if (!useContents) { logger.error( colors.red( - `${prefix}Failed to find transcend.yml file for organization: "${apiKey.organizationName}".` - ) + `${prefix}Failed to find transcend.yml file for organization: "${apiKey.organizationName}".`, + ), ); encounteredErrors.push(apiKey.organizationName); return; @@ -228,7 +228,7 @@ export async function push( if (success) { logger.info( - colors.green(`${prefix}Successfully pushed configuration!`) + colors.green(`${prefix}Successfully pushed configuration!`), ); } else { logger.error(colors.red(`${prefix}Failed to sync configuration.`)); @@ -240,9 +240,9 @@ export async function push( logger.info( colors.red( `Sync encountered errors for "${encounteredErrors.join( - "," - )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}` - ) + ',', + )}". View output above for more information, or check out ${ADMIN_DASH_INTEGRATIONS}`, + ), ); process.exit(1); @@ -252,7 +252,7 @@ export async function push( // Indicate success logger.info( colors.green( - `Successfully synced yaml file to Transcend! View at ${ADMIN_DASH_INTEGRATIONS}` - ) + `Successfully synced yaml file to Transcend! View at ${ADMIN_DASH_INTEGRATIONS}`, + ), ); } diff --git a/src/commands/inventory/scan-packages/impl.ts b/src/commands/inventory/scan-packages/impl.ts index c2198f90..6b6abe12 100644 --- a/src/commands/inventory/scan-packages/impl.ts +++ b/src/commands/inventory/scan-packages/impl.ts @@ -1,17 +1,17 @@ -import { execSync } from "node:child_process"; -import colors from "colors"; -import { ADMIN_DASH } from "../../../constants"; -import type { LocalContext } from "../../../context"; -import { findCodePackagesInFolder } from "../../../lib/code-scanning"; +import { execSync } from 'node:child_process'; +import colors from 'colors'; +import { ADMIN_DASH } from '../../../constants'; +import type { LocalContext } from '../../../context'; +import { findCodePackagesInFolder } from '../../../lib/code-scanning'; import { buildTranscendGraphQLClient, syncCodePackages, -} from "../../../lib/graphql"; -import { logger } from "../../../logger"; +} from '../../../lib/graphql'; +import { logger } from '../../../logger'; const REPO_ERROR = - "A repository name must be provided. " + - "You can specify using --repositoryName=$REPO_NAME or by ensuring the " + + 'A repository name must be provided. ' + + 'You can specify using --repositoryName=$REPO_NAME or by ensuring the ' + 'command "git config --get remote.origin.url" returns the name of the repository'; interface ScanPackagesCommandFlags { @@ -30,20 +30,20 @@ export async function scanPackages( ignoreDirs, repositoryName, transcendUrl, - }: ScanPackagesCommandFlags + }: ScanPackagesCommandFlags, ): Promise { // Ensure repository name is specified let gitRepositoryName = repositoryName; if (!gitRepositoryName) { try { const name = execSync( - `cd ${scanPath} && git config --get remote.origin.url` + `cd ${scanPath} && git config --get remote.origin.url`, ); // Trim and parse the URL - const url = name.toString("utf-8").trim(); - [gitRepositoryName] = url.includes("https:") - ? url.split("/").slice(3).join("/").split(".") - : (url.split(":").pop() || "").split("."); + const url = name.toString('utf-8').trim(); + [gitRepositoryName] = url.includes('https:') + ? url.split('/').slice(3).join('/').split('.') + : (url.split(':').pop() || '').split('.'); if (!gitRepositoryName) { logger.error(colors.red(REPO_ERROR)); process.exit(1); @@ -68,13 +68,13 @@ export async function scanPackages( await syncCodePackages(client, results); const newUrl = new URL(ADMIN_DASH); - newUrl.pathname = "/code-scanning/code-packages"; + newUrl.pathname = '/code-scanning/code-packages'; // Indicate success logger.info( colors.green( `Scan found ${results.length} packages at ${scanPath}! ` + - `View results at '${newUrl.href}'` - ) + `View results at '${newUrl.href}'`, + ), ); } diff --git a/src/commands/migration/sync-ot/impl.ts b/src/commands/migration/sync-ot/impl.ts index 443e91e1..fd8914f6 100644 --- a/src/commands/migration/sync-ot/impl.ts +++ b/src/commands/migration/sync-ot/impl.ts @@ -1,17 +1,17 @@ -import colors from "colors"; -import type { LocalContext } from "../../../context"; +import colors from 'colors'; +import type { LocalContext } from '../../../context'; import { OneTrustFileFormat, OneTrustPullResource, OneTrustPullSource, -} from "../../../enums"; -import { buildTranscendGraphQLClient } from "../../../lib/graphql"; -import { createOneTrustGotInstance } from "../../../lib/oneTrust"; +} from '../../../enums'; +import { buildTranscendGraphQLClient } from '../../../lib/graphql'; +import { createOneTrustGotInstance } from '../../../lib/oneTrust'; import { syncOneTrustAssessmentsFromFile, syncOneTrustAssessmentsFromOneTrust, -} from "../../../lib/oneTrust/helpers"; -import { logger } from "../../../logger"; +} from '../../../lib/oneTrust/helpers'; +import { logger } from '../../../logger'; // Command flag interface interface SyncOtCommandFlags { @@ -39,34 +39,34 @@ export async function syncOt( file, dryRun, debug, - }: SyncOtCommandFlags + }: SyncOtCommandFlags, ): Promise { // Must be able to authenticate to transcend to sync resources to it if (!dryRun && !transcendAuth) { throw new Error( - 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}' + 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}', ); } // If trying to sync to disk, must specify a file path if (dryRun && !file) { throw new Error( - 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json' + 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json', ); } if (file) { - const splitFile = file.split("."); + const splitFile = file.split('.'); if (splitFile.length < 2) { throw new Error( - 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.' + 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.', ); } if (splitFile.at(-1) !== OneTrustFileFormat.Json) { throw new Error( `Expected the format of the "file" parameters '${file}' to be '${ OneTrustFileFormat.Json - }', but got '${splitFile.at(-1)}'.` + }', but got '${splitFile.at(-1)}'.`, ); } } @@ -76,28 +76,28 @@ export async function syncOt( // must specify the OneTrust hostname if (!hostname) { throw new Error( - 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com' + 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com', ); } // must specify the OneTrust auth if (!oneTrustAuth) { throw new Error( - 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN' + 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN', ); } } else { // if reading the assessments from a file, must specify a file to read from if (!file) { throw new Error( - 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json' + 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json', ); } // Cannot try reading from file and save assessments to a file simultaneously if (dryRun) { throw new Error( - "Cannot read and write to a file simultaneously." + - ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.` + 'Cannot read and write to a file simultaneously.' + + ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.`, ); } } @@ -134,7 +134,7 @@ export async function syncOt( throw new Error( `An error occurred syncing the resource ${resource} from OneTrust: ${ debug ? error.stack : error.message - }` + }`, ); } @@ -142,8 +142,8 @@ export async function syncOt( logger.info( colors.green( `Successfully synced OneTrust ${resource} to ${ - dryRun ? `disk at "${file}"` : "Transcend" - }!` - ) + dryRun ? `disk at "${file}"` : 'Transcend' + }!`, + ), ); } diff --git a/src/commands/request/cron/pull-identifiers/impl.ts b/src/commands/request/cron/pull-identifiers/impl.ts index 140efaf2..2f1dd235 100644 --- a/src/commands/request/cron/pull-identifiers/impl.ts +++ b/src/commands/request/cron/pull-identifiers/impl.ts @@ -1,14 +1,14 @@ -import { RequestAction } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { uniq } from "lodash-es"; -import type { LocalContext } from "../../../../context"; +import { RequestAction } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { uniq } from 'lodash-es'; +import type { LocalContext } from '../../../../context'; import { CsvFormattedIdentifier, parseFilePath, pullChunkedCustomSiloOutstandingIdentifiers, writeCsv, -} from "../../../../lib/cron"; -import { logger } from "../../../../logger"; +} from '../../../../lib/cron'; +import { logger } from '../../../../logger'; interface PullIdentifiersCommandFlags { file: string; @@ -34,13 +34,13 @@ export async function pullIdentifiers( pageLimit, skipRequestCount, chunkSize, - }: PullIdentifiersCommandFlags + }: PullIdentifiersCommandFlags, ): Promise { if (skipRequestCount) { logger.info( colors.yellow( - "Skipping request count as requested. This may help speed up the call." - ) + 'Skipping request count as requested. This may help speed up the call.', + ), ); } @@ -51,8 +51,8 @@ export async function pullIdentifiers( ) { logger.error( colors.red( - `Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${pageLimit}.` - ) + `Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${pageLimit}.`, + ), ); process.exit(1); } @@ -64,16 +64,16 @@ export async function pullIdentifiers( const numberedFileName = `${baseName}-${fileCount}${extension}`; logger.info( colors.blue( - `Saving ${chunk.length} identifiers to file "${numberedFileName}"` - ) + `Saving ${chunk.length} identifiers to file "${numberedFileName}"`, + ), ); const headers = uniq(chunk.flatMap((d) => Object.keys(d))); writeCsv(numberedFileName, chunk, headers); logger.info( colors.green( - `Successfully wrote ${chunk.length} identifiers to file "${numberedFileName}"` - ) + `Successfully wrote ${chunk.length} identifiers to file "${numberedFileName}"`, + ), ); fileCount += 1; return Promise.resolve(); diff --git a/src/commands/request/cron/pull-profiles/impl.ts b/src/commands/request/cron/pull-profiles/impl.ts index 3f800a1d..c6ff6848 100644 --- a/src/commands/request/cron/pull-profiles/impl.ts +++ b/src/commands/request/cron/pull-profiles/impl.ts @@ -1,19 +1,19 @@ -import type { RequestAction } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { uniq } from "lodash-es"; -import type { LocalContext } from "../../../../context"; -import { map } from "../../../../lib/bluebird-replace"; +import type { RequestAction } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { uniq } from 'lodash-es'; +import type { LocalContext } from '../../../../context'; +import { map } from '../../../../lib/bluebird-replace'; import { parseFilePath, pullChunkedCustomSiloOutstandingIdentifiers, writeCsv, type CsvFormattedIdentifier, -} from "../../../../lib/cron"; +} from '../../../../lib/cron'; import { buildTranscendGraphQLClient, fetchRequestFilesForRequest, -} from "../../../../lib/graphql"; -import { logger } from "../../../../logger"; +} from '../../../../lib/graphql'; +import { logger } from '../../../../logger'; interface PullProfilesCommandFlags { file: string; @@ -43,13 +43,13 @@ export async function pullProfiles( skipRequestCount, pageLimit, chunkSize, - }: PullProfilesCommandFlags + }: PullProfilesCommandFlags, ): Promise { if (skipRequestCount) { logger.info( colors.yellow( - "Skipping request count as requested. This may help speed up the call." - ) + 'Skipping request count as requested. This may help speed up the call.', + ), ); } @@ -60,8 +60,8 @@ export async function pullProfiles( ) { logger.error( colors.red( - `Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${pageLimit}.` - ) + `Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${pageLimit}.`, + ), ); process.exit(1); } @@ -95,24 +95,24 @@ export async function pullProfiles( return results.map(({ fileName, remoteId }) => { if (!remoteId) { throw new Error( - `Failed to find remoteId for ${fileName} request: ${requestId}` + `Failed to find remoteId for ${fileName} request: ${requestId}`, ); } return { RecordId: remoteId, Object: fileName - .replace(".json", "") - .split("/") + .replace('.json', '') + .split('/') .pop() - ?.replace(" Information", ""), + ?.replace(' Information', ''), Comment: - "Customer data deletion request submitted via transcend.io", + 'Customer data deletion request submitted via transcend.io', }; }); }, { concurrency: 10, - } + }, ); allTargetIdentifiersCount += results.flat().length; @@ -124,8 +124,8 @@ export async function pullProfiles( writeCsv(numberedFileName, chunk, headers); logger.info( colors.green( - `Successfully wrote ${chunk.length} identifiers to file "${file}"` - ) + `Successfully wrote ${chunk.length} identifiers to file "${file}"`, + ), ); const targetIdentifiers = results.flat(); @@ -133,14 +133,14 @@ export async function pullProfiles( writeCsv(numberedFileNameTarget, targetIdentifiers, headers2); logger.info( colors.green( - `Successfully wrote ${targetIdentifiers.length} identifiers to file "${fileTarget}"` - ) + `Successfully wrote ${targetIdentifiers.length} identifiers to file "${fileTarget}"`, + ), ); logger.info( colors.blue( - `Processed chunk of ${chunk.length} identifiers, found ${targetIdentifiers.length} target identifiers` - ) + `Processed chunk of ${chunk.length} identifiers, found ${targetIdentifiers.length} target identifiers`, + ), ); fileCount += 1; }; @@ -160,12 +160,12 @@ export async function pullProfiles( logger.info( colors.green( - `Successfully wrote ${allIdentifiersCount} identifiers to file "${file}"` - ) + `Successfully wrote ${allIdentifiersCount} identifiers to file "${file}"`, + ), ); logger.info( colors.green( - `Successfully wrote ${allTargetIdentifiersCount} identifiers to file "${fileTarget}"` - ) + `Successfully wrote ${allTargetIdentifiersCount} identifiers to file "${fileTarget}"`, + ), ); } diff --git a/src/commands/request/export/impl.ts b/src/commands/request/export/impl.ts index c40bb266..1e538445 100644 --- a/src/commands/request/export/impl.ts +++ b/src/commands/request/export/impl.ts @@ -1,10 +1,10 @@ -import type { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { uniq } from "lodash-es"; -import type { LocalContext } from "../../../context"; -import { writeCsv } from "../../../lib/cron"; -import { pullPrivacyRequests } from "../../../lib/requests"; -import { logger } from "../../../logger"; +import type { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { uniq } from 'lodash-es'; +import type { LocalContext } from '../../../context'; +import { writeCsv } from '../../../lib/cron'; +import { pullPrivacyRequests } from '../../../lib/requests'; +import { logger } from '../../../logger'; interface ExportCommandFlags { auth: string; @@ -35,7 +35,7 @@ export async function _export( createdAtBefore, createdAtAfter, showTests, - }: ExportCommandFlags + }: ExportCommandFlags, ): Promise { const { requestsFormattedForCsv } = await pullPrivacyRequests({ transcendUrl, @@ -54,7 +54,7 @@ export async function _export( writeCsv(file, requestsFormattedForCsv, headers); logger.info( colors.green( - `Successfully wrote ${requestsFormattedForCsv.length} requests to file "${file}"` - ) + `Successfully wrote ${requestsFormattedForCsv.length} requests to file "${file}"`, + ), ); } diff --git a/src/constants.ts b/src/constants.ts index 42fd2711..dbead72b 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,11 +1,11 @@ -import { ScopeName } from "@transcend-io/privacy-types"; -import { TranscendInput } from "./codecs"; -import { TranscendPullResource } from "./enums"; +import { ScopeName } from '@transcend-io/privacy-types'; +import { TranscendInput } from './codecs'; +import { TranscendPullResource } from './enums'; -export { description, version } from "../package.json"; -export const name = "transcend"; +export { description, version } from '../package.json'; +export const name = 'transcend'; -export const ADMIN_DASH = "https://app.transcend.io"; +export const ADMIN_DASH = 'https://app.transcend.io'; export const ADMIN_DASH_INTEGRATIONS = `${ADMIN_DASH}/infrastructure/integrations`; export const ADMIN_DASH_DATAPOINTS = `${ADMIN_DASH}/data-map/data-inventory/data-points`; @@ -15,14 +15,14 @@ export const ADMIN_DASH_DATAPOINTS = `${ADMIN_DASH}/data-map/data-inventory/data * TRANSCEND_API_URL=https://api.us.transcend.io transcend ... */ export const DEFAULT_TRANSCEND_API = - process.env.TRANSCEND_API_URL || "https://api.transcend.io"; + process.env.TRANSCEND_API_URL || 'https://api.transcend.io'; /** * Override default transcend API url using * TRANSCEND_CONSENT_API_URL=https://consent.us.transcend.io transcend ... */ export const DEFAULT_TRANSCEND_CONSENT_API = - process.env.TRANSCEND_CONSENT_API_URL || "https://consent.transcend.io"; + process.env.TRANSCEND_CONSENT_API_URL || 'https://consent.transcend.io'; /** * Mapping between resource type and scopes required for cli @@ -133,35 +133,35 @@ export const TR_YML_RESOURCE_TO_FIELD_NAME: Record< TranscendPullResource, keyof TranscendInput > = { - [TranscendPullResource.ApiKeys]: "api-keys", - [TranscendPullResource.Attributes]: "attributes", - [TranscendPullResource.DataFlows]: "data-flows", - [TranscendPullResource.Cookies]: "cookies", - [TranscendPullResource.ConsentManager]: "consent-manager", - [TranscendPullResource.Partitions]: "partitions", - [TranscendPullResource.Actions]: "actions", - [TranscendPullResource.DataSubjects]: "data-subjects", - [TranscendPullResource.BusinessEntities]: "business-entities", - [TranscendPullResource.Identifiers]: "identifiers", - [TranscendPullResource.Enrichers]: "enrichers", - [TranscendPullResource.DataSilos]: "data-silos", - [TranscendPullResource.Templates]: "templates", - [TranscendPullResource.Prompts]: "prompts", - [TranscendPullResource.PromptPartials]: "prompt-partials", - [TranscendPullResource.PromptGroups]: "prompt-groups", - [TranscendPullResource.Agents]: "agents", - [TranscendPullResource.AgentFunctions]: "agent-functions", - [TranscendPullResource.AgentFiles]: "agent-files", - [TranscendPullResource.Vendors]: "vendors", - [TranscendPullResource.DataCategories]: "data-categories", - [TranscendPullResource.ProcessingPurposes]: "processing-purposes", - [TranscendPullResource.ActionItems]: "action-items", - [TranscendPullResource.ActionItemCollections]: "action-item-collections", - [TranscendPullResource.Teams]: "teams", - [TranscendPullResource.Messages]: "messages", - [TranscendPullResource.PrivacyCenters]: "privacy-center", - [TranscendPullResource.Policies]: "policies", - [TranscendPullResource.Assessments]: "assessments", - [TranscendPullResource.AssessmentTemplates]: "assessment-templates", - [TranscendPullResource.Purposes]: "purposes", + [TranscendPullResource.ApiKeys]: 'api-keys', + [TranscendPullResource.Attributes]: 'attributes', + [TranscendPullResource.DataFlows]: 'data-flows', + [TranscendPullResource.Cookies]: 'cookies', + [TranscendPullResource.ConsentManager]: 'consent-manager', + [TranscendPullResource.Partitions]: 'partitions', + [TranscendPullResource.Actions]: 'actions', + [TranscendPullResource.DataSubjects]: 'data-subjects', + [TranscendPullResource.BusinessEntities]: 'business-entities', + [TranscendPullResource.Identifiers]: 'identifiers', + [TranscendPullResource.Enrichers]: 'enrichers', + [TranscendPullResource.DataSilos]: 'data-silos', + [TranscendPullResource.Templates]: 'templates', + [TranscendPullResource.Prompts]: 'prompts', + [TranscendPullResource.PromptPartials]: 'prompt-partials', + [TranscendPullResource.PromptGroups]: 'prompt-groups', + [TranscendPullResource.Agents]: 'agents', + [TranscendPullResource.AgentFunctions]: 'agent-functions', + [TranscendPullResource.AgentFiles]: 'agent-files', + [TranscendPullResource.Vendors]: 'vendors', + [TranscendPullResource.DataCategories]: 'data-categories', + [TranscendPullResource.ProcessingPurposes]: 'processing-purposes', + [TranscendPullResource.ActionItems]: 'action-items', + [TranscendPullResource.ActionItemCollections]: 'action-item-collections', + [TranscendPullResource.Teams]: 'teams', + [TranscendPullResource.Messages]: 'messages', + [TranscendPullResource.PrivacyCenters]: 'privacy-center', + [TranscendPullResource.Policies]: 'policies', + [TranscendPullResource.Assessments]: 'assessments', + [TranscendPullResource.AssessmentTemplates]: 'assessment-templates', + [TranscendPullResource.Purposes]: 'purposes', }; diff --git a/src/lib/ai/TranscendPromptManager.ts b/src/lib/ai/TranscendPromptManager.ts index 7fed5f5d..86571d25 100644 --- a/src/lib/ai/TranscendPromptManager.ts +++ b/src/lib/ai/TranscendPromptManager.ts @@ -1,28 +1,28 @@ -import type { Handlebars } from "@transcend-io/handlebars-utils"; +import type { Handlebars } from '@transcend-io/handlebars-utils'; import { createHandlebarsWithHelpers, HandlebarsInput, -} from "@transcend-io/handlebars-utils"; +} from '@transcend-io/handlebars-utils'; import { ChatCompletionRole, LargeLanguageModelClient, PromptRunProductArea, PromptStatus, QueueStatus, -} from "@transcend-io/privacy-types"; -import { Secret } from "@transcend-io/secret-value"; +} from '@transcend-io/privacy-types'; +import { Secret } from '@transcend-io/secret-value'; import { apply, decodeCodec, getValues, Optionalize, Requirize, -} from "@transcend-io/type-utils"; -import { GraphQLClient } from "graphql-request"; -import * as t from "io-ts"; -import { chunk, groupBy, keyBy, uniq } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { mapSeries } from "../bluebird-replace"; +} from '@transcend-io/type-utils'; +import { GraphQLClient } from 'graphql-request'; +import * as t from 'io-ts'; +import { chunk, groupBy, keyBy, uniq } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { mapSeries } from '../bluebird-replace'; import { Agent, AgentFile, @@ -32,27 +32,27 @@ import { fetchAllAgents, reportPromptRun, ReportPromptRunInput, -} from "../graphql"; +} from '../graphql'; import { fetchAllLargeLanguageModels, LargeLanguageModel, -} from "../graphql/fetchLargeLanguageModels"; +} from '../graphql/fetchLargeLanguageModels'; import { fetchPromptsWithVariables, TranscendPromptsAndVariables, TranscendPromptTemplated, -} from "../graphql/fetchPrompts"; +} from '../graphql/fetchPrompts'; import { fetchAllPromptThreads, PromptThread, -} from "../graphql/fetchPromptThreads"; +} from '../graphql/fetchPromptThreads'; /** * An LLM Prompt definition */ export type TranscendPrompt< TInputParameters extends t.Any, - TOutputCodec extends t.Any + TOutputCodec extends t.Any, > = ( | { /** ID of the prompt */ @@ -108,7 +108,7 @@ export function createRegexForTag(tagName: string): RegExp { */ export function defineTranscendPrompts< TPromptNames extends string, - TPrompts extends Record> + TPrompts extends Record>, >(prompts: TPrompts): TPrompts { return prompts; } @@ -118,24 +118,24 @@ export function defineTranscendPrompts< */ export type GetPromptParamType< TPromptName extends keyof TPrompts, - TPrompts extends Record> -> = t.TypeOf; + TPrompts extends Record>, +> = t.TypeOf; /** * Helper to get the type of the parameter for a given prompt */ export type GetPromptResponseType< TPromptName extends keyof TPrompts, - TPrompts extends Record> -> = t.TypeOf; + TPrompts extends Record>, +> = t.TypeOf; /** * Input for reporting a prompt run */ export interface ReportPromptRunOptions extends Optionalize< - Omit, - "name" | "productArea" + Omit, + 'name' | 'productArea' > { /** The large language model being run. Either the ID of the LLM or the client/name pairing */ largeLanguageModel: @@ -162,7 +162,7 @@ const jsonParseSafe = (object: string): unknown => { */ export class TranscendPromptManager< TPromptNames extends string, - TPrompts extends Record> + TPrompts extends Record>, > { /** Prompt definitions */ public prompts: TPrompts; @@ -260,9 +260,9 @@ export class TranscendPromptManager< this.defaultVariables = defaultVariables; this.graphQLClient = buildTranscendGraphQLClient( transcendUrl, - typeof transcendApiKey === "object" + typeof transcendApiKey === 'object' ? transcendApiKey.release() - : transcendApiKey + : transcendApiKey, ); this.requireApproval = requireApproval; this.cacheDuration = cacheDuration; @@ -285,7 +285,7 @@ export class TranscendPromptManager< .map(({ title }) => title) .filter((x): x is string => !!x); const agentNames = uniq( - promptDefinitions.flatMap(({ agentNames }) => agentNames || []) + promptDefinitions.flatMap(({ agentNames }) => agentNames || []), ); // Fetch prompts and data @@ -297,15 +297,15 @@ export class TranscendPromptManager< fetchAllLargeLanguageModels(this.graphQLClient), fetchAllAgents(this.graphQLClient, { names: agentNames }), ]); - this.agentsByName = keyBy(agents, "name"); - this.agentsByAgentId = keyBy(agents, "agentId"); + this.agentsByName = keyBy(agents, 'name'); + this.agentsByAgentId = keyBy(agents, 'agentId'); this.largeLanguageModels = largeLanguageModels.filter( - (model) => !model.isTranscendHosted + (model) => !model.isTranscendHosted, ); // Lookup prompts by id/title - const promptByTitle = keyBy(response.prompts, "title"); - const promptById = keyBy(response.prompts, "id"); + const promptByTitle = keyBy(response.prompts, 'title'); + const promptById = keyBy(response.prompts, 'id'); // Update variables this.variables = { @@ -313,7 +313,7 @@ export class TranscendPromptManager< response.calculatedVariables.map((v) => [ v.name, v.data ? JSON.parse(v.data) : v.data, - ]) + ]), ), ...this.defaultVariables, }; @@ -335,11 +335,11 @@ export class TranscendPromptManager< const result = id ? promptById[id] : title - ? promptByTitle[title] - : undefined; + ? promptByTitle[title] + : undefined; if (!result) { throw new Error( - `Failed to find prompt with title: "${title}" and id: "${id}"` + `Failed to find prompt with title: "${title}" and id: "${id}"`, ); } return result; @@ -380,7 +380,7 @@ export class TranscendPromptManager< * @returns Large language model configuration */ async getPromptThreadBySlackTs( - ts: string + ts: string, ): Promise { const [thread] = await fetchAllPromptThreads(this.graphQLClient, { slackMessageTs: [ts], @@ -398,10 +398,10 @@ export class TranscendPromptManager< */ async getAgentsByName(names: string[]): Promise { if (names.length === 0) { - throw new Error("Expected at least one name to be provided"); + throw new Error('Expected at least one name to be provided'); } const { hasCache = [], missingCache = [] } = groupBy(names, (name) => - this.agentsByName[name] ? "hasCache" : "missingCache" + this.agentsByName[name] ? 'hasCache' : 'missingCache', ); const cachedAgents = hasCache.map((name) => this.agentsByName[name]); if (missingCache.length === 0) { @@ -439,21 +439,21 @@ export class TranscendPromptManager< * @returns Large language model configuration */ getLargeLanguageModel( - largeLanguageModel: ReportPromptRunOptions["largeLanguageModel"] + largeLanguageModel: ReportPromptRunOptions['largeLanguageModel'], ): LargeLanguageModel { const matching = this.largeLanguageModels.find((model) => - typeof largeLanguageModel === "string" + typeof largeLanguageModel === 'string' ? model.id === largeLanguageModel : model.name === largeLanguageModel.name && - model.client === largeLanguageModel.client + model.client === largeLanguageModel.client, ); if (!matching) { throw new Error( `Failed to find model matching: ${ - typeof largeLanguageModel === "string" + typeof largeLanguageModel === 'string' ? largeLanguageModel : JSON.stringify(largeLanguageModel) - }` + }`, ); } return matching; @@ -466,7 +466,7 @@ export class TranscendPromptManager< * @returns Parsed content */ async getPromptDefinition( - promptName: TPromptName + promptName: TPromptName, ): Promise { // Determine if prompts need to be fetched if ( @@ -484,12 +484,12 @@ export class TranscendPromptManager< // Lookup prompt const { promptContentMap } = this; if (!promptContentMap) { - throw new Error("Expected this.promptContentMap to be defined"); + throw new Error('Expected this.promptContentMap to be defined'); } const promptTemplate = promptContentMap[promptName]; if (!promptTemplate) { throw new Error( - `Expected this.promptContentMap[${promptName}] to be defined` + `Expected this.promptContentMap[${promptName}] to be defined`, ); } return promptTemplate; @@ -504,7 +504,7 @@ export class TranscendPromptManager< */ async compilePrompt( promptName: TPromptName, - parameters: t.TypeOf + parameters: t.TypeOf, ): Promise { // Grab the prompt const promptTemplate = await this.getPromptDefinition(promptName); @@ -519,14 +519,14 @@ export class TranscendPromptManager< promptTemplate.status !== PromptStatus.Approved ) { throw new Error( - `Assessment "${promptTemplate.title}" cannot be used because its in status: "${promptTemplate.status}"` + `Assessment "${promptTemplate.title}" cannot be used because its in status: "${promptTemplate.status}"`, ); } // If prompt is rejected, throw error if (promptTemplate.status === PromptStatus.Rejected) { throw new Error( - `Assessment "${promptTemplate.title}" cannot be used because it's in status: "${promptTemplate.status}"` + `Assessment "${promptTemplate.title}" cannot be used because it's in status: "${promptTemplate.status}"`, ); } @@ -551,8 +551,8 @@ export class TranscendPromptManager< */ parseAiResponse( promptName: TPromptName, - response: string - ): t.TypeOf { + response: string, + ): t.TypeOf { // Look up prompt info const promptInput = this.prompts[promptName]; if (!promptInput) { @@ -569,7 +569,7 @@ export class TranscendPromptManager< return decodeCodec( promptInput.outputCodec, jsonParseSafe(extracted), - false + false, ); } @@ -582,11 +582,11 @@ export class TranscendPromptManager< */ async reportAndParsePromptRun( promptName: TPromptName, - { largeLanguageModel, ...options }: ReportPromptRunOptions + { largeLanguageModel, ...options }: ReportPromptRunOptions, ): Promise< PromptRunResult & { /** Resulting prompt run */ - result: t.TypeOf; + result: t.TypeOf; } > { // Determine if prompts need to be fetched @@ -607,7 +607,7 @@ export class TranscendPromptManager< `@transcend-io/cli-prompt-run-${new Date().toISOString()}`; if (!this.promptContentMap) { - throw new Error("Expected this.promptContentMap to be defined"); + throw new Error('Expected this.promptContentMap to be defined'); } // Look up prompt info const promptInput = this.promptContentMap[promptName]; @@ -617,11 +617,11 @@ export class TranscendPromptManager< // Ensure the first message in `promptRunMessages` is of type=system if (options.promptRunMessages.length === 0) { - throw new Error("promptRunMessages is expected to have length > 0"); + throw new Error('promptRunMessages is expected to have length > 0'); } if (options.promptRunMessages[0].role !== ChatCompletionRole.System) { throw new Error( - `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}` + `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}`, ); } if ( @@ -630,12 +630,12 @@ export class TranscendPromptManager< throw new Error( `promptRunMessages[${ options.promptRunMessages.length - 1 - }].role is expected to be = ${ChatCompletionRole.Assistant}` + }].role is expected to be = ${ChatCompletionRole.Assistant}`, ); } const response = options.promptRunMessages.at(-1).content; - let parsed: t.TypeOf; + let parsed: t.TypeOf; try { // Parse the response parsed = this.parseAiResponse(promptName, response); @@ -646,7 +646,7 @@ export class TranscendPromptManager< name, error: error.message, status: QueueStatus.Error, - ...(typeof largeLanguageModel === "string" + ...(typeof largeLanguageModel === 'string' ? { largeLanguageModelId: largeLanguageModel } : { largeLanguageModelName: largeLanguageModel.name, @@ -667,7 +667,7 @@ export class TranscendPromptManager< ...options, name, status: QueueStatus.Resolved, - ...(typeof largeLanguageModel === "string" + ...(typeof largeLanguageModel === 'string' ? { largeLanguageModelId: largeLanguageModel } : { largeLanguageModelName: largeLanguageModel.name, @@ -699,7 +699,7 @@ export class TranscendPromptManager< { largeLanguageModel, ...options - }: Requirize + }: Requirize, ): Promise { // Determine if prompts need to be fetched if ( @@ -719,7 +719,7 @@ export class TranscendPromptManager< `@transcend-io/cli-prompt-run-${new Date().toISOString()}`; if (!this.promptContentMap) { - throw new Error("Expected this.promptContentMap to be defined"); + throw new Error('Expected this.promptContentMap to be defined'); } // Look up prompt info const promptInput = this.promptContentMap[promptName]; @@ -729,11 +729,11 @@ export class TranscendPromptManager< // Ensure the first message in `promptRunMessages` is of type=system if (options.promptRunMessages.length === 0) { - throw new Error("promptRunMessages is expected to have length > 0"); + throw new Error('promptRunMessages is expected to have length > 0'); } if (options.promptRunMessages[0].role !== ChatCompletionRole.System) { throw new Error( - `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}` + `promptRunMessages[0].role is expected to be = ${ChatCompletionRole.System}`, ); } @@ -742,7 +742,7 @@ export class TranscendPromptManager< ...options, name, status: QueueStatus.Error, - ...(typeof largeLanguageModel === "string" + ...(typeof largeLanguageModel === 'string' ? { largeLanguageModelId: largeLanguageModel } : { largeLanguageModelName: largeLanguageModel.name, diff --git a/src/lib/ai/filterNullishValuesFromObject.ts b/src/lib/ai/filterNullishValuesFromObject.ts index ea36f39c..707ba966 100644 --- a/src/lib/ai/filterNullishValuesFromObject.ts +++ b/src/lib/ai/filterNullishValuesFromObject.ts @@ -1,4 +1,4 @@ -import { ObjByString } from "@transcend-io/type-utils"; +import { ObjByString } from '@transcend-io/type-utils'; /** * Given an object, remove all keys that are null-ish @@ -7,17 +7,17 @@ import { ObjByString } from "@transcend-io/type-utils"; * @returns Object with null-ish values removed */ export function filterNullishValuesFromObject( - object: T + object: T, ): T { return Object.entries(object).reduce( (accumulator, [k, v]) => v !== null && v !== undefined && - v !== "" && + v !== '' && !(Array.isArray(v) && v.length === 0) && - !(typeof v === "object" && Object.keys(v).length === 0) + !(typeof v === 'object' && Object.keys(v).length === 0) ? Object.assign(accumulator, { [k]: v }) : accumulator, - {} as T + {} as T, ); } diff --git a/src/lib/ai/getGitFilesThatChanged.ts b/src/lib/ai/getGitFilesThatChanged.ts index af27c02b..7ccd2034 100644 --- a/src/lib/ai/getGitFilesThatChanged.ts +++ b/src/lib/ai/getGitFilesThatChanged.ts @@ -1,6 +1,6 @@ -import { execSync } from "node:child_process"; -import fastGlob from "fast-glob"; -import { difference } from "lodash-es"; +import { execSync } from 'node:child_process'; +import fastGlob from 'fast-glob'; +import { difference } from 'lodash-es'; /** * Function thats gets the git files that have changed @@ -42,17 +42,17 @@ export function getGitFilesThatChanged({ // Latest commit on base branch. If we are on the base branch, we take the prior commit const latestBasedCommit = execSync( `git ls-remote ${githubRepo} "refs/heads/${baseBranch}" | cut -f 1`, - { encoding: "utf-8" } - ).split("\n")[0]; + { encoding: 'utf-8' }, + ).split('\n')[0]; // This commit - const latestThisCommit = execSync("git rev-parse HEAD", { - encoding: "utf-8", - }).split("\n")[0]; + const latestThisCommit = execSync('git rev-parse HEAD', { + encoding: 'utf-8', + }).split('\n')[0]; // Ensure commits are present if (!latestBasedCommit || !latestThisCommit) { - throw new Error("FAILED TO FIND COMMIT RANGE"); + throw new Error('FAILED TO FIND COMMIT RANGE'); } // Get the diff between the given branch and base branch @@ -60,13 +60,13 @@ export function getGitFilesThatChanged({ `git fetch && git diff --name-only "${ baseBranch || latestBasedCommit }...${latestThisCommit}" -- ${rootDirectory}`, - { encoding: "utf-8" } + { encoding: 'utf-8' }, ); // Filter out block list const changedFiles = difference( - diff.split("\n").filter(Boolean), - fileBlockList + diff.split('\n').filter(Boolean), + fileBlockList, ); // Filter out globs @@ -79,13 +79,13 @@ export function getGitFilesThatChanged({ const fileDiffs: Record = {}; for (const file of filteredChanges) { const contents = execSync(`git show ${latestThisCommit}:${file}`, { - encoding: "utf-8", + encoding: 'utf-8', }); fileDiffs[file] = contents; } // Pull the github repo name - const repoName = githubRepo.split("/").pop()!.split(".")[0]; + const repoName = githubRepo.split('/').pop()!.split('.')[0]; return { changedFiles, diff --git a/src/lib/ai/removeLinks.ts b/src/lib/ai/removeLinks.ts index dcbe2bee..83424a77 100644 --- a/src/lib/ai/removeLinks.ts +++ b/src/lib/ai/removeLinks.ts @@ -6,5 +6,5 @@ */ export function removeLinks(inputString: string): string { const regex = /(https?:\/\/[^\s]+)/g; - return inputString.replaceAll(regex, ""); + return inputString.replaceAll(regex, ''); } diff --git a/src/lib/api-keys/generateCrossAccountApiKeys.ts b/src/lib/api-keys/generateCrossAccountApiKeys.ts index d189b540..7a357386 100644 --- a/src/lib/api-keys/generateCrossAccountApiKeys.ts +++ b/src/lib/api-keys/generateCrossAccountApiKeys.ts @@ -1,9 +1,9 @@ -import { ScopeName } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { StoredApiKey } from "../../codecs"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import { ScopeName } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { StoredApiKey } from '../../codecs'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { assumeRole, buildTranscendGraphQLClientGeneric, @@ -11,7 +11,7 @@ import { deleteApiKey, fetchAllApiKeys, loginUser, -} from "../graphql"; +} from '../graphql'; export interface ApiKeyGenerateError { /** Name of instance */ @@ -64,14 +64,14 @@ export async function generateCrossAccountApiKeys({ const client = await buildTranscendGraphQLClientGeneric(transcendUrl, {}); // Login the user - logger.info(colors.magenta("Logging in using email and password.")); + logger.info(colors.magenta('Logging in using email and password.')); const { roles, loginCookie } = await loginUser(client, { email, password }); logger.info( colors.green( `Successfully logged in and found ${roles.length} role${ - roles.length === 1 ? "" : "s" - }!` - ) + roles.length === 1 ? '' : 's' + }!`, + ), ); // Filter down by parentOrganizationId @@ -79,7 +79,7 @@ export async function generateCrossAccountApiKeys({ ? roles.filter( (role) => role.organization.id === parentOrganizationId || - role.organization.parentOrganizationId === parentOrganizationId + role.organization.parentOrganizationId === parentOrganizationId, ) : roles; @@ -96,9 +96,9 @@ export async function generateCrossAccountApiKeys({ logger.info( colors.magenta( `Generating API keys with title: ${apiKeyTitle}, scopes: ${scopes.join( - "," - )}.` - ) + ',', + )}.`, + ), ); // Map over each role @@ -110,8 +110,8 @@ export async function generateCrossAccountApiKeys({ // Grab API keys with that title logger.info( colors.magenta( - `Checking if API key already exists in organization "${role.organization.name}" with title: "${apiKeyTitle}".` - ) + `Checking if API key already exists in organization "${role.organization.name}" with title: "${apiKeyTitle}".`, + ), ); // Delete existing API key @@ -119,14 +119,14 @@ export async function generateCrossAccountApiKeys({ if (apiKeyWithTitle && deleteExistingApiKey) { logger.info( colors.yellow( - `Deleting existing API key in "${role.organization.name}" with title: "${apiKeyTitle}".` - ) + `Deleting existing API key in "${role.organization.name}" with title: "${apiKeyTitle}".`, + ), ); await deleteApiKey(client, apiKeyWithTitle.id); logger.info( colors.green( - `Successfully deleted API key in "${role.organization.name}" with title: "${apiKeyTitle}".` - ) + `Successfully deleted API key in "${role.organization.name}" with title: "${apiKeyTitle}".`, + ), ); } else if (apiKeyWithTitle) { // throw error if one exists but not configured to delete @@ -137,8 +137,8 @@ export async function generateCrossAccountApiKeys({ if (createNewApiKey) { logger.info( colors.magenta( - `Creating API key in "${role.organization.name}" with title: "${apiKeyTitle}".` - ) + `Creating API key in "${role.organization.name}" with title: "${apiKeyTitle}".`, + ), ); const { apiKey } = await createApiKey(client, { title: apiKeyTitle, @@ -151,22 +151,22 @@ export async function generateCrossAccountApiKeys({ }); logger.info( colors.green( - `Successfully created API key in "${role.organization.name}" with title: "${apiKeyTitle}".` - ) + `Successfully created API key in "${role.organization.name}" with title: "${apiKeyTitle}".`, + ), ); } else { // Delete only results.push({ organizationName: role.organization.name, organizationId: role.organization.id, - apiKey: "", + apiKey: '', }); } } catch (error) { logger.error( colors.red( - `Failed to create API key in organization "${role.organization.name}"! - ${error.message}` - ) + `Failed to create API key in organization "${role.organization.name}"! - ${error.message}`, + ), ); errors.push({ organizationName: role.organization.name, @@ -178,18 +178,18 @@ export async function generateCrossAccountApiKeys({ logger.info( colors.green( `Successfully created ${results.length} API key${ - results.length === 1 ? "" : "s" - }` - ) + results.length === 1 ? '' : 's' + }`, + ), ); if (errors.length > 0) { logger.error( colors.red( `Failed to create ${errors.length} API key${ - errors.length === 1 ? "" : "s" - }!` - ) + errors.length === 1 ? '' : 's' + }!`, + ), ); } diff --git a/src/lib/api-keys/listDirectories.ts b/src/lib/api-keys/listDirectories.ts index 72d69c8a..fb9a29e0 100644 --- a/src/lib/api-keys/listDirectories.ts +++ b/src/lib/api-keys/listDirectories.ts @@ -1,5 +1,5 @@ -import { readdirSync, statSync } from "node:fs"; -import { join } from "node:path"; +import { readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; /** * List the folders in a directory @@ -9,6 +9,6 @@ import { join } from "node:path"; */ export function listDirectories(startDir: string): string[] { return readdirSync(startDir).filter((entryName) => - statSync(join(startDir, entryName)).isDirectory() + statSync(join(startDir, entryName)).isDirectory(), ); } diff --git a/src/lib/api-keys/listFiles.ts b/src/lib/api-keys/listFiles.ts index b3000f77..9c696e91 100644 --- a/src/lib/api-keys/listFiles.ts +++ b/src/lib/api-keys/listFiles.ts @@ -1,4 +1,4 @@ -import { existsSync, readdirSync } from "node:fs"; +import { existsSync, readdirSync } from 'node:fs'; /** * List the files in a directory @@ -18,7 +18,7 @@ import { existsSync, readdirSync } from "node:fs"; export function listFiles( directory: string, validExtensions?: string[], - removeExtensions = false + removeExtensions = false, ): string[] { if (!existsSync(directory)) { return []; @@ -28,11 +28,11 @@ export function listFiles( .filter((fil) => validExtensions ? validExtensions.filter((extension) => fil.endsWith(extension)).length - : true + : true, ) - .filter((fil) => fil.indexOf(".") > 0); + .filter((fil) => fil.indexOf('.') > 0); return removeExtensions - ? files.map((fil) => fil.replace(/\.[^/.]+$/, "")) + ? files.map((fil) => fil.replace(/\.[^/.]+$/, '')) : files; } diff --git a/src/lib/api-keys/validateTranscendAuth.ts b/src/lib/api-keys/validateTranscendAuth.ts index 45761e1e..7d69ba4a 100644 --- a/src/lib/api-keys/validateTranscendAuth.ts +++ b/src/lib/api-keys/validateTranscendAuth.ts @@ -1,9 +1,9 @@ -import { existsSync, readFileSync } from "node:fs"; -import { decodeCodec } from "@transcend-io/type-utils"; -import colors from "colors"; -import * as t from "io-ts"; -import { StoredApiKey } from "../../codecs"; -import { logger } from "../../logger"; +import { existsSync, readFileSync } from 'node:fs'; +import { decodeCodec } from '@transcend-io/type-utils'; +import colors from 'colors'; +import * as t from 'io-ts'; +import { StoredApiKey } from '../../codecs'; +import { logger } from '../../logger'; /** * Determine if the `--auth` parameter is an API key or a path to a JSON @@ -17,8 +17,8 @@ export function validateTranscendAuth(auth: string): string | StoredApiKey[] { if (!auth) { logger.error( colors.red( - "A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY" - ) + 'A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY', + ), ); process.exit(1); } @@ -26,7 +26,7 @@ export function validateTranscendAuth(auth: string): string | StoredApiKey[] { // Read from disk if (existsSync(auth)) { // validate that file is a list of API keys - return decodeCodec(t.array(StoredApiKey), readFileSync(auth, "utf-8")); + return decodeCodec(t.array(StoredApiKey), readFileSync(auth, 'utf-8')); } // Return as single API key diff --git a/src/lib/bluebird-replace.ts b/src/lib/bluebird-replace.ts index 46c2fa2e..25e44138 100644 --- a/src/lib/bluebird-replace.ts +++ b/src/lib/bluebird-replace.ts @@ -7,7 +7,7 @@ */ export async function mapSeries( array: R[], - iterator: (item: R, index: number, arrayLength: number) => Promise + iterator: (item: R, index: number, arrayLength: number) => Promise, ): Promise { const results = []; for (let index = 0; index < array.length; index += 1) { @@ -30,7 +30,7 @@ export async function map( options: { /** Concurrency level for the Promise.all call */ concurrency?: number; - } = {} + } = {}, ): Promise { const { concurrency = Infinity } = options; const results: U[] = Array.from({ length: array.length }); @@ -46,7 +46,7 @@ export async function map( const promise = iterator( array[currentIndex], currentIndex, - array.length + array.length, ).then((result) => { results[currentIndex] = result; }); diff --git a/src/lib/code-scanning/constants.ts b/src/lib/code-scanning/constants.ts index 092aee78..889bfc1f 100644 --- a/src/lib/code-scanning/constants.ts +++ b/src/lib/code-scanning/constants.ts @@ -1,4 +1,4 @@ -import { CodePackageType } from "@transcend-io/privacy-types"; +import { CodePackageType } from '@transcend-io/privacy-types'; import { cocoaPods, composerJson, @@ -8,8 +8,8 @@ import { pubspec, pythonRequirementsTxt, swift, -} from "./integrations"; -import { CodeScanningConfig } from "./types"; +} from './integrations'; +import { CodeScanningConfig } from './types'; /** * @deprecated TODO: https://transcend.height.app/T-32325 - use code scanning instead diff --git a/src/lib/code-scanning/findCodePackagesInFolder.ts b/src/lib/code-scanning/findCodePackagesInFolder.ts index 9abd8f34..d4bec9cc 100644 --- a/src/lib/code-scanning/findCodePackagesInFolder.ts +++ b/src/lib/code-scanning/findCodePackagesInFolder.ts @@ -1,9 +1,9 @@ -import { getEntries } from "@transcend-io/type-utils"; -import colors from "colors"; -import fastGlob from "fast-glob"; -import { CodePackageInput } from "../../codecs"; -import { logger } from "../../logger"; -import { CODE_SCANNING_CONFIGS } from "./constants"; +import { getEntries } from '@transcend-io/type-utils'; +import colors from 'colors'; +import fastGlob from 'fast-glob'; +import { CodePackageInput } from '../../codecs'; +import { logger } from '../../logger'; +import { CODE_SCANNING_CONFIGS } from './constants'; /** * Helper to scan and discovery all of the code packages within a folder @@ -36,34 +36,34 @@ export async function findCodePackagesInFolder({ ].filter((dir) => dir.length > 0); try { const filesToScan: string[] = await fastGlob( - `${scanPath}/**/${supportedFiles.join("|")}`, + `${scanPath}/**/${supportedFiles.join('|')}`, { ignore: directoriesToIgnore.map( - (dir: string) => `${scanPath}/**/${dir}` + (dir: string) => `${scanPath}/**/${dir}`, ), unique: true, onlyFiles: true, - } + }, ); logger.info( colors.magenta( - `Scanning: ${filesToScan.length} files of type ${codePackageType}` - ) + `Scanning: ${filesToScan.length} files of type ${codePackageType}`, + ), ); const allPackages = filesToScan.flatMap((filePath) => scanFunction(filePath).map((result) => ({ ...result, - relativePath: filePath.replace(`${scanPath}/`, ""), - })) + relativePath: filePath.replace(`${scanPath}/`, ''), + })), ); logger.info( colors.green( `Found: ${allPackages.length} packages and ${ allPackages.flatMap( - ({ softwareDevelopmentKits = [] }) => softwareDevelopmentKits + ({ softwareDevelopmentKits = [] }) => softwareDevelopmentKits, ).length - } sdks` - ) + } sdks`, + ), ); return allPackages.map( @@ -71,14 +71,14 @@ export async function findCodePackagesInFolder({ ...package_, type: codePackageType, repositoryName, - }) + }), ); } catch (error) { throw new Error( - `Error scanning globs ${supportedFiles} with error: ${error}` + `Error scanning globs ${supportedFiles} with error: ${error}`, ); } - }) + }), ); return allCodePackages.flat(); diff --git a/src/lib/code-scanning/findFilesToScan.ts b/src/lib/code-scanning/findFilesToScan.ts index 4ffe20d4..f6535395 100644 --- a/src/lib/code-scanning/findFilesToScan.ts +++ b/src/lib/code-scanning/findFilesToScan.ts @@ -1,6 +1,6 @@ -import fastGlob from "fast-glob"; -import { logger } from "../../logger"; -import { CodeScanningConfig } from "./types"; +import fastGlob from 'fast-glob'; +import { logger } from '../../logger'; +import { CodeScanningConfig } from './types'; export interface SiloDiscoveryRawResults { /** The name of the potential data silo entry */ @@ -37,29 +37,29 @@ export async function findFilesToScan({ }): Promise { const { ignoreDirs: IGNORE_DIRS, supportedFiles, scanFunction } = config; const globsToSupport = - fileGlobs === "" + fileGlobs === '' ? supportedFiles - : supportedFiles.concat(fileGlobs.split(",")); - const directoriesToIgnore = [...ignoreDirs.split(","), ...IGNORE_DIRS].filter( - (dir) => dir.length > 0 + : supportedFiles.concat(fileGlobs.split(',')); + const directoriesToIgnore = [...ignoreDirs.split(','), ...IGNORE_DIRS].filter( + (dir) => dir.length > 0, ); try { const filesToScan: string[] = await fastGlob( - `${scanPath}/**/${globsToSupport.join("|")}`, + `${scanPath}/**/${globsToSupport.join('|')}`, { ignore: directoriesToIgnore.map( - (dir: string) => `${scanPath}/**/${dir}` + (dir: string) => `${scanPath}/**/${dir}`, ), unique: true, onlyFiles: true, - } + }, ); logger.info(`Scanning: ${filesToScan.length} files`); const allPackages = filesToScan.flatMap((filePath: string) => - scanFunction(filePath) + scanFunction(filePath), ); const allSdks = allPackages.flatMap( - (appPackage) => appPackage.softwareDevelopmentKits || [] + (appPackage) => appPackage.softwareDevelopmentKits || [], ); const uniqueDeps = new Set(allSdks.map((sdk) => sdk.name)); const deps = [...uniqueDeps]; @@ -71,7 +71,7 @@ export async function findFilesToScan({ })); } catch (error) { throw new Error( - `Error scanning globs ${findFilesToScan} with error: ${error}` + `Error scanning globs ${findFilesToScan} with error: ${error}`, ); } } diff --git a/src/lib/code-scanning/integrations/cocoaPods.ts b/src/lib/code-scanning/integrations/cocoaPods.ts index 9a2d57c9..30dfebc5 100644 --- a/src/lib/code-scanning/integrations/cocoaPods.ts +++ b/src/lib/code-scanning/integrations/cocoaPods.ts @@ -1,39 +1,39 @@ -import { readFileSync } from "node:fs"; -import { CodePackageType } from "@transcend-io/privacy-types"; -import { findAllWithRegex } from "@transcend-io/type-utils"; -import { CodePackageSdk } from "../../../codecs"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { CodePackageType } from '@transcend-io/privacy-types'; +import { findAllWithRegex } from '@transcend-io/type-utils'; +import { CodePackageSdk } from '../../../codecs'; +import { CodeScanningConfig } from '../types'; const POD_TARGET_REGEX = /target ('|")(.*?)('|")/; const POD_PACKAGE_REGEX = /pod ('|")(.*?)('|")(, ('|")~> (.+?)('|")|)/; export const cocoaPods: CodeScanningConfig = { - supportedFiles: ["Podfile"], - ignoreDirs: ["Pods"], + supportedFiles: ['Podfile'], + ignoreDirs: ['Pods'], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); const targets = findAllWithRegex( { - value: new RegExp(POD_TARGET_REGEX, "g"), - matches: ["quote1", "name", "quote2"], + value: new RegExp(POD_TARGET_REGEX, 'g'), + matches: ['quote1', 'name', 'quote2'], }, - fileContents + fileContents, ); const packages = findAllWithRegex( { - value: new RegExp(POD_PACKAGE_REGEX, "g"), + value: new RegExp(POD_PACKAGE_REGEX, 'g'), matches: [ - "quote1", - "name", - "quote2", - "extra", - "quote3", - "version", - "quote4", + 'quote1', + 'name', + 'quote2', + 'extra', + 'quote3', + 'version', + 'quote4', ], }, - fileContents + fileContents, ); const deps: CodePackageSdk[] = targets.map((target, ind) => ({ @@ -44,7 +44,7 @@ export const cocoaPods: CodeScanningConfig = { (package_) => package_.matchIndex > target.matchIndex && (!targets[ind + 1] || - package_.matchIndex < targets[ind + 1].matchIndex) + package_.matchIndex < targets[ind + 1].matchIndex), ) .map((package_) => ({ name: package_.name, diff --git a/src/lib/code-scanning/integrations/composerJson.ts b/src/lib/code-scanning/integrations/composerJson.ts index 96eb782f..3cf697c5 100644 --- a/src/lib/code-scanning/integrations/composerJson.ts +++ b/src/lib/code-scanning/integrations/composerJson.ts @@ -1,39 +1,39 @@ -import { readFileSync } from "node:fs"; -import { dirname } from "node:path"; -import { CodePackageSdk } from "../../../codecs"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { CodePackageSdk } from '../../../codecs'; +import { CodeScanningConfig } from '../types'; export const composerJson: CodeScanningConfig = { - supportedFiles: ["composer.json"], - ignoreDirs: ["vendor", "node_modules", "cache", "build", "dist"], + supportedFiles: ['composer.json'], + ignoreDirs: ['vendor', 'node_modules', 'cache', 'build', 'dist'], scanFunction: (filePath) => { - const file = readFileSync(filePath, "utf-8"); + const file = readFileSync(filePath, 'utf-8'); const directory = dirname(filePath); const asJson = JSON.parse(file); const { name, description, require: requireDependencies = {}, - "require-dev": requiredDevelopmentDependencies = {}, + 'require-dev': requiredDevelopmentDependencies = {}, } = asJson; return [ { // name of the package - name: name || directory.split("/").pop()!, + name: name || directory.split('/').pop()!, description, softwareDevelopmentKits: [ ...Object.entries(requireDependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === "string" ? version : undefined, - }) + version: typeof version === 'string' ? version : undefined, + }), ), ...Object.entries(requiredDevelopmentDependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === "string" ? version : undefined, + version: typeof version === 'string' ? version : undefined, isDevDependency: true, - }) + }), ), ], }, diff --git a/src/lib/code-scanning/integrations/gemfile.ts b/src/lib/code-scanning/integrations/gemfile.ts index 1d16ce8d..28b631df 100644 --- a/src/lib/code-scanning/integrations/gemfile.ts +++ b/src/lib/code-scanning/integrations/gemfile.ts @@ -1,9 +1,9 @@ -import { readFileSync } from "node:fs"; -import { dirname } from "node:path"; -import { CodePackageType } from "@transcend-io/privacy-types"; -import { findAllWithRegex } from "@transcend-io/type-utils"; -import { listFiles } from "../../api-keys"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { CodePackageType } from '@transcend-io/privacy-types'; +import { findAllWithRegex } from '@transcend-io/type-utils'; +import { listFiles } from '../../api-keys'; +import { CodeScanningConfig } from '../types'; const GEM_PACKAGE_REGEX = /gem *('|")(.+?)('|")(, *('|")(.+?)('|")|)/; const GEMFILE_PACKAGE_NAME_REGEX = /spec\.name *= *('|")(.+?)('|")/; @@ -12,17 +12,17 @@ const GEMFILE_PACKAGE_DESCRIPTION_REGEX = const GEMFILE_PACKAGE_SUMMARY_REGEX = /spec\.summary *= *('|")(.+?)('|")/; export const gemfile: CodeScanningConfig = { - supportedFiles: ["Gemfile"], - ignoreDirs: ["bin"], + supportedFiles: ['Gemfile'], + ignoreDirs: ['bin'], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); const directory = dirname(filePath); const filesInFolder = listFiles(directory); // parse gemspec file for name - const gemspec = filesInFolder.find((file) => file === ".gemspec"); + const gemspec = filesInFolder.find((file) => file === '.gemspec'); const gemspecContents = gemspec - ? readFileSync(gemspec, "utf-8") + ? readFileSync(gemspec, 'utf-8') : undefined; const gemfileName = gemspecContents ? (GEMFILE_PACKAGE_NAME_REGEX.exec(gemspecContents) || [])[2] @@ -35,23 +35,23 @@ export const gemfile: CodeScanningConfig = { const targets = findAllWithRegex( { - value: new RegExp(GEM_PACKAGE_REGEX, "g"), + value: new RegExp(GEM_PACKAGE_REGEX, 'g'), matches: [ - "quote1", - "name", - "quote2", - "hasVersion", - "quote3", - "version", - "quote4", + 'quote1', + 'name', + 'quote2', + 'hasVersion', + 'quote3', + 'version', + 'quote4', ], }, - fileContents + fileContents, ); return [ { - name: gemfileName || directory.split("/").pop()!, + name: gemfileName || directory.split('/').pop()!, description: gemfileDescription || undefined, type: CodePackageType.RequirementsTxt, softwareDevelopmentKits: targets.map((package_) => ({ diff --git a/src/lib/code-scanning/integrations/gradle.ts b/src/lib/code-scanning/integrations/gradle.ts index aa871cee..70983adf 100644 --- a/src/lib/code-scanning/integrations/gradle.ts +++ b/src/lib/code-scanning/integrations/gradle.ts @@ -1,7 +1,7 @@ -import { readFileSync } from "node:fs"; -import { dirname } from "node:path"; -import { findAllWithRegex } from "@transcend-io/type-utils"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { findAllWithRegex } from '@transcend-io/type-utils'; +import { CodeScanningConfig } from '../types'; const GRADLE_IMPLEMENTATION_REGEX = /implementation( *)('|")(.+?):(.+?):(.+?|)('|")/; @@ -21,58 +21,58 @@ const GRADLE_APPLICATION_NAME_REGEX = /applicationId( *)"(.+?)"/; * single and double quotes are both recognized */ export const gradle: CodeScanningConfig = { - supportedFiles: ["build.gradle**"], + supportedFiles: ['build.gradle**'], ignoreDirs: [ - "gradle-app.setting", - "gradle-wrapper.jar", - "gradle-wrapper.properties", + 'gradle-app.setting', + 'gradle-wrapper.jar', + 'gradle-wrapper.properties', ], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); const directory = dirname(filePath); const targets = findAllWithRegex( { - value: new RegExp(GRADLE_IMPLEMENTATION_REGEX, "g"), - matches: ["space", "quote1", "name", "path", "version", "quote2"], + value: new RegExp(GRADLE_IMPLEMENTATION_REGEX, 'g'), + matches: ['space', 'quote1', 'name', 'path', 'version', 'quote2'], }, - fileContents + fileContents, ); const targetPlugins = findAllWithRegex( { - value: new RegExp(GRADLE_PLUGIN_REGEX, "g"), - matches: ["quote1", "name", "group", "version", "quote2"], + value: new RegExp(GRADLE_PLUGIN_REGEX, 'g'), + matches: ['quote1', 'name', 'group', 'version', 'quote2'], }, - fileContents + fileContents, ); const targetGroups = findAllWithRegex( { - value: new RegExp(GRADLE_IMPLEMENTATION_GROUP_REGEX, "g"), + value: new RegExp(GRADLE_IMPLEMENTATION_GROUP_REGEX, 'g'), matches: [ - "space1", - "quote1", - "group", - "quote2", - "space2", - "space3", - "quote3", - "name", - "quote4", - "space4", - "space5", - "quote5", - "version", - "quote6", + 'space1', + 'quote1', + 'group', + 'quote2', + 'space2', + 'space3', + 'quote3', + 'name', + 'quote4', + 'space4', + 'space5', + 'quote5', + 'version', + 'quote6', ], }, - fileContents + fileContents, ); const applications = findAllWithRegex( { - value: new RegExp(GRADLE_APPLICATION_NAME_REGEX, "g"), - matches: ["space", "name"], + value: new RegExp(GRADLE_APPLICATION_NAME_REGEX, 'g'), + matches: ['space', 'name'], }, - fileContents + fileContents, ); if (applications.length > 1) { throw new Error(`Expected only one applicationId per file: ${filePath}`); @@ -80,7 +80,7 @@ export const gradle: CodeScanningConfig = { return [ { - name: applications[0]?.name || directory.split("/").pop()!, + name: applications[0]?.name || directory.split('/').pop()!, softwareDevelopmentKits: [ ...targets, ...targetGroups, diff --git a/src/lib/code-scanning/integrations/javascriptPackageJson.ts b/src/lib/code-scanning/integrations/javascriptPackageJson.ts index b7464e3c..fe437c32 100644 --- a/src/lib/code-scanning/integrations/javascriptPackageJson.ts +++ b/src/lib/code-scanning/integrations/javascriptPackageJson.ts @@ -1,13 +1,13 @@ -import { readFileSync } from "node:fs"; -import { dirname } from "node:path"; -import { CodePackageSdk } from "../../../codecs"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { CodePackageSdk } from '../../../codecs'; +import { CodeScanningConfig } from '../types'; export const javascriptPackageJson: CodeScanningConfig = { - supportedFiles: ["package.json"], - ignoreDirs: ["node_modules", "serverless-build", "lambda-build"], + supportedFiles: ['package.json'], + ignoreDirs: ['node_modules', 'serverless-build', 'lambda-build'], scanFunction: (filePath) => { - const file = readFileSync(filePath, "utf-8"); + const file = readFileSync(filePath, 'utf-8'); const directory = dirname(filePath); const asJson = JSON.parse(file); const { @@ -20,27 +20,27 @@ export const javascriptPackageJson: CodeScanningConfig = { return [ { // name of the package - name: name || directory.split("/").pop()!, + name: name || directory.split('/').pop()!, description, softwareDevelopmentKits: [ ...Object.entries(dependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === "string" ? version : undefined, - }) + version: typeof version === 'string' ? version : undefined, + }), ), ...Object.entries(devDependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === "string" ? version : undefined, + version: typeof version === 'string' ? version : undefined, isDevDependency: true, - }) + }), ), ...Object.entries(optionalDependencies).map( ([name, version]): CodePackageSdk => ({ name, - version: typeof version === "string" ? version : undefined, - }) + version: typeof version === 'string' ? version : undefined, + }), ), ], }, diff --git a/src/lib/code-scanning/integrations/pubspec.ts b/src/lib/code-scanning/integrations/pubspec.ts index d2edf734..0f8e1b72 100644 --- a/src/lib/code-scanning/integrations/pubspec.ts +++ b/src/lib/code-scanning/integrations/pubspec.ts @@ -1,8 +1,8 @@ -import { readFileSync } from "node:fs"; -import { dirname } from "node:path"; -import { CodePackageType } from "@transcend-io/privacy-types"; -import yaml from "js-yaml"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { CodePackageType } from '@transcend-io/privacy-types'; +import yaml from 'js-yaml'; +import { CodeScanningConfig } from '../types'; /** * Remove YAML comments from a string @@ -12,10 +12,10 @@ import { CodeScanningConfig } from "../types"; */ function removeYAMLComments(yamlString: string): string { return yamlString - .split("\n") + .split('\n') .map((line) => { // Remove inline comments - const commentIndex = line.indexOf("#"); + const commentIndex = line.indexOf('#'); if ( commentIndex !== -1 && // Check if '#' is not inside a string !line.slice(0, Math.max(0, commentIndex)).includes('"') && @@ -26,15 +26,15 @@ function removeYAMLComments(yamlString: string): string { return line; }) .filter((line) => line.length > 0) - .join("\n"); + .join('\n'); } export const pubspec: CodeScanningConfig = { - supportedFiles: ["pubspec.yml"], - ignoreDirs: ["build"], + supportedFiles: ['pubspec.yml'], + ignoreDirs: ['build'], scanFunction: (filePath) => { const directory = dirname(filePath); - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); const { name, description, @@ -52,30 +52,30 @@ export const pubspec: CodeScanningConfig = { }; return [ { - name: name || directory.split("/").pop()!, + name: name || directory.split('/').pop()!, description, type: CodePackageType.RequirementsTxt, softwareDevelopmentKits: [ ...Object.entries(dependencies).map(([name, version]) => ({ name, version: - typeof version === "string" + typeof version === 'string' ? version - : typeof version === "number" - ? version.toString() - : version?.sdk, + : typeof version === 'number' + ? version.toString() + : version?.sdk, })), ...Object.entries(development_dependencies).map( ([name, version]) => ({ name, version: - typeof version === "string" + typeof version === 'string' ? version - : typeof version === "number" - ? version.toString() - : version?.sdk, + : typeof version === 'number' + ? version.toString() + : version?.sdk, isDevDependency: true, - }) + }), ), ], }, diff --git a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts index 97659eab..fa119fe9 100644 --- a/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts +++ b/src/lib/code-scanning/integrations/pythonRequirementsTxt.ts @@ -1,26 +1,26 @@ -import { readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { CodePackageType } from "@transcend-io/privacy-types"; -import { findAllWithRegex } from "@transcend-io/type-utils"; -import { listFiles } from "../../api-keys"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { CodePackageType } from '@transcend-io/privacy-types'; +import { findAllWithRegex } from '@transcend-io/type-utils'; +import { listFiles } from '../../api-keys'; +import { CodeScanningConfig } from '../types'; const REQUIREMENTS_PACKAGE_MATCH = /(.+?)(=+)(.+)/; const PACKAGE_NAME = /name *= *('|")(.+?)('|")/; const PACKAGE_DESCRIPTION = /description *= *('|")(.+?)('|")/; export const pythonRequirementsTxt: CodeScanningConfig = { - supportedFiles: ["requirements.txt"], - ignoreDirs: ["build", "lib", "lib64"], + supportedFiles: ['requirements.txt'], + ignoreDirs: ['build', 'lib', 'lib64'], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); const directory = dirname(filePath); const filesInFolder = listFiles(directory); // parse setup file for name - const setupFile = filesInFolder.find((file) => file === "setup.py"); + const setupFile = filesInFolder.find((file) => file === 'setup.py'); const setupFileContents = setupFile - ? readFileSync(join(directory, setupFile), "utf-8") + ? readFileSync(join(directory, setupFile), 'utf-8') : undefined; const packageName = setupFileContents ? (PACKAGE_NAME.exec(setupFileContents) || [])[2] @@ -31,15 +31,15 @@ export const pythonRequirementsTxt: CodeScanningConfig = { const targets = findAllWithRegex( { - value: new RegExp(REQUIREMENTS_PACKAGE_MATCH, "g"), - matches: ["name", "equals", "version"], + value: new RegExp(REQUIREMENTS_PACKAGE_MATCH, 'g'), + matches: ['name', 'equals', 'version'], }, - fileContents + fileContents, ); return [ { - name: packageName || directory.split("/").pop()!, + name: packageName || directory.split('/').pop()!, description: packageDescription || undefined, type: CodePackageType.RequirementsTxt, softwareDevelopmentKits: targets.map((package_) => ({ diff --git a/src/lib/code-scanning/integrations/swift.ts b/src/lib/code-scanning/integrations/swift.ts index 410e75bc..df1f1804 100644 --- a/src/lib/code-scanning/integrations/swift.ts +++ b/src/lib/code-scanning/integrations/swift.ts @@ -1,9 +1,9 @@ -import { readFileSync } from "node:fs"; -import { dirname } from "node:path"; -import { CodePackageType } from "@transcend-io/privacy-types"; -import { decodeCodec } from "@transcend-io/type-utils"; -import * as t from "io-ts"; -import { CodeScanningConfig } from "../types"; +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { CodePackageType } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import * as t from 'io-ts'; +import { CodeScanningConfig } from '../types'; const SwiftPackage = t.type({ pins: t.array( @@ -15,22 +15,22 @@ const SwiftPackage = t.type({ revision: t.string, version: t.string, }), - }) + }), ), version: t.number, }); export const swift: CodeScanningConfig = { - supportedFiles: ["Package.resolved"], + supportedFiles: ['Package.resolved'], ignoreDirs: [], scanFunction: (filePath) => { - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); const parsed = decodeCodec(SwiftPackage, fileContents); return [ { - name: dirname(filePath).split("/").pop() || "", // FIXME pull from Package.swift ->> name if possible + name: dirname(filePath).split('/').pop() || '', // FIXME pull from Package.swift ->> name if possible type: CodePackageType.CocoaPods, // FIXME should be swift softwareDevelopmentKits: parsed.pins.map((target) => ({ name: target.identity, diff --git a/src/lib/consent-manager/buildXdiSyncEndpoint.ts b/src/lib/consent-manager/buildXdiSyncEndpoint.ts index af8dcdaa..756bcd2d 100644 --- a/src/lib/consent-manager/buildXdiSyncEndpoint.ts +++ b/src/lib/consent-manager/buildXdiSyncEndpoint.ts @@ -1,11 +1,11 @@ -import colors from "colors"; -import { difference } from "lodash-es"; -import { StoredApiKey } from "../../codecs"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { buildTranscendGraphQLClient, fetchConsentManager } from "../graphql"; -import { domainToHost } from "./domainToHost"; +import colors from 'colors'; +import { difference } from 'lodash-es'; +import { StoredApiKey } from '../../codecs'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { buildTranscendGraphQLClient, fetchConsentManager } from '../graphql'; +import { domainToHost } from './domainToHost'; /** * Sync group configuration mapping @@ -38,8 +38,8 @@ export async function buildXdiSyncEndpoint( xdiLocation, transcendUrl = DEFAULT_TRANSCEND_API, removeIpAddresses = true, - domainBlockList = ["localhost"], - xdiAllowedCommands = "ConsentManager:Sync", + domainBlockList = ['localhost'], + xdiAllowedCommands = 'ConsentManager:Sync', }: { /** The file location where the XDI file is hosted */ xdiLocation: string; @@ -51,7 +51,7 @@ export async function buildXdiSyncEndpoint( domainBlockList?: string[]; /** Allows XDI commands */ xdiAllowedCommands?: string; - } + }, ): Promise<{ /** Sync group configurations */ syncGroups: XdiSyncGroups; @@ -61,7 +61,7 @@ export async function buildXdiSyncEndpoint( // Convert API keys to list const apiKeysAsList = Array.isArray(apiKeys) ? apiKeys - : [{ apiKey: apiKeys, organizationId: "", organizationName: "" }]; + : [{ apiKey: apiKeys, organizationId: '', organizationName: '' }]; // Fetch configuration for each account const consentManagers = await map( @@ -69,8 +69,8 @@ export async function buildXdiSyncEndpoint( async (apiKey) => { logger.info( colors.magenta( - `Pulling consent metadata for organization - ${apiKey.organizationName}` - ) + `Pulling consent metadata for organization - ${apiKey.organizationName}`, + ), ); // Create a GraphQL client @@ -80,7 +80,7 @@ export async function buildXdiSyncEndpoint( const consentManager = await fetchConsentManager(client); return consentManager; }, - { concurrency: 5 } + { concurrency: 5 }, ); // construct the sync groups @@ -91,7 +91,7 @@ export async function buildXdiSyncEndpoint( // take explicit key first consentManager.partition?.partition || // fallback to bundle ID - consentManager.bundleURL.split("/").reverse()[1]; + consentManager.bundleURL.split('/').reverse()[1]; // Ensure that partition exists in the sync groups if (!syncGroups[partitionKey]) { @@ -103,11 +103,11 @@ export async function buildXdiSyncEndpoint( consentManager.configuration.domains .filter( // ignore IP addresses - (domain) => !removeIpAddresses || !IP_ADDRESS_REGEX.test(domain) + (domain) => !removeIpAddresses || !IP_ADDRESS_REGEX.test(domain), ) .map((domain) => domainToHost(domain)), // ignore block list - domainBlockList + domainBlockList, ); // merge existing sync group with hosts for this consent manager syncGroups[partitionKey] = [ diff --git a/src/lib/consent-manager/consentManagersToBusinessEntities.ts b/src/lib/consent-manager/consentManagersToBusinessEntities.ts index 1007b33e..76bb356e 100644 --- a/src/lib/consent-manager/consentManagersToBusinessEntities.ts +++ b/src/lib/consent-manager/consentManagersToBusinessEntities.ts @@ -1,5 +1,5 @@ -import { BusinessEntityInput, ConsentManagerInput } from "../../codecs"; -import { logger } from "../../logger"; +import { BusinessEntityInput, ConsentManagerInput } from '../../codecs'; +import { logger } from '../../logger'; /** * Combine multiple consent manager configurations into a list of business entity configurations @@ -13,19 +13,19 @@ export function consentManagersToBusinessEntities( name: string; /** Consent manager input */ input?: ConsentManagerInput; - }[] + }[], ): BusinessEntityInput[] { // Construct the business entities YAML definition const businessEntities = inputs.map( ({ name, input }): BusinessEntityInput => ({ // Title of Transcend Instance - title: name.replace(".yml", ""), + title: name.replace('.yml', ''), attributes: [ // Sync domain list ...(input?.domains ? [ { - key: "Transcend Domain List", + key: 'Transcend Domain List', values: [...new Set(input.domains)], }, ] @@ -34,17 +34,17 @@ export function consentManagersToBusinessEntities( ...(input?.bundleUrls ? [ { - key: "Airgap Production URL", + key: 'Airgap Production URL', values: [input.bundleUrls.PRODUCTION], }, { - key: "Airgap Test URL", + key: 'Airgap Test URL', values: [input.bundleUrls.TEST], }, { - key: "Airgap XDI URL", + key: 'Airgap XDI URL', values: [ - input.bundleUrls.PRODUCTION.replace("airgap.js", "xdi.js"), + input.bundleUrls.PRODUCTION.replace('airgap.js', 'xdi.js'), ], }, ] @@ -53,20 +53,20 @@ export function consentManagersToBusinessEntities( ...(input?.partition ? [ { - key: "Consent Partition Key", + key: 'Consent Partition Key', values: [input.partition], }, ] : []), ], - }) + }), ); // Log out info on airgap scripts to host - logger.info("\n\n~~~~~~~~~~~\nAirgap scripts to host:"); + logger.info('\n\n~~~~~~~~~~~\nAirgap scripts to host:'); for (const [ind, { attributes, title }] of businessEntities.entries()) { attributes - ?.find((attribute) => attribute.key === "Airgap Production URL") + ?.find((attribute) => attribute.key === 'Airgap Production URL') ?.values?.forEach((url) => { logger.info(`${ind}) ${title} - ${url}`); }); diff --git a/src/lib/consent-manager/createConsentToken.ts b/src/lib/consent-manager/createConsentToken.ts index b628613f..22e7e9aa 100644 --- a/src/lib/consent-manager/createConsentToken.ts +++ b/src/lib/consent-manager/createConsentToken.ts @@ -1,5 +1,5 @@ -import * as crypto from "node:crypto"; -import * as jwt from "jsonwebtoken"; +import * as crypto from 'node:crypto'; +import * as jwt from 'jsonwebtoken'; /** * Function to create a consent manager token @@ -13,16 +13,16 @@ import * as jwt from "jsonwebtoken"; export function createConsentToken( userId: string, base64EncryptionKey: string, - base64SigningKey: string + base64SigningKey: string, ): string { // Read on for where to find these keys - const signingKey = Buffer.from(base64SigningKey, "base64"); - const encryptionKey = Buffer.from(base64EncryptionKey, "base64"); + const signingKey = Buffer.from(base64SigningKey, 'base64'); + const encryptionKey = Buffer.from(base64EncryptionKey, 'base64'); // NIST's AES-KWP implementation { aes 48 } - see https://tools.ietf.org/html/rfc5649 - const encryptionAlgorithm = "id-aes256-wrap-pad"; + const encryptionAlgorithm = 'id-aes256-wrap-pad'; // Initial Value for AES-KWP integrity check - see https://tools.ietf.org/html/rfc5649#section-3 - const iv = Buffer.from("A65959A6", "hex"); + const iv = Buffer.from('A65959A6', 'hex'); // Set up encryption algorithm const cipher = crypto.createCipheriv(encryptionAlgorithm, encryptionKey, iv); @@ -30,7 +30,7 @@ export function createConsentToken( const encryptedIdentifier = Buffer.concat([ cipher.update(userId), cipher.final(), - ]).toString("base64"); + ]).toString('base64'); // Create the JWT content - jwt.sign will add a 'iat' (issued at) field to the payload // If you wanted to add something manually, consider @@ -42,7 +42,7 @@ export function createConsentToken( // Create a JSON web token and HMAC it with SHA-384 const consentToken = jwt.sign(jwtPayload, signingKey, { - algorithm: "HS384", + algorithm: 'HS384', }); return consentToken; diff --git a/src/lib/consent-manager/dataFlowsToDataSilos.ts b/src/lib/consent-manager/dataFlowsToDataSilos.ts index be5a98da..aa89cc2a 100644 --- a/src/lib/consent-manager/dataFlowsToDataSilos.ts +++ b/src/lib/consent-manager/dataFlowsToDataSilos.ts @@ -1,6 +1,6 @@ -import { union } from "lodash-es"; -import { DataFlowInput, DataSiloInput } from "../../codecs"; -import { IndexedCatalogs } from "../graphql"; +import { union } from 'lodash-es'; +import { DataFlowInput, DataSiloInput } from '../../codecs'; +import { IndexedCatalogs } from '../graphql'; /** * Convert data flow configurations into a set of data silo configurations @@ -12,13 +12,13 @@ import { IndexedCatalogs } from "../graphql"; export function dataFlowsToDataSilos( inputs: DataFlowInput[], { - adTechPurposes = ["SaleOfInfo"], + adTechPurposes = ['SaleOfInfo'], serviceToTitle, serviceToSupportedIntegration, }: IndexedCatalogs & { /** List of purposes that are considered "Ad Tech" */ adTechPurposes?: string[]; - } + }, ): { /** List of data silo configurations for site-tech services */ siteTechDataSilos: DataSiloInput[]; @@ -38,13 +38,13 @@ export function dataFlowsToDataSilos( for (const flow of inputs) { // process data flows with services const { service, attributes = [] } = flow; - if (!service || service === "internalService") { + if (!service || service === 'internalService') { continue; } // create mapping to found on domain const foundOnDomain = attributes.find( - (attribute) => attribute.key === "Found on Domain" + (attribute) => attribute.key === 'Found on Domain', ); // Create a list of all domains where the data flow was found @@ -54,8 +54,8 @@ export function dataFlowsToDataSilos( } serviceToFoundOnDomain[service].push( ...foundOnDomain.values.map((v) => - v.replace("https://", "").replace("http://", "") - ) + v.replace('https://', '').replace('http://', ''), + ), ); serviceToFoundOnDomain[service] = [ ...new Set(serviceToFoundOnDomain[service]), @@ -70,7 +70,7 @@ export function dataFlowsToDataSilos( // remove from site tech list if (siteTechIntegrations.includes(service)) { siteTechIntegrations = siteTechIntegrations.filter( - (s) => s !== service + (s) => s !== service, ); } } else if (!adTechIntegrations.includes(service)) { @@ -84,14 +84,14 @@ export function dataFlowsToDataSilos( title: serviceToTitle[service], ...(serviceToSupportedIntegration[service] ? { integrationName: service } - : { integrationName: "promptAPerson", "outer-type": service }), + : { integrationName: 'promptAPerson', 'outer-type': service }), attributes: [ { - key: "Tech Type", - values: ["Ad Tech"], + key: 'Tech Type', + values: ['Ad Tech'], }, { - key: "Found On Domain", + key: 'Found On Domain', values: serviceToFoundOnDomain[service] || [], }, ], @@ -103,18 +103,18 @@ export function dataFlowsToDataSilos( title: serviceToTitle[service], ...(serviceToSupportedIntegration[service] ? { integrationName: service } - : { integrationName: "promptAPerson", outerType: service }), + : { integrationName: 'promptAPerson', outerType: service }), attributes: [ { - key: "Tech Type", - values: ["Site Tech"], + key: 'Tech Type', + values: ['Site Tech'], }, { - key: "Found On Domain", + key: 'Found On Domain', values: serviceToFoundOnDomain[service] || [], }, ], - }) + }), ); return { diff --git a/src/lib/consent-manager/uploadConsents.ts b/src/lib/consent-manager/uploadConsents.ts index 071bd1b8..c682baca 100644 --- a/src/lib/consent-manager/uploadConsents.ts +++ b/src/lib/consent-manager/uploadConsents.ts @@ -1,20 +1,20 @@ -import { ConsentPreferencesBody } from "@transcend-io/airgap.js-types"; -import { decodeCodec } from "@transcend-io/type-utils"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import * as t from "io-ts"; -import { DEFAULT_TRANSCEND_CONSENT_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { createTranscendConsentGotInstance } from "../graphql"; -import { createConsentToken } from "./createConsentToken"; -import type { ConsentPreferenceUpload } from "./types"; +import { ConsentPreferencesBody } from '@transcend-io/airgap.js-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import * as t from 'io-ts'; +import { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { createTranscendConsentGotInstance } from '../graphql'; +import { createConsentToken } from './createConsentToken'; +import type { ConsentPreferenceUpload } from './types'; export const USP_STRING_REGEX = /^[0-9][Y|N]([Y|N])[Y|N]$/; export const PurposeMap = t.record( t.string, - t.union([t.boolean, t.literal("Auto")]) + t.union([t.boolean, t.literal('Auto')]), ); /** @@ -48,15 +48,15 @@ export async function uploadConsents({ // Ensure usp strings are valid const invalidUspStrings = preferences.filter( - (pref) => pref.usp && !USP_STRING_REGEX.test(pref.usp) + (pref) => pref.usp && !USP_STRING_REGEX.test(pref.usp), ); if (invalidUspStrings.length > 0) { throw new Error( `Received invalid usp strings: ${JSON.stringify( invalidUspStrings, null, - 2 - )}` + 2, + )}`, ); } @@ -79,29 +79,29 @@ export async function uploadConsents({ `Received invalid purpose maps: ${JSON.stringify( invalidPurposeMaps, null, - 2 - )}` + 2, + )}`, ); } // Ensure usp or preferences are provided const invalidInputs = preferences.filter( - (pref) => !pref.usp && !pref.purposes + (pref) => !pref.usp && !pref.purposes, ); if (invalidInputs.length > 0) { throw new Error( `Received invalid inputs, expected either purposes or usp to be defined: ${JSON.stringify( invalidInputs, null, - 2 - )}` + 2, + )}`, ); } logger.info( colors.magenta( - `Uploading ${preferences.length} user preferences to partition ${partition}` - ) + `Uploading ${preferences.length} user preferences to partition ${partition}`, + ), ); // Time duration @@ -109,7 +109,7 @@ export async function uploadConsents({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Build a GraphQL client @@ -119,7 +119,7 @@ export async function uploadConsents({ preferences, async ({ userId, - confirmed = "true", + confirmed = 'true', updated, prompted, purposes, @@ -128,7 +128,7 @@ export async function uploadConsents({ const token = createConsentToken( userId, base64EncryptionKey, - base64SigningKey + base64SigningKey, ); // parse usp string @@ -140,14 +140,14 @@ export async function uploadConsents({ token, partition, consent: { - confirmed: confirmed === "true", + confirmed: confirmed === 'true', purposes: purposes ? decodeCodec(PurposeMap, purposes) : consent.usp - ? { SaleOfInfo: saleStatus === "Y" } - : {}, - ...(updated ? { updated: updated === "true" } : {}), - ...(prompted ? { prompted: prompted === "true" } : {}), + ? { SaleOfInfo: saleStatus === 'Y' } + : {}, + ...(updated ? { updated: updated === 'true' } : {}), + ...(prompted ? { prompted: prompted === 'true' } : {}), ...consent, }, } as ConsentPreferencesBody; @@ -155,13 +155,13 @@ export async function uploadConsents({ // Make the request try { await transcendConsentApi - .post("sync", { + .post('sync', { json: input, }) .json(); } catch (error) { try { - const parsed = JSON.parse(error?.response?.body || "{}"); + const parsed = JSON.parse(error?.response?.body || '{}'); if (parsed.error) { logger.error(colors.red(`Error: ${parsed.error}`)); } @@ -171,14 +171,14 @@ export async function uploadConsents({ throw new Error( `Received an error from server: ${ error?.response?.body || error?.message - }` + }`, ); } total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -191,7 +191,7 @@ export async function uploadConsents({ preferences.length } user preferences to partition ${partition} in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); } diff --git a/src/lib/consent-manager/uploadCookiesFromCsv.ts b/src/lib/consent-manager/uploadCookiesFromCsv.ts index f22487d7..1ef399b2 100644 --- a/src/lib/consent-manager/uploadCookiesFromCsv.ts +++ b/src/lib/consent-manager/uploadCookiesFromCsv.ts @@ -1,22 +1,22 @@ -import { ConsentTrackerStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { CookieCsvInput, CookieInput } from "../../codecs"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { buildTranscendGraphQLClient, syncCookies } from "../graphql"; -import { splitCsvToList } from "../requests"; -import { readCsv } from "../requests/readCsv"; +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { CookieCsvInput, CookieInput } from '../../codecs'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { buildTranscendGraphQLClient, syncCookies } from '../graphql'; +import { splitCsvToList } from '../requests'; +import { readCsv } from '../requests/readCsv'; const OMIT_COLUMNS = new Set([ - "ID", - "Activity", - "Encounters", - "Last Seen At", - "Has Native Do Not Sell/Share Support", - "IAB USP API Support", - "Service Description", - "Website URL", - "Categories of Recipients", + 'ID', + 'Activity', + 'Encounters', + 'Last Seen At', + 'Has Native Do Not Sell/Share Support', + 'IAB USP API Support', + 'Service Description', + 'Website URL', + 'Categories of Recipients', ]); /** @@ -49,7 +49,7 @@ export async function uploadCookiesFromCsv({ // Convert these inputs into a format that the other function can use const validatedCookieInputs = cookieInputs.map( ({ - "Is Regex?": isRegex, + 'Is Regex?': isRegex, Notes, // TODO: https://transcend.height.app/T-26391 - export in CSV // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -61,8 +61,8 @@ export async function uploadCookiesFromCsv({ Name, ...rest }): CookieInput => ({ - ...(typeof isRegex === "string" - ? { isRegex: isRegex.toLowerCase() === "true" } + ...(typeof isRegex === 'string' + ? { isRegex: isRegex.toLowerCase() === 'true' } : {}), name: Name, description: Notes, @@ -83,7 +83,7 @@ export async function uploadCookiesFromCsv({ key, values: splitCsvToList(value), })), - }) + }), ); // Upload the cookies into Transcend dashboard @@ -93,8 +93,8 @@ export async function uploadCookiesFromCsv({ if (!syncedCookies) { logger.error( colors.red( - "Encountered error(s) syncing cookies from CSV, see logs above for more info. " - ) + 'Encountered error(s) syncing cookies from CSV, see logs above for more info. ', + ), ); process.exit(1); } diff --git a/src/lib/consent-manager/uploadDataFlowsFromCsv.ts b/src/lib/consent-manager/uploadDataFlowsFromCsv.ts index e01f1c4a..d79817ce 100644 --- a/src/lib/consent-manager/uploadDataFlowsFromCsv.ts +++ b/src/lib/consent-manager/uploadDataFlowsFromCsv.ts @@ -1,22 +1,22 @@ -import { ConsentTrackerStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { DataFlowCsvInput, DataFlowInput } from "../../codecs"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { buildTranscendGraphQLClient, syncDataFlows } from "../graphql"; -import { splitCsvToList } from "../requests"; -import { readCsv } from "../requests/readCsv"; +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { DataFlowCsvInput, DataFlowInput } from '../../codecs'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { buildTranscendGraphQLClient, syncDataFlows } from '../graphql'; +import { splitCsvToList } from '../requests'; +import { readCsv } from '../requests/readCsv'; const OMIT_COLUMNS = new Set([ - "ID", - "Activity", - "Encounters", - "Last Seen At", - "Has Native Do Not Sell/Share Support", - "IAB USP API Support", - "Service Description", - "Website URL", - "Categories of Recipients", + 'ID', + 'Activity', + 'Encounters', + 'Last Seen At', + 'Has Native Do Not Sell/Share Support', + 'IAB USP API Support', + 'Service Description', + 'Website URL', + 'Categories of Recipients', ]); /** @@ -61,7 +61,7 @@ export async function uploadDataFlowsFromCsv({ Status, Owners, Teams, - "Connections Made To": value, + 'Connections Made To': value, ...rest }): DataFlowInput => ({ value, @@ -84,22 +84,22 @@ export async function uploadDataFlowsFromCsv({ key, values: splitCsvToList(value), })), - }) + }), ); // Upload the data flows into Transcend dashboard const syncedDataFlows = await syncDataFlows( client, validatedDataFlowInputs, - classifyService + classifyService, ); // Log errors if (!syncedDataFlows) { logger.error( colors.red( - "Encountered error(s) syncing data flows from CSV, see logs above for more info. " - ) + 'Encountered error(s) syncing data flows from CSV, see logs above for more info. ', + ), ); process.exit(1); } diff --git a/src/lib/cron/markCronIdentifierCompleted.ts b/src/lib/cron/markCronIdentifierCompleted.ts index ce1ce7d3..47b70c9b 100644 --- a/src/lib/cron/markCronIdentifierCompleted.ts +++ b/src/lib/cron/markCronIdentifierCompleted.ts @@ -1,5 +1,5 @@ -import type { Got } from "got"; -import * as t from "io-ts"; +import type { Got } from 'got'; +import * as t from 'io-ts'; /** * Minimal set required to mark as completed @@ -22,13 +22,13 @@ export type CronIdentifierPush = t.TypeOf; */ export async function markCronIdentifierCompleted( sombra: Got, - { nonce, identifier }: CronIdentifierPush + { nonce, identifier }: CronIdentifierPush, ): Promise { try { // Make the GraphQL request - await sombra.put("v1/data-silo", { + await sombra.put('v1/data-silo', { headers: { - "x-transcend-nonce": nonce, + 'x-transcend-nonce': nonce, }, json: { profiles: [ @@ -47,7 +47,7 @@ export async function markCronIdentifierCompleted( throw new Error( `Received an error from server: ${ error?.response?.body || error?.message - }` + }`, ); } } diff --git a/src/lib/cron/markRequestDataSiloIdsCompleted.ts b/src/lib/cron/markRequestDataSiloIdsCompleted.ts index f05f1b53..ff0d03d1 100644 --- a/src/lib/cron/markRequestDataSiloIdsCompleted.ts +++ b/src/lib/cron/markRequestDataSiloIdsCompleted.ts @@ -1,15 +1,15 @@ -import { RequestDataSiloStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestDataSiloStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, CHANGE_REQUEST_DATA_SILO_STATUS, fetchRequestDataSilo, makeGraphQLRequest, -} from "../graphql"; +} from '../graphql'; /** * Given a CSV of Request IDs, mark associated RequestDataSilos as completed @@ -46,14 +46,14 @@ export async function markRequestDataSiloIdsCompleted({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Notify Transcend logger.info( colors.magenta( - `Notifying Transcend for data silo "${dataSiloId}" marking "${requestIds.length}" requests as completed.` - ) + `Notifying Transcend for data silo "${dataSiloId}" marking "${requestIds.length}" requests as completed.`, + ), ); let total = 0; @@ -75,7 +75,7 @@ export async function markRequestDataSiloIdsCompleted({ status, }); } catch (error) { - if (!error.message.includes("Client error: Request must be active:")) { + if (!error.message.includes('Client error: Request must be active:')) { throw error; } } @@ -83,7 +83,7 @@ export async function markRequestDataSiloIdsCompleted({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -92,8 +92,8 @@ export async function markRequestDataSiloIdsCompleted({ logger.info( colors.green( - `Successfully notified Transcend in "${totalTime / 1000}" seconds!` - ) + `Successfully notified Transcend in "${totalTime / 1000}" seconds!`, + ), ); return requestIds.length; } diff --git a/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts b/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts index 8594279c..607d9134 100644 --- a/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts +++ b/src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts @@ -1,18 +1,18 @@ -import { RequestAction } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import { RequestAction } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchRequestDataSiloActiveCount, -} from "../graphql"; +} from '../graphql'; import { CronIdentifier, pullCronPageOfIdentifiers, -} from "./pullCronPageOfIdentifiers"; +} from './pullCronPageOfIdentifiers'; /** * A CSV formatted identifier @@ -72,7 +72,7 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ // Validate savePageSize if (savePageSize % apiPageSize !== 0) { throw new Error( - `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}` + `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`, ); } @@ -92,12 +92,12 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ logger.info( colors.magenta( `Pulling ${ - skipRequestCount ? "all" : totalRequestCount + skipRequestCount ? 'all' : totalRequestCount } outstanding request identifiers ` + `for data silo: "${dataSiloId}" for requests of types "${actions.join( - '", "' - )}"` - ) + '", "', + )}"`, + ), ); // Time duration @@ -105,7 +105,7 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); const foundRequestIds = new Set(); @@ -144,9 +144,9 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ ({ attributes, ...identifier }) => ({ ...identifier, ...Object.fromEntries( - attributes.map((value) => [value.key, value.values.join(",")]) + attributes.map((value) => [value.key, value.values.join(',')]), ), - }) + }), ); identifiers.push(...identifiersWithAction); @@ -163,8 +163,8 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ if (skipRequestCount) { logger.info( colors.magenta( - `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests` - ) + `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`, + ), ); } else { progressBar.update(foundRequestIds.size); @@ -187,8 +187,8 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ colors.green( `Successfully pulled ${identifiers.length} outstanding identifiers from ${ foundRequestIds.size - } requests in "${totalTime / 1000}" seconds!` - ) + } requests in "${totalTime / 1000}" seconds!`, + ), ); return { identifiers }; diff --git a/src/lib/cron/pullCronPageOfIdentifiers.ts b/src/lib/cron/pullCronPageOfIdentifiers.ts index 9688fd62..a1e7b9f4 100644 --- a/src/lib/cron/pullCronPageOfIdentifiers.ts +++ b/src/lib/cron/pullCronPageOfIdentifiers.ts @@ -1,7 +1,7 @@ -import { RequestAction } from "@transcend-io/privacy-types"; -import { decodeCodec } from "@transcend-io/type-utils"; -import type { Got } from "got"; -import * as t from "io-ts"; +import { RequestAction } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import type { Got } from 'got'; +import * as t from 'io-ts'; export const CronIdentifier = t.type({ /** The identifier value */ @@ -25,7 +25,7 @@ export const CronIdentifier = t.type({ t.type({ key: t.string, values: t.array(t.string), - }) + }), ), }); @@ -56,7 +56,7 @@ export async function pullCronPageOfIdentifiers( limit?: number; /** Page to pull in */ offset?: number; - } + }, ): Promise { try { // Make the GraphQL request @@ -73,14 +73,14 @@ export async function pullCronPageOfIdentifiers( t.type({ items: t.array(CronIdentifier), }), - response + response, ); return items; } catch (error) { throw new Error( `Received an error from server: ${ error?.response?.body || error?.message - }` + }`, ); } } diff --git a/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts b/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts index 8594279c..607d9134 100644 --- a/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts +++ b/src/lib/cron/pullCustomSiloOutstandingIdentifiers.ts @@ -1,18 +1,18 @@ -import { RequestAction } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import { RequestAction } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchRequestDataSiloActiveCount, -} from "../graphql"; +} from '../graphql'; import { CronIdentifier, pullCronPageOfIdentifiers, -} from "./pullCronPageOfIdentifiers"; +} from './pullCronPageOfIdentifiers'; /** * A CSV formatted identifier @@ -72,7 +72,7 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ // Validate savePageSize if (savePageSize % apiPageSize !== 0) { throw new Error( - `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}` + `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`, ); } @@ -92,12 +92,12 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ logger.info( colors.magenta( `Pulling ${ - skipRequestCount ? "all" : totalRequestCount + skipRequestCount ? 'all' : totalRequestCount } outstanding request identifiers ` + `for data silo: "${dataSiloId}" for requests of types "${actions.join( - '", "' - )}"` - ) + '", "', + )}"`, + ), ); // Time duration @@ -105,7 +105,7 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); const foundRequestIds = new Set(); @@ -144,9 +144,9 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ ({ attributes, ...identifier }) => ({ ...identifier, ...Object.fromEntries( - attributes.map((value) => [value.key, value.values.join(",")]) + attributes.map((value) => [value.key, value.values.join(',')]), ), - }) + }), ); identifiers.push(...identifiersWithAction); @@ -163,8 +163,8 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ if (skipRequestCount) { logger.info( colors.magenta( - `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests` - ) + `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`, + ), ); } else { progressBar.update(foundRequestIds.size); @@ -187,8 +187,8 @@ export async function pullChunkedCustomSiloOutstandingIdentifiers({ colors.green( `Successfully pulled ${identifiers.length} outstanding identifiers from ${ foundRequestIds.size - } requests in "${totalTime / 1000}" seconds!` - ) + } requests in "${totalTime / 1000}" seconds!`, + ), ); return { identifiers }; diff --git a/src/lib/cron/pushCronIdentifiersFromCsv.ts b/src/lib/cron/pushCronIdentifiersFromCsv.ts index ca4cab47..72076830 100644 --- a/src/lib/cron/pushCronIdentifiersFromCsv.ts +++ b/src/lib/cron/pushCronIdentifiersFromCsv.ts @@ -1,15 +1,15 @@ -import cliProgress from "cli-progress"; -import colors from "colors"; -import { chunk } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map, mapSeries } from "../bluebird-replace"; -import { createSombraGotInstance } from "../graphql"; -import { readCsv } from "../requests"; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { chunk } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; +import { createSombraGotInstance } from '../graphql'; +import { readCsv } from '../requests'; import { CronIdentifierPush, markCronIdentifierCompleted, -} from "./markCronIdentifierCompleted"; +} from './markCronIdentifierCompleted'; /** * Given a CSV of cron job outputs, mark all requests as completed in Transcend @@ -51,8 +51,8 @@ export async function pushCronIdentifiersFromCsv({ // Notify Transcend logger.info( colors.magenta( - `Notifying Transcend for data silo "${dataSiloId}" marking "${activeResults.length}" identifiers as completed.` - ) + `Notifying Transcend for data silo "${dataSiloId}" marking "${activeResults.length}" identifiers as completed.`, + ), ); // Time duration @@ -60,7 +60,7 @@ export async function pushCronIdentifiersFromCsv({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); let successCount = 0; @@ -73,14 +73,14 @@ export async function pushCronIdentifiersFromCsv({ const totalChunks = chunks.length; const processChunk = async ( items: CronIdentifierPush[], - chunkIndex: number + chunkIndex: number, ): Promise => { logger.info( colors.blue( `Processing chunk ${chunkIndex + 1}/${totalChunks} (${ chunk.length - } items)` - ) + } items)`, + ), ); // Process the items of the chunk concurrently @@ -95,8 +95,8 @@ export async function pushCronIdentifiersFromCsv({ } catch (error) { logger.error( colors.red( - `Error notifying Transcend for identifier "${identifier.identifier}" - ${error?.message}` - ) + `Error notifying Transcend for identifier "${identifier.identifier}" - ${error?.message}`, + ), ); errorCount += 1; } @@ -106,7 +106,7 @@ export async function pushCronIdentifiersFromCsv({ // Sleep between chunks (except for the last chunk) if (sleepSeconds > 0 && chunkIndex < totalChunks - 1) { logger.info( - colors.yellow(`Sleeping for ${sleepSeconds}s before next chunk...`) + colors.yellow(`Sleeping for ${sleepSeconds}s before next chunk...`), ); await new Promise((resolve) => { @@ -126,24 +126,24 @@ export async function pushCronIdentifiersFromCsv({ colors.green( `Successfully notified Transcend for ${successCount} identifiers in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); if (failureCount) { logger.info( colors.magenta( `There were ${failureCount} identifiers that were not in a state to be updated.` + - "They likely have already been resolved." - ) + 'They likely have already been resolved.', + ), ); } if (errorCount) { logger.error( colors.red( - `There were ${errorCount} identifiers that failed to be updated. Please review the logs for more information.` - ) + `There were ${errorCount} identifiers that failed to be updated. Please review the logs for more information.`, + ), ); - throw new Error("Failed to update all identifiers"); + throw new Error('Failed to update all identifiers'); } return activeResults.length; } diff --git a/src/lib/cron/writeCsv.ts b/src/lib/cron/writeCsv.ts index c4e19451..f12f8f8c 100644 --- a/src/lib/cron/writeCsv.ts +++ b/src/lib/cron/writeCsv.ts @@ -1,6 +1,6 @@ -import { appendFileSync, createWriteStream, writeFileSync } from "node:fs"; -import { ObjByString } from "@transcend-io/type-utils"; -import * as fastcsv from "fast-csv"; +import { appendFileSync, createWriteStream, writeFileSync } from 'node:fs'; +import { ObjByString } from '@transcend-io/type-utils'; +import * as fastcsv from 'fast-csv'; /** * Escape a CSV value @@ -9,7 +9,7 @@ import * as fastcsv from "fast-csv"; * @returns Escaped value */ function escapeCsvValue(value: string): string { - if (value.includes('"') || value.includes(",") || value.includes("\n")) { + if (value.includes('"') || value.includes(',') || value.includes('\n')) { return `"${value.replaceAll('"', '""')}"`; } return value; @@ -24,7 +24,7 @@ function escapeCsvValue(value: string): string { export function writeCsvSync( filePath: string, data: ObjByString[], - headers: string[] + headers: string[], ): void { const rows: string[][] = []; @@ -33,8 +33,8 @@ export function writeCsvSync( // Build CSV content with proper escaping const csvContent = rows - .map((row) => row.map(escapeCsvValue).join(",")) - .join("\n"); + .map((row) => row.map(escapeCsvValue).join(',')) + .join('\n'); // Write to file, overwriting existing content writeFileSync(filePath, csvContent); @@ -53,8 +53,8 @@ export function appendCsvSync(filePath: string, data: ObjByString[]): void { // Build CSV content with proper escaping const csvContent = rows - .map((row) => row.map(escapeCsvValue).join(",")) - .join("\n"); + .map((row) => row.map(escapeCsvValue).join(',')) + .join('\n'); // Append to file with leading newline appendFileSync(filePath, `\n${csvContent}`); @@ -70,7 +70,7 @@ export function appendCsvSync(filePath: string, data: ObjByString[]): void { export async function writeCsv( filePath: string, data: ObjByString[], - headers: boolean | string[] = true + headers: boolean | string[] = true, ): Promise { const ws = createWriteStream(filePath); await new Promise((resolve, reject) => { @@ -78,8 +78,8 @@ export async function writeCsv( fastcsv .write(data, { headers, objectMode: true }) .pipe(ws) - .on("error", reject) - .on("end", () => { + .on('error', reject) + .on('end', () => { resolve(true); }); } catch (error) { @@ -100,14 +100,14 @@ export function parseFilePath(filePath: string): { /** Extension of the file */ extension: string; } { - const lastDotIndex = filePath.lastIndexOf("."); + const lastDotIndex = filePath.lastIndexOf('.'); return { baseName: lastDotIndex === -1 ? filePath : filePath.slice(0, Math.max(0, lastDotIndex)), extension: - lastDotIndex === -1 ? ".csv" : filePath.slice(Math.max(0, lastDotIndex)), + lastDotIndex === -1 ? '.csv' : filePath.slice(Math.max(0, lastDotIndex)), }; } @@ -124,7 +124,7 @@ export async function writeLargeCsv( filePath: string, data: ObjByString[], headers: boolean | string[] = true, - chunkSize = 100_000 + chunkSize = 100_000, ): Promise { if (data.length <= chunkSize) { // If data is small enough, write to single file @@ -145,7 +145,7 @@ export async function writeLargeCsv( // Create filename with chunk number and zero-padding const chunkNumber = String(index + 1).padStart( String(totalChunks).length, - "0" + '0', ); const chunkFilePath = `${baseName}_part${chunkNumber}_of_${totalChunks}${extension}`; diff --git a/src/lib/data-inventory/pullAllDatapoints.ts b/src/lib/data-inventory/pullAllDatapoints.ts index ca56995b..9dbbc78d 100644 --- a/src/lib/data-inventory/pullAllDatapoints.ts +++ b/src/lib/data-inventory/pullAllDatapoints.ts @@ -1,22 +1,22 @@ import { SubDataPointDataSubCategoryGuessStatus, type DataCategoryType, -} from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { gql } from "graphql-request"; -import type { GraphQLClient } from "graphql-request"; -import { chunk, keyBy, sortBy, uniq } from "lodash-es"; -import type { DataCategoryInput, ProcessingPurposeInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +} from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { gql } from 'graphql-request'; +import type { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy, sortBy, uniq } from 'lodash-es'; +import type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { DATA_SILO_EXPORT, DATAPOINT_EXPORT, makeGraphQLRequest, SUB_DATA_POINTS_COUNT, type DataSiloAttributeValue, -} from "../graphql"; +} from '../graphql'; export interface DataSiloCsvPreview { /** ID of dataSilo */ @@ -99,7 +99,7 @@ async function pullSubDatapoints( }: DatapointFilterOptions & { /** Page size to pull in */ pageSize?: number; - } = {} + } = {}, ): Promise { const subDataPoints: SubDataPointCsvPreview[] = []; @@ -109,7 +109,7 @@ async function pullSubDatapoints( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Filters @@ -138,7 +138,7 @@ async function pullSubDatapoints( filterBy, }); - logger.info(colors.magenta("[Step 1/3] Pulling in all subdatapoints")); + logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints')); progressBar.start(totalCount, 0); let total = 0; @@ -193,7 +193,7 @@ async function pullSubDatapoints( status classifierVersion }` - : "" + : '' } ${ includeAttributes @@ -203,7 +203,7 @@ async function pullSubDatapoints( } name }` - : "" + : '' } } } @@ -217,7 +217,7 @@ async function pullSubDatapoints( // TODO: https://transcend.height.app/T-40484 - add cursor support // ...(cursor ? { cursor: { id: cursor } } : {}), }, - } + }, ); cursor = nodes.at(-1)?.id; @@ -229,8 +229,8 @@ async function pullSubDatapoints( } catch (error) { logger.error( colors.red( - `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}` - ) + `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`, + ), ); throw error; } @@ -240,14 +240,14 @@ async function pullSubDatapoints( const t1 = Date.now(); const totalTime = t1 - t0; - const sorted = sortBy(subDataPoints, "name"); + const sorted = sortBy(subDataPoints, 'name'); logger.info( colors.green( `Successfully pulled in ${sorted.length} subdatapoints in ${ totalTime / 1000 - } seconds!` - ) + } seconds!`, + ), ); return sorted; } @@ -269,7 +269,7 @@ async function pullDatapoints( dataPointIds: string[]; /** Page size to pull in */ pageSize?: number; - } + }, ): Promise { const dataPoints: DataPointCsvPreview[] = []; @@ -279,13 +279,13 @@ async function pullDatapoints( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); logger.info( colors.magenta( - `[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints` - ) + `[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`, + ), ); // Group by 100 @@ -317,9 +317,9 @@ async function pullDatapoints( logger.error( colors.red( `An error fetching subdatapoints for IDs ${dataPointIdsGroup.join( - ", " - )}` - ) + ', ', + )}`, + ), ); throw error; } @@ -333,8 +333,8 @@ async function pullDatapoints( colors.green( `Successfully pulled in ${dataPoints.length} dataPoints in ${ totalTime / 1000 - } seconds!` - ) + } seconds!`, + ), ); return dataPoints; } @@ -356,7 +356,7 @@ async function pullDataSilos( dataSiloIds: string[]; /** Page size to pull in */ pageSize?: number; - } + }, ): Promise { const dataSilos: DataSiloCsvPreview[] = []; @@ -366,13 +366,13 @@ async function pullDataSilos( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); logger.info( colors.magenta( - `[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos` - ) + `[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`, + ), ); // Group by 100 @@ -403,8 +403,8 @@ async function pullDataSilos( } catch (error) { logger.error( colors.red( - `An error fetching data silos for IDs ${dataSiloIdsGroup.join(", ")}` - ) + `An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`, + ), ); throw error; } @@ -418,8 +418,8 @@ async function pullDataSilos( colors.green( `Successfully pulled in ${dataSilos.length} data silos in ${ totalTime / 1000 - } seconds!` - ) + } seconds!`, + ), ); return dataSilos; } @@ -443,7 +443,7 @@ export async function pullAllDatapoints( }: DatapointFilterOptions & { /** Page size to pull in */ pageSize?: number; - } = {} + } = {}, ): Promise< (SubDataPointCsvPreview & { /** Data point information */ @@ -467,14 +467,14 @@ export async function pullAllDatapoints( const dataPoints = await pullDatapoints(client, { dataPointIds, }); - const dataPointById = keyBy(dataPoints, "id"); + const dataPointById = keyBy(dataPoints, 'id'); // The data silo IDs to grab const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId)); const dataSilos = await pullDataSilos(client, { dataSiloIds: allDataSiloIds, }); - const dataSiloById = keyBy(dataSilos, "id"); + const dataSiloById = keyBy(dataSilos, 'id'); return subDatapoints.map((subDataPoint) => ({ ...subDataPoint, diff --git a/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts b/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts index f595d356..5dbad1b6 100644 --- a/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts +++ b/src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts @@ -1,11 +1,11 @@ -import type { UnstructuredSubDataPointRecommendationStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { gql, type GraphQLClient } from "graphql-request"; -import { sortBy } from "lodash-es"; -import type { DataCategoryInput } from "../../codecs"; -import { logger } from "../../logger"; -import { ENTRY_COUNT, makeGraphQLRequest } from "../graphql"; +import type { UnstructuredSubDataPointRecommendationStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { gql, type GraphQLClient } from 'graphql-request'; +import { sortBy } from 'lodash-es'; +import type { DataCategoryInput } from '../../codecs'; +import { logger } from '../../logger'; +import { ENTRY_COUNT, makeGraphQLRequest } from '../graphql'; interface UnstructuredSubDataPointRecommendationCsvPreview { /** ID of subDatapoint */ @@ -68,7 +68,7 @@ export async function pullUnstructuredSubDataPointRecommendations( }: EntryFilterOptions & { /** Page size to pull in */ pageSize?: number; - } = {} + } = {}, ): Promise { const unstructuredSubDataPointRecommendations: UnstructuredSubDataPointRecommendationCsvPreview[] = []; @@ -79,7 +79,7 @@ export async function pullUnstructuredSubDataPointRecommendations( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Filters @@ -102,7 +102,7 @@ export async function pullUnstructuredSubDataPointRecommendations( filterBy, }); - logger.info(colors.magenta("[Step 1/3] Pulling in all subdatapoints")); + logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints')); progressBar.start(totalCount, 0); let total = 0; @@ -138,8 +138,8 @@ export async function pullUnstructuredSubDataPointRecommendations( dataSiloId scannedObjectPathId scannedObjectId - ${includeEncryptedSnippets ? "name" : ""} - ${includeEncryptedSnippets ? "contextSnippet" : ""} + ${includeEncryptedSnippets ? 'name' : ''} + ${includeEncryptedSnippets ? 'contextSnippet' : ''} dataSubCategory { name category @@ -158,7 +158,7 @@ export async function pullUnstructuredSubDataPointRecommendations( filterBy: { ...filterBy, }, - } + }, ); cursor = nodes.at(-1)?.id; @@ -170,8 +170,8 @@ export async function pullUnstructuredSubDataPointRecommendations( } catch (error) { logger.error( colors.red( - `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}` - ) + `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`, + ), ); throw error; } @@ -181,14 +181,14 @@ export async function pullUnstructuredSubDataPointRecommendations( const t1 = Date.now(); const totalTime = t1 - t0; - const sorted = sortBy(unstructuredSubDataPointRecommendations, "name"); + const sorted = sortBy(unstructuredSubDataPointRecommendations, 'name'); logger.info( colors.green( `Successfully pulled in ${sorted.length} subdatapoints in ${ totalTime / 1000 - } seconds!` - ) + } seconds!`, + ), ); return sorted; } diff --git a/src/lib/graphql/fetchAllAssessments.ts b/src/lib/graphql/fetchAllAssessments.ts index 4400472d..19319714 100644 --- a/src/lib/graphql/fetchAllAssessments.ts +++ b/src/lib/graphql/fetchAllAssessments.ts @@ -9,10 +9,10 @@ import { ProcessingPurpose, RetentionScheduleOperation, RetentionScheduleType, -} from "@transcend-io/privacy-types"; -import { GraphQLClient } from "graphql-request"; -import { ASSESSMENTS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { ASSESSMENTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Represents an assessment with various properties and metadata. @@ -340,7 +340,7 @@ const PAGE_SIZE = 20; * @returns All assessments in the organization */ export async function fetchAllAssessments( - client: GraphQLClient + client: GraphQLClient, ): Promise { const assessments: Assessment[] = []; let offset = 0; diff --git a/src/lib/graphql/fetchAllRequestIdentifiers.ts b/src/lib/graphql/fetchAllRequestIdentifiers.ts index 52646f7a..d81ff2c8 100644 --- a/src/lib/graphql/fetchAllRequestIdentifiers.ts +++ b/src/lib/graphql/fetchAllRequestIdentifiers.ts @@ -1,13 +1,13 @@ -import { IdentifierType } from "@transcend-io/privacy-types"; -import { decodeCodec, valuesOf } from "@transcend-io/type-utils"; -import type { Got } from "got"; -import { GraphQLClient } from "graphql-request"; -import * as t from "io-ts"; -import semver from "semver"; -import { SOMBRA_VERSION } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { IdentifierType } from '@transcend-io/privacy-types'; +import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; +import type { Got } from 'got'; +import { GraphQLClient } from 'graphql-request'; +import * as t from 'io-ts'; +import semver from 'semver'; +import { SOMBRA_VERSION } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; -const MIN_SOMBRA_VERSION_TO_DECRYPT = "7.180"; +const MIN_SOMBRA_VERSION_TO_DECRYPT = '7.180'; const RequestIdentifier = t.type({ /** ID of request */ @@ -45,7 +45,7 @@ export async function fetchAllRequestIdentifiers( }: { /** ID of request to filter on */ requestId: string; - } + }, ): Promise { const requestIdentifiers: RequestIdentifier[] = []; let offset = 0; @@ -69,7 +69,7 @@ export async function fetchAllRequestIdentifiers( if (version && semver.lt(version, MIN_SOMBRA_VERSION_TO_DECRYPT)) { throw new Error( - `Please upgrade Sombra to ${MIN_SOMBRA_VERSION_TO_DECRYPT} or greater to use this command.` + `Please upgrade Sombra to ${MIN_SOMBRA_VERSION_TO_DECRYPT} or greater to use this command.`, ); } @@ -80,7 +80,7 @@ export async function fetchAllRequestIdentifiers( .post<{ /** Decrypted identifiers */ identifiers: RequestIdentifier[]; - }>("v1/request-identifiers", { + }>('v1/request-identifiers', { json: { first: PAGE_SIZE, offset, @@ -92,13 +92,13 @@ export async function fetchAllRequestIdentifiers( throw new Error( `Failed to fetch request identifiers: ${ error?.response?.body || error?.message - }` + }`, ); } const { identifiers: nodes } = decodeCodec( RequestIdentifiersResponse, - response + response, ); requestIdentifiers.push(...nodes); diff --git a/src/lib/graphql/fetchAllRequests.ts b/src/lib/graphql/fetchAllRequests.ts index 3d231e8b..44ac99dc 100644 --- a/src/lib/graphql/fetchAllRequests.ts +++ b/src/lib/graphql/fetchAllRequests.ts @@ -1,19 +1,19 @@ -import { LanguageKey } from "@transcend-io/internationalization"; +import { LanguageKey } from '@transcend-io/internationalization'; import { IsoCountryCode, IsoCountrySubdivisionCode, RequestAction, RequestOrigin, RequestStatus, -} from "@transcend-io/privacy-types"; -import { valuesOf } from "@transcend-io/type-utils"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import * as t from "io-ts"; -import { logger } from "../../logger"; -import { REQUESTS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from '@transcend-io/privacy-types'; +import { valuesOf } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import * as t from 'io-ts'; +import { logger } from '../../logger'; +import { REQUESTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export const PrivacyRequest = t.intersection([ t.type({ @@ -53,7 +53,7 @@ export const PrivacyRequest = t.intersection([ id: t.string, attributeKey: t.type({ name: t.string, id: t.string }), name: t.string, - }) + }), ), }), t.partial({ @@ -111,15 +111,15 @@ export async function fetchAllRequests( * at runtime while other filters are applied at the GraphQL level. */ requestIds?: string[]; - } = {} + } = {}, ): Promise { - logger.info(colors.magenta("Fetching requests...")); + logger.info(colors.magenta('Fetching requests...')); // create a new progress bar instance and use shades_classic theme const t0 = Date.now(); const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // read in requests @@ -178,20 +178,20 @@ export async function fetchAllRequests( colors.green( `Completed fetching of ${requests.length} request in "${ totalTime / 1000 - }" seconds.` - ) + }" seconds.`, + ), ); // Filter down requests by request ID let allRequests = requests; if (requestIds && requestIds.length > 0) { allRequests = allRequests.filter((request) => - requestIds.includes(request.id) + requestIds.includes(request.id), ); logger.info( colors.green( - `Filtered down to ${allRequests.length} requests based on ${requestIds.length} provided IDs.` - ) + `Filtered down to ${allRequests.length} requests based on ${requestIds.length} provided IDs.`, + ), ); } diff --git a/src/lib/graphql/fetchApiKeys.ts b/src/lib/graphql/fetchApiKeys.ts index cd7a0bec..64cdb963 100644 --- a/src/lib/graphql/fetchApiKeys.ts +++ b/src/lib/graphql/fetchApiKeys.ts @@ -1,10 +1,10 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { difference, keyBy, uniq } from "lodash-es"; -import { TranscendInput } from "../../codecs"; -import { logger } from "../../logger"; -import { API_KEYS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference, keyBy, uniq } from 'lodash-es'; +import { TranscendInput } from '../../codecs'; +import { logger } from '../../logger'; +import { API_KEYS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface ApiKey { /** ID of API key */ @@ -15,7 +15,7 @@ export interface ApiKey { const PAGE_SIZE = 20; -const ADMIN_LINK = "https://app.transcend.io/infrastructure/api-keys"; +const ADMIN_LINK = 'https://app.transcend.io/infrastructure/api-keys'; /** * Fetch all API keys in an organization @@ -26,7 +26,7 @@ const ADMIN_LINK = "https://app.transcend.io/infrastructure/api-keys"; */ export async function fetchAllApiKeys( client: GraphQLClient, - titles?: string[] + titles?: string[], ): Promise { const apiKeys: ApiKey[] = []; let offset = 0; @@ -65,36 +65,36 @@ export async function fetchAllApiKeys( */ export async function fetchApiKeys( { - "api-keys": apiKeyInputs = [], - "data-silos": dataSilos = [], + 'api-keys': apiKeyInputs = [], + 'data-silos': dataSilos = [], }: TranscendInput, client: GraphQLClient, - fetchAll = false + fetchAll = false, ): Promise> { logger.info( colors.magenta( - `Fetching ${fetchAll ? "all" : apiKeyInputs.length} API keys...` - ) + `Fetching ${fetchAll ? 'all' : apiKeyInputs.length} API keys...`, + ), ); const titles = apiKeyInputs.map(({ title }) => title); const expectedApiKeyTitles = uniq( dataSilos - .map((silo) => silo["api-key-title"]) - .filter((x): x is string => !!x) + .map((silo) => silo['api-key-title']) + .filter((x): x is string => !!x), ); const allTitlesExpected = [...expectedApiKeyTitles, ...titles]; const apiKeys = await fetchAllApiKeys( client, - fetchAll ? undefined : [...expectedApiKeyTitles, ...titles] + fetchAll ? undefined : [...expectedApiKeyTitles, ...titles], ); // Create a map - const apiKeysByTitle = keyBy(apiKeys, "title"); + const apiKeysByTitle = keyBy(apiKeys, 'title'); // Determine expected set of apiKeys expected const missingApiKeys = difference( allTitlesExpected, - apiKeys.map(({ title }) => title) + apiKeys.map(({ title }) => title), ); // If there are missing apiKeys, throw an error @@ -102,9 +102,9 @@ export async function fetchApiKeys( logger.info( colors.red( `Failed to find API keys "${missingApiKeys.join( - '", "' - )}"! Make sure these API keys are created at: ${ADMIN_LINK}` - ) + '", "', + )}"! Make sure these API keys are created at: ${ADMIN_LINK}`, + ), ); process.exit(1); } diff --git a/src/lib/graphql/fetchCatalogs.ts b/src/lib/graphql/fetchCatalogs.ts index d8992af2..75502a95 100644 --- a/src/lib/graphql/fetchCatalogs.ts +++ b/src/lib/graphql/fetchCatalogs.ts @@ -1,6 +1,6 @@ -import { GraphQLClient } from "graphql-request"; -import { CATALOGS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { GraphQLClient } from 'graphql-request'; +import { CATALOGS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Catalog { /** Integration name */ @@ -20,7 +20,7 @@ const PAGE_SIZE = 100; * @returns Integration catalogs */ export async function fetchAllCatalogs( - client: GraphQLClient + client: GraphQLClient, ): Promise { const catalogs: Catalog[] = []; let offset = 0; @@ -45,7 +45,7 @@ export async function fetchAllCatalogs( shouldContinue = nodes.length === PAGE_SIZE; } while (shouldContinue); return catalogs.sort((a, b) => - a.integrationName.localeCompare(b.integrationName) + a.integrationName.localeCompare(b.integrationName), ); } @@ -76,7 +76,7 @@ export async function fetchAndIndexCatalogs(client: GraphQLClient): Promise< catalogs.map>((catalog) => [ catalog.integrationName, catalog.title, - ]) + ]), ); // Create mapping from service name to boolean indicate if service has API integration support @@ -84,7 +84,7 @@ export async function fetchAndIndexCatalogs(client: GraphQLClient): Promise< catalogs.map>((catalog) => [ catalog.integrationName, catalog.hasApiFunctionality, - ]) + ]), ); return { diff --git a/src/lib/graphql/fetchDataSubjects.ts b/src/lib/graphql/fetchDataSubjects.ts index 3d36bc9d..fcfcb0ed 100644 --- a/src/lib/graphql/fetchDataSubjects.ts +++ b/src/lib/graphql/fetchDataSubjects.ts @@ -1,12 +1,12 @@ -import { RequestActionObjectResolver } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { difference, flatten, keyBy, uniq } from "lodash-es"; -import { TranscendInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { CREATE_DATA_SUBJECT, DATA_SUBJECTS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { RequestActionObjectResolver } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference, flatten, keyBy, uniq } from 'lodash-es'; +import { TranscendInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { CREATE_DATA_SUBJECT, DATA_SUBJECTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface DataSubject { /** ID of data subject */ @@ -36,7 +36,7 @@ export interface DataSubject { * @returns List of data subject configurations */ export async function fetchAllDataSubjects( - client: GraphQLClient + client: GraphQLClient, ): Promise { // Fetch all data subjects in the organization const { internalSubjects } = await makeGraphQLRequest<{ @@ -56,18 +56,18 @@ export async function fetchAllDataSubjects( */ export async function ensureAllDataSubjectsExist( { - "data-silos": dataSilos = [], - "data-subjects": dataSubjects = [], + 'data-silos': dataSilos = [], + 'data-subjects': dataSubjects = [], enrichers = [], }: TranscendInput, client: GraphQLClient, - fetchAll = false + fetchAll = false, ): Promise> { // Only need to fetch data subjects if specified in config const expectedDataSubjects = uniq([ - ...flatten(dataSilos.map((silo) => silo["data-subjects"] || []) || []), + ...flatten(dataSilos.map((silo) => silo['data-subjects'] || []) || []), ...flatten( - enrichers.map((enricher) => enricher["data-subjects"] || []) || [] + enrichers.map((enricher) => enricher['data-subjects'] || []) || [], ), ...dataSubjects.map((subject) => subject.type), ]); @@ -77,20 +77,20 @@ export async function ensureAllDataSubjectsExist( // Fetch all data subjects in the organization const internalSubjects = await fetchAllDataSubjects(client); - const dataSubjectByName = keyBy(internalSubjects, "type"); + const dataSubjectByName = keyBy(internalSubjects, 'type'); // Determine expected set of data subjects to create const missingDataSubjects = difference( expectedDataSubjects, - internalSubjects.map(({ type }) => type) + internalSubjects.map(({ type }) => type), ); // If there are missing data subjects, create new ones if (missingDataSubjects.length > 0) { logger.info( colors.magenta( - `Creating ${missingDataSubjects.length} new data subjects...` - ) + `Creating ${missingDataSubjects.length} new data subjects...`, + ), ); await mapSeries(missingDataSubjects, async (dataSubject) => { logger.info(colors.magenta(`Creating data subject ${dataSubject}...`)); @@ -122,7 +122,7 @@ export async function ensureAllDataSubjectsExist( */ export function convertToDataSubjectBlockList( dataSubjectTypes: string[], - allDataSubjects: Record + allDataSubjects: Record, ): string[] { for (const type of dataSubjectTypes) { if (!allDataSubjects[type]) { @@ -144,7 +144,7 @@ export function convertToDataSubjectBlockList( */ export function convertToDataSubjectAllowlist( dataSubjectTypes: string[], - allDataSubjects: Record + allDataSubjects: Record, ): string[] { for (const type of dataSubjectTypes) { if (!allDataSubjects[type]) { diff --git a/src/lib/graphql/fetchIdentifiers.ts b/src/lib/graphql/fetchIdentifiers.ts index d17e0913..9ff7b3ef 100644 --- a/src/lib/graphql/fetchIdentifiers.ts +++ b/src/lib/graphql/fetchIdentifiers.ts @@ -1,12 +1,12 @@ -import { IdentifierType, RequestAction } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { difference, flatten, keyBy, uniq } from "lodash-es"; -import { TranscendInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { CREATE_IDENTIFIER, IDENTIFIERS, NEW_IDENTIFIER_TYPES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { IdentifierType, RequestAction } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference, flatten, keyBy, uniq } from 'lodash-es'; +import { TranscendInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { CREATE_IDENTIFIER, IDENTIFIERS, NEW_IDENTIFIER_TYPES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Identifier { /** ID of identifier */ @@ -53,7 +53,7 @@ const PAGE_SIZE = 20; * @returns All identifiers in the organization */ export async function fetchAllIdentifiers( - client: GraphQLClient + client: GraphQLClient, ): Promise { const identifiers: Identifier[] = []; let offset = 0; @@ -93,38 +93,40 @@ export async function fetchAllIdentifiers( export async function fetchIdentifiersAndCreateMissing( { enrichers = [], - "data-silos": dataSilos = [], + 'data-silos': dataSilos = [], identifiers = [], }: TranscendInput, client: GraphQLClient, - skipPublish = false + skipPublish = false, ): Promise> { // Grab all existing identifiers const allIdentifiers = await fetchAllIdentifiers(client); // Create a map - const identifiersByName = keyBy(allIdentifiers, "name"); + const identifiersByName = keyBy(allIdentifiers, 'name'); // Determine expected set of identifiers const expectedIdentifiers = uniq([ ...flatten( enrichers.map((enricher) => [ - enricher["input-identifier"], - ...enricher["output-identifiers"], - ]) + enricher['input-identifier'], + ...enricher['output-identifiers'], + ]), ), - ...flatten(dataSilos.map((dataSilo) => dataSilo["identity-keys"])), + ...flatten(dataSilos.map((dataSilo) => dataSilo['identity-keys'])), ...identifiers.map(({ name }) => name), ]).filter((x) => !!x); const missingIdentifiers = difference( expectedIdentifiers, - allIdentifiers.map(({ name }) => name) + allIdentifiers.map(({ name }) => name), ); // If there are missing identifiers, create new ones if (missingIdentifiers.length > 0) { logger.info( - colors.magenta(`Creating ${missingIdentifiers.length} new identifiers...`) + colors.magenta( + `Creating ${missingIdentifiers.length} new identifiers...`, + ), ); const { newIdentifierTypes } = await makeGraphQLRequest<{ /** Query response */ @@ -134,7 +136,7 @@ export async function fetchIdentifiersAndCreateMissing( }[]; }>(client, NEW_IDENTIFIER_TYPES); const nativeTypesRemaining = new Set( - newIdentifierTypes.map(({ name }) => name) + newIdentifierTypes.map(({ name }) => name), ); await mapSeries(missingIdentifiers, async (identifier) => { logger.info(colors.magenta(`Creating identifier ${identifier}...`)); @@ -147,7 +149,7 @@ export async function fetchIdentifiersAndCreateMissing( }>(client, CREATE_IDENTIFIER, { input: { name: identifier, - type: nativeTypesRemaining.has(identifier!) ? identifier : "custom", + type: nativeTypesRemaining.has(identifier!) ? identifier : 'custom', skipPublish, }, }); diff --git a/src/lib/graphql/fetchPrompts.ts b/src/lib/graphql/fetchPrompts.ts index 6d26c475..fbe02686 100644 --- a/src/lib/graphql/fetchPrompts.ts +++ b/src/lib/graphql/fetchPrompts.ts @@ -1,10 +1,10 @@ import { PromptResponseFormat, PromptStatus, -} from "@transcend-io/privacy-types"; -import { GraphQLClient } from "graphql-request"; -import { PROMPTS, PROMPTS_WITH_VARIABLES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { PROMPTS, PROMPTS_WITH_VARIABLES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Prompt { /** ID of prompt */ @@ -45,7 +45,7 @@ export async function fetchAllPrompts( ids?: string[]; /** Filter by titles */ titles?: string[]; - } = {} + } = {}, ): Promise { const prompts: Prompt[] = []; let offset = 0; @@ -165,7 +165,7 @@ export async function fetchPromptsWithVariables( promptIds?: string[]; /** Filter by prompt titles */ promptTitles?: string[]; - } = {} + } = {}, ): Promise { const { promptsWithVariables } = await makeGraphQLRequest<{ /** Prompts */ diff --git a/src/lib/graphql/fetchRequestDataSilo.ts b/src/lib/graphql/fetchRequestDataSilo.ts index bd1c3c31..5e01d303 100644 --- a/src/lib/graphql/fetchRequestDataSilo.ts +++ b/src/lib/graphql/fetchRequestDataSilo.ts @@ -1,13 +1,13 @@ import { RequestDataSiloStatus, RequestStatus, -} from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { logger } from "../../logger"; -import { REQUEST_DATA_SILOS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { logger } from '../../logger'; +import { REQUEST_DATA_SILOS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface RequestDataSilo { /** ID of RequestDataSilo */ @@ -44,13 +44,13 @@ export async function fetchRequestDataSilos( requestStatuses?: RequestStatus[]; /** When true, skip logging */ skipLog?: boolean; - } + }, ): Promise { // create a new progress bar instance and use shades_classic theme const t0 = Date.now(); const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); const requestDataSilos: RequestDataSilo[] = []; @@ -101,8 +101,8 @@ export async function fetchRequestDataSilos( colors.green( `Completed fetching of ${ requestDataSilos.length - } request data silos in "${totalTime / 1000}" seconds.` - ) + } request data silos in "${totalTime / 1000}" seconds.`, + ), ); } @@ -126,7 +126,7 @@ export async function fetchRequestDataSilo( requestId: string; /** Data silo ID */ dataSiloId: string; - } + }, ): Promise { const nodes = await fetchRequestDataSilos(client, { requestId, @@ -135,7 +135,7 @@ export async function fetchRequestDataSilo( }); if (nodes.length !== 1) { throw new Error( - `Failed to find RequestDataSilo with requestId:${requestId},dataSiloId:${dataSiloId}` + `Failed to find RequestDataSilo with requestId:${requestId},dataSiloId:${dataSiloId}`, ); } diff --git a/src/lib/graphql/formatAttributeValues.ts b/src/lib/graphql/formatAttributeValues.ts index f3e0a96f..87af79a8 100644 --- a/src/lib/graphql/formatAttributeValues.ts +++ b/src/lib/graphql/formatAttributeValues.ts @@ -1,4 +1,4 @@ -import type { DataSiloAttributeValue } from "./syncDataSilos"; +import type { DataSiloAttributeValue } from './syncDataSilos'; export interface FormattedAttribute { /** Attribute key */ @@ -14,13 +14,13 @@ export interface FormattedAttribute { * @returns formatted attributes */ export function formatAttributeValues( - vals: DataSiloAttributeValue[] + vals: DataSiloAttributeValue[], ): FormattedAttribute[] { const attributes: FormattedAttribute[] = []; vals.map((value) => { let foundKey = attributes.find( - (att) => att.key === value.attributeKey.name + (att) => att.key === value.attributeKey.name, ); if (foundKey === undefined) { diff --git a/src/lib/graphql/gqls/consentManager.ts b/src/lib/graphql/gqls/consentManager.ts index da2af09f..72b4428b 100644 --- a/src/lib/graphql/gqls/consentManager.ts +++ b/src/lib/graphql/gqls/consentManager.ts @@ -1,4 +1,4 @@ -import { gql } from "graphql-request"; +import { gql } from 'graphql-request'; // TODO: https://transcend.height.app/T-27909 - order by createdAt // TODO: https://transcend.height.app/T-27909 - enable optimizations diff --git a/src/lib/graphql/loginUser.ts b/src/lib/graphql/loginUser.ts index 1bd6afb9..cb1cb462 100644 --- a/src/lib/graphql/loginUser.ts +++ b/src/lib/graphql/loginUser.ts @@ -1,6 +1,6 @@ -import { GraphQLClient } from "graphql-request"; -import { ASSUME_ROLE, DETERMINE_LOGIN_METHOD, LOGIN } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { GraphQLClient } from 'graphql-request'; +import { ASSUME_ROLE, DETERMINE_LOGIN_METHOD, LOGIN } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface OrganizationPreview { /** Name of organization */ @@ -37,7 +37,7 @@ export async function loginUser( email: string; /** Password of user */ password: string; - } + }, ): Promise<{ /** Cookie to be used to make subsequent requests */ loginCookie: string; @@ -80,9 +80,9 @@ export async function loginUser( } = res.data; // Get login cookie from response - const loginCookie = res.headers.get("set-cookie"); - if (!loginCookie?.includes("laravel")) { - throw new Error("Failed to get login cookie in response"); + const loginCookie = res.headers.get('set-cookie'); + if (!loginCookie?.includes('laravel')) { + throw new Error('Failed to get login cookie in response'); } return { @@ -107,7 +107,7 @@ export async function assumeRole( email: string; /** Role of user assuming into */ roleId: string; - } + }, ): Promise { const { determineLoginMethod: { loginMethod }, diff --git a/src/lib/graphql/makeGraphQLRequest.ts b/src/lib/graphql/makeGraphQLRequest.ts index a4bfe985..ea13b0f8 100644 --- a/src/lib/graphql/makeGraphQLRequest.ts +++ b/src/lib/graphql/makeGraphQLRequest.ts @@ -1,10 +1,10 @@ -import colors from "colors"; +import colors from 'colors'; import type { GraphQLClient, RequestDocument, Variables, -} from "graphql-request"; -import { logger } from "../../logger"; +} from 'graphql-request'; +import { logger } from '../../logger'; const MAX_RETRIES = 4; @@ -23,11 +23,11 @@ function sleepPromise(sleepTime: number): Promise { } const KNOWN_ERRORS = [ - "syntax error", - "got invalid value", - "Client error", - "cannot affect row a second time", - "GRAPHQL_VALIDATION_FAILED", + 'syntax error', + 'got invalid value', + 'Client error', + 'cannot affect row a second time', + 'GRAPHQL_VALIDATION_FAILED', ]; /** @@ -45,7 +45,7 @@ export async function makeGraphQLRequest( document: RequestDocument, variables?: V, requestHeaders?: Record | string[][] | Headers, - maxRequests = MAX_RETRIES + maxRequests = MAX_RETRIES, ): Promise { let retryCount = 0; @@ -54,13 +54,13 @@ export async function makeGraphQLRequest( const result = await client.request(document, variables, requestHeaders); return result as T; } catch (error) { - if (error.message.includes("API key is invalid")) { + if (error.message.includes('API key is invalid')) { logger.error( colors.red( - "API key is invalid. " + - "Please ensure that the key provided to `transcendAuth` has the proper scope and is not expired, " + - "and that `transcendUrl` corresponds to the correct backend for your organization." - ) + 'API key is invalid. ' + + 'Please ensure that the key provided to `transcendAuth` has the proper scope and is not expired, ' + + 'and that `transcendUrl` corresponds to the correct backend for your organization.', + ), ); process.exit(1); } @@ -70,16 +70,16 @@ export async function makeGraphQLRequest( } // wait for rate limit to resolve - if (error.message.startsWith("Client error: Too many requests")) { + if (error.message.startsWith('Client error: Too many requests')) { const rateLimitResetAt = - error.response.headers?.get("x-ratelimit-reset"); + error.response.headers?.get('x-ratelimit-reset'); const sleepTime = rateLimitResetAt ? new Date(rateLimitResetAt).getTime() - Date.now() + 100 : 1000 * 10; logger.log( colors.yellow( - `DETECTED RATE LIMIT: ${error.message}. Sleeping for ${sleepTime}ms` - ) + `DETECTED RATE LIMIT: ${error.message}. Sleeping for ${sleepTime}ms`, + ), ); await sleepPromise(sleepTime); @@ -91,8 +91,8 @@ export async function makeGraphQLRequest( retryCount += 1; logger.log( colors.yellow( - `REQUEST FAILED: ${error.message}. Retrying ${retryCount}/${maxRequests}...` - ) + `REQUEST FAILED: ${error.message}. Retrying ${retryCount}/${maxRequests}...`, + ), ); } } diff --git a/src/lib/graphql/pullTranscendConfiguration.ts b/src/lib/graphql/pullTranscendConfiguration.ts index 4d9ffdc1..c07bc102 100644 --- a/src/lib/graphql/pullTranscendConfiguration.ts +++ b/src/lib/graphql/pullTranscendConfiguration.ts @@ -1,12 +1,12 @@ -import { LanguageKey } from "@transcend-io/internationalization"; +import { LanguageKey } from '@transcend-io/internationalization'; import { ActionItemCode, ConsentTrackerStatus, RequestAction, -} from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { flatten, keyBy, mapValues } from "lodash-es"; +} from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { flatten, keyBy, mapValues } from 'lodash-es'; import { ActionInput, ActionItemCollectionInput, @@ -41,53 +41,53 @@ import { TeamInput, TranscendInput, VendorInput, -} from "../../codecs"; -import { TranscendPullResource } from "../../enums"; -import { logger } from "../../logger"; -import { fetchAllActionItemCollections } from "./fetchAllActionItemCollections"; -import { fetchAllActionItems } from "./fetchAllActionItems"; -import { fetchAllActions } from "./fetchAllActions"; -import { fetchAllAgentFiles } from "./fetchAllAgentFiles"; -import { fetchAllAgentFunctions } from "./fetchAllAgentFunctions"; -import { fetchAllAgents } from "./fetchAllAgents"; -import { fetchAllAssessments } from "./fetchAllAssessments"; -import { fetchAllAssessmentTemplates } from "./fetchAllAssessmentTemplates"; -import { fetchAllAttributes } from "./fetchAllAttributes"; -import { fetchAllBusinessEntities } from "./fetchAllBusinessEntities"; -import { fetchAllCookies } from "./fetchAllCookies"; -import { fetchAllDataCategories } from "./fetchAllDataCategories"; -import { fetchAllDataFlows } from "./fetchAllDataFlows"; -import { fetchAllMessages } from "./fetchAllMessages"; -import { fetchAllPolicies } from "./fetchAllPolicies"; -import { fetchAllPrivacyCenters } from "./fetchAllPrivacyCenters"; -import { fetchAllProcessingPurposes } from "./fetchAllProcessingPurposes"; -import { fetchAllPurposesAndPreferences } from "./fetchAllPurposesAndPreferences"; -import { fetchAllTeams } from "./fetchAllTeams"; -import { fetchAllVendors } from "./fetchAllVendors"; -import { fetchApiKeys } from "./fetchApiKeys"; +} from '../../codecs'; +import { TranscendPullResource } from '../../enums'; +import { logger } from '../../logger'; +import { fetchAllActionItemCollections } from './fetchAllActionItemCollections'; +import { fetchAllActionItems } from './fetchAllActionItems'; +import { fetchAllActions } from './fetchAllActions'; +import { fetchAllAgentFiles } from './fetchAllAgentFiles'; +import { fetchAllAgentFunctions } from './fetchAllAgentFunctions'; +import { fetchAllAgents } from './fetchAllAgents'; +import { fetchAllAssessments } from './fetchAllAssessments'; +import { fetchAllAssessmentTemplates } from './fetchAllAssessmentTemplates'; +import { fetchAllAttributes } from './fetchAllAttributes'; +import { fetchAllBusinessEntities } from './fetchAllBusinessEntities'; +import { fetchAllCookies } from './fetchAllCookies'; +import { fetchAllDataCategories } from './fetchAllDataCategories'; +import { fetchAllDataFlows } from './fetchAllDataFlows'; +import { fetchAllMessages } from './fetchAllMessages'; +import { fetchAllPolicies } from './fetchAllPolicies'; +import { fetchAllPrivacyCenters } from './fetchAllPrivacyCenters'; +import { fetchAllProcessingPurposes } from './fetchAllProcessingPurposes'; +import { fetchAllPurposesAndPreferences } from './fetchAllPurposesAndPreferences'; +import { fetchAllTeams } from './fetchAllTeams'; +import { fetchAllVendors } from './fetchAllVendors'; +import { fetchApiKeys } from './fetchApiKeys'; import { fetchConsentManager, fetchConsentManagerExperiences, fetchConsentManagerTheme, -} from "./fetchConsentManagerId"; +} from './fetchConsentManagerId'; import { convertToDataSubjectAllowlist, fetchAllDataSubjects, -} from "./fetchDataSubjects"; -import { fetchAllIdentifiers } from "./fetchIdentifiers"; -import { fetchAllPromptGroups } from "./fetchPromptGroups"; -import { fetchAllPromptPartials } from "./fetchPromptPartials"; -import { fetchAllPrompts } from "./fetchPrompts"; -import { formatAttributeValues } from "./formatAttributeValues"; +} from './fetchDataSubjects'; +import { fetchAllIdentifiers } from './fetchIdentifiers'; +import { fetchAllPromptGroups } from './fetchPromptGroups'; +import { fetchAllPromptPartials } from './fetchPromptPartials'; +import { fetchAllPrompts } from './fetchPrompts'; +import { formatAttributeValues } from './formatAttributeValues'; import { AssessmentNestedRule, parseAssessmentDisplayLogic, -} from "./parseAssessmentDisplayLogic"; -import { parseAssessmentRiskLogic } from "./parseAssessmentRiskLogic"; -import { fetchEnrichedDataSilos } from "./syncDataSilos"; -import { fetchAllEnrichers } from "./syncEnrichers"; -import { fetchPartitions } from "./syncPartitions"; -import { fetchAllTemplates } from "./syncTemplates"; +} from './parseAssessmentDisplayLogic'; +import { parseAssessmentRiskLogic } from './parseAssessmentRiskLogic'; +import { fetchEnrichedDataSilos } from './syncDataSilos'; +import { fetchAllEnrichers } from './syncEnrichers'; +import { fetchPartitions } from './syncPartitions'; +import { fetchAllTemplates } from './syncTemplates'; export const DEFAULT_TRANSCEND_PULL_RESOURCES = [ TranscendPullResource.DataSilos, @@ -136,11 +136,11 @@ export async function pullTranscendConfiguration( includeGuessedCategories, skipSubDatapoints, trackerStatuses = Object.values(ConsentTrackerStatus), - }: TranscendPullConfigurationInput + }: TranscendPullConfigurationInput, ): Promise { if (dataSiloIds.length > 0 && integrationNames.length > 0) { throw new Error( - "Only 1 of integrationNames OR dataSiloIds can be provided" + 'Only 1 of integrationNames OR dataSiloIds can be provided', ); } @@ -343,21 +343,21 @@ export async function pullTranscendConfiguration( // Save API keys const apiKeyTitles = flatten( - dataSilos.map(([{ apiKeys }]) => apiKeys.map(({ title }) => title)) + dataSilos.map(([{ apiKeys }]) => apiKeys.map(({ title }) => title)), ); const relevantApiKeys = Object.values(apiKeyTitleMap).filter(({ title }) => resources.includes(TranscendPullResource.ApiKeys) ? true - : apiKeyTitles.includes(title) + : apiKeyTitles.includes(title), ); if ( relevantApiKeys.length > 0 && resources.includes(TranscendPullResource.ApiKeys) ) { - result["api-keys"] = relevantApiKeys.map( + result['api-keys'] = relevantApiKeys.map( ({ title }): ApiKeyInput => ({ title, - }) + }), ); } @@ -377,7 +377,7 @@ export async function pullTranscendConfiguration( consentManager && resources.includes(TranscendPullResource.ConsentManager) ) { - result["consent-manager"] = { + result['consent-manager'] = { bundleUrls: { TEST: consentManager.testBundleURL, PRODUCTION: consentManager.bundleURL, @@ -503,30 +503,30 @@ export async function pullTranscendConfiguration( return { title, type, - "sub-type": subType, + 'sub-type': subType, placeholder, description, - "is-required": isRequired, - "reference-id": referenceId, - "display-logic": + 'is-required': isRequired, + 'reference-id': referenceId, + 'display-logic': displayLogicParsed && Object.keys(displayLogicParsed).length > 0 ? { action: displayLogicParsed.action!, rule: displayLogicParsed.rule ? { - "depends-on-question-reference-id": + 'depends-on-question-reference-id': displayLogicParsed.rule .dependsOnQuestionReferenceId, - "comparison-operator": + 'comparison-operator': displayLogicParsed.rule.comparisonOperator, - "comparison-operands": + 'comparison-operands': displayLogicParsed.rule.comparisonOperands, } : undefined, - "nested-rule": displayLogicParsed.nestedRule + 'nested-rule': displayLogicParsed.nestedRule ? { - "logic-operator": + 'logic-operator': displayLogicParsed.nestedRule.logicOperator, rules: /* eslint-disable @typescript-eslint/no-explicit-any */ @@ -536,11 +536,11 @@ export async function pullTranscendConfiguration( .nestedRule as AssessmentNestedRule ).rules || [] ).map((rule) => ({ - "depends-on-question-reference-id": + 'depends-on-question-reference-id': rule.dependsOnQuestionReferenceId, - "comparison-operator": + 'comparison-operator': rule.comparisonOperator, - "comparison-operands": + 'comparison-operands': rule.comparisonOperands, })), /* eslint-enable @typescript-eslint/no-explicit-any */ @@ -548,55 +548,55 @@ export async function pullTranscendConfiguration( : undefined, } : undefined, - "risk-logic": riskLogic.map((logic): RiskLogicInput => { + 'risk-logic': riskLogic.map((logic): RiskLogicInput => { const parsed = parseAssessmentRiskLogic(logic); return { - "risk-level": parsed.riskAssignment?.riskLevelId, - "comparison-operands": parsed.comparisonOperands, - "comparison-operator": parsed.comparisonOperator, + 'risk-level': parsed.riskAssignment?.riskLevelId, + 'comparison-operands': parsed.comparisonOperands, + 'comparison-operator': parsed.comparisonOperator, }; }), - "risk-categories": riskCategories.map(({ title }) => title), - "risk-framework": riskFramework?.title, - "answer-options": answerOptions.map(({ value }) => ({ + 'risk-categories': riskCategories.map(({ title }) => title), + 'risk-framework': riskFramework?.title, + 'answer-options': answerOptions.map(({ value }) => ({ value, })), - "selected-answers": selectedAnswers.map(({ value }) => value), - "allowed-mime-types": allowedMimeTypes, - "allow-select-other": allowSelectOther, - "sync-model": syncModel || undefined, - "sync-column": syncColumn || undefined, - "attribute-key": attributeKey?.name, - "require-risk-evaluation": requireRiskEvaluation, - "require-risk-matrix-evaluation": requireRiskMatrixEvaluation, + 'selected-answers': selectedAnswers.map(({ value }) => value), + 'allowed-mime-types': allowedMimeTypes, + 'allow-select-other': allowSelectOther, + 'sync-model': syncModel || undefined, + 'sync-column': syncColumn || undefined, + 'attribute-key': attributeKey?.name, + 'require-risk-evaluation': requireRiskEvaluation, + 'require-risk-matrix-evaluation': requireRiskMatrixEvaluation, }; - } + }, ), assignees: assignees.map(({ email }) => email), - "external-assignees": externalAssignees.map(({ email }) => email), - "is-reviewed": isReviewed, - }) + 'external-assignees': externalAssignees.map(({ email }) => email), + 'is-reviewed': isReviewed, + }), ), creator: creator?.email, description, status, assignees: assignees.map(({ email }) => email), - "external-assignees": externalAssignees.map(({ email }) => email), + 'external-assignees': externalAssignees.map(({ email }) => email), reviewers: reviewers.map(({ email }) => email), locked: isLocked, archived: isArchived, external: isExternallyCreated, - "title-is-internal": titleIsInternal, - "due-date": dueDate || undefined, - "created-at": createdAt || undefined, - "assigned-at": assignedAt || undefined, - "submitted-at": submittedAt || undefined, - "approved-at": approvedAt || undefined, - "rejected-at": rejectedAt || undefined, - "retention-schedule": retentionSchedule + 'title-is-internal': titleIsInternal, + 'due-date': dueDate || undefined, + 'created-at': createdAt || undefined, + 'assigned-at': assignedAt || undefined, + 'submitted-at': submittedAt || undefined, + 'approved-at': approvedAt || undefined, + 'rejected-at': rejectedAt || undefined, + 'retention-schedule': retentionSchedule ? { type: retentionSchedule.type, - "duration-days": retentionSchedule.durationDays, + 'duration-days': retentionSchedule.durationDays, operand: retentionSchedule.operation, } : undefined, @@ -610,9 +610,9 @@ export async function pullTranscendConfiguration( title: category ? `${category} - ${name}` : purpose - ? `${purpose} - ${name}` - : title || name || type || "", - }) + ? `${purpose} - ${name}` + : title || name || type || '', + }), ), rows: syncedRows.map( ({ resourceType, title, name, category, type, purpose }) => ({ @@ -620,11 +620,11 @@ export async function pullTranscendConfiguration( title: category ? `${category} - ${name}` : purpose - ? `${purpose} - ${name}` - : title || name || type || "", - }) + ? `${purpose} - ${name}` + : title || name || type || '', + }), ), - }) + }), ); } @@ -633,7 +633,7 @@ export async function pullTranscendConfiguration( assessmentTemplates.length > 0 && resources.includes(TranscendPullResource.AssessmentTemplates) ) { - result["assessment-templates"] = assessmentTemplates.map( + result['assessment-templates'] = assessmentTemplates.map( ({ title, description, @@ -679,30 +679,30 @@ export async function pullTranscendConfiguration( return { title, type, - "sub-type": subType, + 'sub-type': subType, placeholder, description, - "is-required": isRequired, - "reference-id": referenceId, - "display-logic": + 'is-required': isRequired, + 'reference-id': referenceId, + 'display-logic': displayLogicParsed && Object.keys(displayLogicParsed).length > 0 ? { action: displayLogicParsed.action!, rule: displayLogicParsed.rule ? { - "depends-on-question-reference-id": + 'depends-on-question-reference-id': displayLogicParsed.rule .dependsOnQuestionReferenceId, - "comparison-operator": + 'comparison-operator': displayLogicParsed.rule.comparisonOperator, - "comparison-operands": + 'comparison-operands': displayLogicParsed.rule.comparisonOperands, } : undefined, - "nested-rule": displayLogicParsed.nestedRule + 'nested-rule': displayLogicParsed.nestedRule ? { - "logic-operator": + 'logic-operator': displayLogicParsed.nestedRule.logicOperator, rules: /* eslint-disable @typescript-eslint/no-explicit-any */ @@ -712,11 +712,11 @@ export async function pullTranscendConfiguration( .nestedRule as AssessmentNestedRule ).rules || [] ).map((rule) => ({ - "depends-on-question-reference-id": + 'depends-on-question-reference-id': rule.dependsOnQuestionReferenceId, - "comparison-operator": + 'comparison-operator': rule.comparisonOperator, - "comparison-operands": + 'comparison-operands': rule.comparisonOperands, })), /* eslint-enable @typescript-eslint/no-explicit-any */ @@ -724,48 +724,48 @@ export async function pullTranscendConfiguration( : undefined, } : undefined, - "risk-logic": riskLogic.map((logic): RiskLogicInput => { + 'risk-logic': riskLogic.map((logic): RiskLogicInput => { const parsed = parseAssessmentRiskLogic(logic); return { - "risk-level": parsed.riskAssignment?.riskLevelId, - "risk-matrix-row": parsed.riskAssignment?.riskMatrixRowId, - "risk-matrix-column": + 'risk-level': parsed.riskAssignment?.riskLevelId, + 'risk-matrix-row': parsed.riskAssignment?.riskMatrixRowId, + 'risk-matrix-column': parsed.riskAssignment?.riskMatrixColumnId, - "comparison-operands": parsed.comparisonOperands, - "comparison-operator": parsed.comparisonOperator, + 'comparison-operands': parsed.comparisonOperands, + 'comparison-operator': parsed.comparisonOperator, }; }), - "risk-categories": riskCategories.map(({ title }) => title), - "risk-framework": riskFramework?.title, - "answer-options": answerOptions.map(({ value }) => ({ + 'risk-categories': riskCategories.map(({ title }) => title), + 'risk-framework': riskFramework?.title, + 'answer-options': answerOptions.map(({ value }) => ({ value, })), - "allowed-mime-types": allowedMimeTypes, - "allow-select-other": allowSelectOther, - "sync-model": syncModel || undefined, - "sync-column": syncColumn || undefined, - "attribute-key": attributeKey?.name, - "require-risk-evaluation": requireRiskEvaluation, - "require-risk-matrix-evaluation": requireRiskMatrixEvaluation, + 'allowed-mime-types': allowedMimeTypes, + 'allow-select-other': allowSelectOther, + 'sync-model': syncModel || undefined, + 'sync-column': syncColumn || undefined, + 'attribute-key': attributeKey?.name, + 'require-risk-evaluation': requireRiskEvaluation, + 'require-risk-matrix-evaluation': requireRiskMatrixEvaluation, }; - } + }, ), - }) + }), ), status, source, creator: creator?.email, locked: isLocked, archived: isArchived, - "created-at": createdAt || undefined, - "retention-schedule": retentionSchedule + 'created-at': createdAt || undefined, + 'retention-schedule': retentionSchedule ? { type: retentionSchedule.type, - "duration-days": retentionSchedule.durationDays, + 'duration-days': retentionSchedule.durationDays, operand: retentionSchedule.operation, } : undefined, - }) + }), ); } @@ -775,7 +775,7 @@ export async function pullTranscendConfiguration( ({ title, content }): PromptInput => ({ title, content, - }) + }), ); } @@ -784,11 +784,11 @@ export async function pullTranscendConfiguration( promptPartials.length > 0 && resources.includes(TranscendPullResource.PromptPartials) ) { - result["prompt-partials"] = promptPartials.map( + result['prompt-partials'] = promptPartials.map( ({ title, content }): PromptPartialInput => ({ title, content, - }) + }), ); } @@ -797,12 +797,12 @@ export async function pullTranscendConfiguration( promptGroups.length > 0 && resources.includes(TranscendPullResource.PromptGroups) ) { - result["prompt-groups"] = promptGroups.map( + result['prompt-groups'] = promptGroups.map( ({ title, description, prompts }): PromptGroupInput => ({ title, description, prompts: prompts.map(({ title }) => title), - }) + }), ); } @@ -820,12 +820,12 @@ export async function pullTranscendConfiguration( }): TeamInput => ({ name, description, - "sso-department": ssoDepartment || undefined, - "sso-group": ssoGroup || undefined, - "sso-title": ssoTitle || undefined, + 'sso-department': ssoDepartment || undefined, + 'sso-group': ssoGroup || undefined, + 'sso-title': ssoTitle || undefined, users: users.map(({ email }) => email), scopes: scopes.map(({ name }) => name), - }) + }), ); } @@ -834,7 +834,7 @@ export async function pullTranscendConfiguration( dataSubjects.length > 0 && resources.includes(TranscendPullResource.DataSubjects) ) { - result["data-subjects"] = dataSubjects.map( + result['data-subjects'] = dataSubjects.map( ({ type, title, @@ -847,7 +847,7 @@ export async function pullTranscendConfiguration( active, adminDashboardDefaultSilentMode, actions: actions.map(({ type }) => type), - }) + }), ); } @@ -858,7 +858,7 @@ export async function pullTranscendConfiguration( title: title?.defaultMessage, content: versions?.[0]?.content?.defaultMessage, disabledLocales, - }) + }), ); } @@ -877,16 +877,16 @@ export async function pullTranscendConfiguration( translations: translations.reduce( (accumulator, { locale, value }) => Object.assign(accumulator, { [locale]: value }), - {} as Record + {} as Record, ), - }) + }), ); } // Save privacy center if (privacyCenters.length > 0) { const privacyCenter = privacyCenters[0]; - result["privacy-center"] = { + result['privacy-center'] = { isDisabled: privacyCenter.isDisabled, showPrivacyRequestButton: privacyCenter.showPrivacyRequestButton, showPolicies: privacyCenter.showPolicies, @@ -914,7 +914,7 @@ export async function pullTranscendConfiguration( businessEntities.length > 0 && resources.includes(TranscendPullResource.BusinessEntities) ) { - result["business-entities"] = businessEntities.map( + result['business-entities'] = businessEntities.map( ({ title, description, @@ -936,7 +936,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -963,7 +963,7 @@ export async function pullTranscendConfiguration( waitingPeriod, regionDetectionMethod, regionList: regionList.length > 0 ? regionList : undefined, - }) + }), ); } @@ -1003,7 +1003,7 @@ export async function pullTranscendConfiguration( displayTitle: displayTitle?.defaultMessage, displayDescription: displayDescription?.defaultMessage, displayOrder, - }) + }), ); } @@ -1031,7 +1031,7 @@ export async function pullTranscendConfiguration( codeInterpreterEnabled, retrievalEnabled, prompt: prompt?.title, - "large-language-model": { + 'large-language-model': { name: largeLanguageModel.name, client: largeLanguageModel.client, }, @@ -1041,15 +1041,15 @@ export async function pullTranscendConfiguration( owners && owners.length > 0 ? owners.map(({ email }) => email) : undefined, - "agent-functions": + 'agent-functions': agentFunctions && agentFunctions.length > 0 ? agentFunctions.map(({ name }) => name) : undefined, - "agent-files": + 'agent-files': agentFiles && agentFiles.length > 0 ? agentFiles.map(({ name }) => name) : undefined, - }) + }), ); } @@ -1058,7 +1058,7 @@ export async function pullTranscendConfiguration( actionItems.length > 0 && resources.includes(TranscendPullResource.ActionItems) ) { - result["action-items"] = actionItems.map( + result['action-items'] = actionItems.map( ({ teams, users, @@ -1088,7 +1088,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -1097,7 +1097,7 @@ export async function pullTranscendConfiguration( actionItemCollections.length > 0 && resources.includes(TranscendPullResource.ActionItemCollections) ) { - result["action-item-collections"] = actionItemCollections.map( + result['action-item-collections'] = actionItemCollections.map( ({ title, description, @@ -1108,7 +1108,7 @@ export async function pullTranscendConfiguration( description: description || undefined, hidden, productLine, - }) + }), ); } @@ -1117,12 +1117,12 @@ export async function pullTranscendConfiguration( agentFunctions.length > 0 && resources.includes(TranscendPullResource.AgentFunctions) ) { - result["agent-functions"] = agentFunctions.map( + result['agent-functions'] = agentFunctions.map( ({ name, description, parameters }): AgentFunctionInput => ({ name, description, parameters: JSON.stringify(parameters), - }) + }), ); } @@ -1131,14 +1131,14 @@ export async function pullTranscendConfiguration( agentFiles.length > 0 && resources.includes(TranscendPullResource.AgentFiles) ) { - result["agent-files"] = agentFiles.map( + result['agent-files'] = agentFiles.map( ({ name, description, fileId, size, purpose }): AgentFileInput => ({ name, description, fileId, size, purpose, - }) + }), ); } @@ -1180,7 +1180,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -1189,7 +1189,7 @@ export async function pullTranscendConfiguration( dataCategories.length > 0 && resources.includes(TranscendPullResource.DataCategories) ) { - result["data-categories"] = dataCategories.map( + result['data-categories'] = dataCategories.map( ({ name, category, @@ -1213,7 +1213,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -1222,7 +1222,7 @@ export async function pullTranscendConfiguration( processingPurposes.length > 0 && resources.includes(TranscendPullResource.ProcessingPurposes) ) { - result["processing-purposes"] = processingPurposes.map( + result['processing-purposes'] = processingPurposes.map( ({ name, purpose, @@ -1244,7 +1244,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -1253,7 +1253,7 @@ export async function pullTranscendConfiguration( dataFlows.length > 0 && resources.includes(TranscendPullResource.DataFlows) ) { - result["data-flows"] = dataFlows.map( + result['data-flows'] = dataFlows.map( ({ value, type, @@ -1277,7 +1277,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -1307,7 +1307,7 @@ export async function pullTranscendConfiguration( attributeValues !== undefined && attributeValues.length > 0 ? formatAttributeValues(attributeValues) : undefined, - }) + }), ); } @@ -1333,7 +1333,7 @@ export async function pullTranscendConfiguration( color: color || undefined, description, })), - }) + }), ); } @@ -1359,15 +1359,15 @@ export async function pullTranscendConfiguration( title, description: description || undefined, trackingType, - "default-consent": defaultConsent, + 'default-consent': defaultConsent, configurable, - "show-in-consent-manager": showInConsentManager, - "show-in-privacy-center": showInPrivacyCenter, - "is-active": isActive, - "display-order": displayOrder, - "opt-out-signals": optOutSignals.length > 0 ? optOutSignals : undefined, - "auth-level": authLevel || undefined, - "preference-topics": topics.map( + 'show-in-consent-manager': showInConsentManager, + 'show-in-privacy-center': showInPrivacyCenter, + 'is-active': isActive, + 'display-order': displayOrder, + 'opt-out-signals': optOutSignals.length > 0 ? optOutSignals : undefined, + 'auth-level': authLevel || undefined, + 'preference-topics': topics.map( ({ title, type, @@ -1379,8 +1379,8 @@ export async function pullTranscendConfiguration( title: title.defaultMessage, type, description: displayDescription.defaultMessage, - "default-configuration": defaultConfiguration, - "show-in-privacy-center": showInPrivacyCenter, + 'default-configuration': defaultConfiguration, + 'show-in-privacy-center': showInPrivacyCenter, ...(preferenceOptionValues.length > 0 ? { options: preferenceOptionValues.map(({ title, slug }) => ({ @@ -1389,9 +1389,9 @@ export async function pullTranscendConfiguration( })), } : {}), - }) + }), ), - }) + }), ); } @@ -1428,9 +1428,9 @@ export async function pullTranscendConfiguration( title, url: url || undefined, type, - "input-identifier": inputIdentifier?.name, - "output-identifiers": identifiers.map(({ name }) => name), - "privacy-actions": + 'input-identifier': inputIdentifier?.name, + 'output-identifiers': identifiers.map(({ name }) => name), + 'privacy-actions': Object.values(RequestAction).length === actions.length ? undefined : actions, @@ -1442,8 +1442,8 @@ export async function pullTranscendConfiguration( phoneNumbers && phoneNumbers.length > 0 ? phoneNumbers : undefined, regionList: regionList && regionList.length > 0 ? regionList : undefined, - "data-subjects": dataSubjects.map(({ type }) => type), - }) + 'data-subjects': dataSubjects.map(({ type }) => type), + }), ); } @@ -1452,8 +1452,8 @@ export async function pullTranscendConfiguration( dataSilos.length > 0 && resources.includes(TranscendPullResource.DataSilos) ) { - const indexedDataSubjects = keyBy(dataSubjects, "type"); - result["data-silos"] = dataSilos.map( + const indexedDataSubjects = keyBy(dataSubjects, 'type'); + result['data-silos'] = dataSilos.map( ([ { title, @@ -1486,16 +1486,16 @@ export async function pullTranscendConfiguration( title, description, integrationName: type, - "outer-type": outerType || undefined, + 'outer-type': outerType || undefined, url: url || undefined, - "api-key-title": apiKeys[0]?.title, - "identity-keys": identifiers + 'api-key-title': apiKeys[0]?.title, + 'identity-keys': identifiers .filter(({ isConnected }) => isConnected) .map(({ name }) => name), ...(dependentDataSilos.length > 0 ? { - "deletion-dependencies": dependentDataSilos.map( - ({ title }) => title + 'deletion-dependencies': dependentDataSilos.map( + ({ title }) => title, ), } : {}), @@ -1514,23 +1514,23 @@ export async function pullTranscendConfiguration( country: country || undefined, countrySubDivision: countrySubDivision || undefined, disabled: !isLive, - "data-subjects": + 'data-subjects': subjectBlocklist.length > 0 ? convertToDataSubjectAllowlist( subjectBlocklist.map(({ type }) => type), - indexedDataSubjects + indexedDataSubjects, ) : undefined, ...(catalog.hasAvcFunctionality ? { - "email-settings": { - "notify-email-address": notifyEmailAddress || undefined, - "send-frequency": promptAVendorEmailSendFrequency, - "send-type": promptAVendorEmailSendType, - "include-identifiers-attachment": + 'email-settings': { + 'notify-email-address': notifyEmailAddress || undefined, + 'send-frequency': promptAVendorEmailSendFrequency, + 'send-type': promptAVendorEmailSendType, + 'include-identifiers-attachment': promptAVendorEmailIncludeIdentifiersAttachment, - "completion-link-type": promptAVendorEmailCompletionLinkType, - "manual-work-retry-frequency": manualWorkRetryFrequency, + 'completion-link-type': promptAVendorEmailCompletionLinkType, + 'manual-work-retry-frequency': manualWorkRetryFrequency, }, } : {}), @@ -1550,18 +1550,18 @@ export async function pullTranscendConfiguration( ...(dataPoint.path.length > 0 ? { path: dataPoint.path } : {}), ...(dataPoint.dataCollection?.title ? { - "data-collection-tag": + 'data-collection-tag': dataPoint.dataCollection.title.defaultMessage, } : {}), ...(dataPoint.dbIntegrationQueries.length > 0 ? { - "privacy-action-queries": mapValues( - keyBy(dataPoint.dbIntegrationQueries, "requestType"), + 'privacy-action-queries': mapValues( + keyBy(dataPoint.dbIntegrationQueries, 'requestType'), (databaseIntegrationQuery) => databaseIntegrationQuery.suggestedQuery || databaseIntegrationQuery.query || - undefined + undefined, ), } : {}), @@ -1577,10 +1577,10 @@ export async function pullTranscendConfiguration( ...(includeGuessedCategories && field.pendingCategoryGuesses ? { - "guessed-categories": + 'guessed-categories': field.pendingCategoryGuesses .filter( - (guess) => guess.status === "PENDING" + (guess) => guess.status === 'PENDING', ) .map((guess) => ({ category: { @@ -1594,31 +1594,31 @@ export async function pullTranscendConfiguration( })), } : {}), - "access-request-visibility-enabled": + 'access-request-visibility-enabled': field.accessRequestVisibilityEnabled, - "erasure-request-redaction-enabled": + 'erasure-request-redaction-enabled': field.erasureRequestRedactionEnabled, attributes: field.attributeValues !== undefined && field.attributeValues.length > 0 ? formatAttributeValues(field.attributeValues) : undefined, - }) + }), ) .sort((a, b) => a.key.localeCompare(b.key)), } : {}), - "privacy-actions": dataPoint.actionSettings + 'privacy-actions': dataPoint.actionSettings .filter(({ active }) => active) .map(({ type }) => type), - }) + }), ) .sort((a, b) => [...(a.path ?? []), a.key] - .join(".") - .localeCompare([...(b.path ?? []), b.key].join(".")) + .join('.') + .localeCompare([...(b.path ?? []), b.key].join('.')), ), - }) + }), ); } return result; diff --git a/src/lib/graphql/syncActionItemCollections.ts b/src/lib/graphql/syncActionItemCollections.ts index 5e31b829..1f4db10d 100644 --- a/src/lib/graphql/syncActionItemCollections.ts +++ b/src/lib/graphql/syncActionItemCollections.ts @@ -1,18 +1,18 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { ActionItemCollectionInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { ActionItemCollectionInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { ActionItemCollection, fetchAllActionItemCollections, -} from "./fetchAllActionItemCollections"; +} from './fetchAllActionItemCollections'; import { CREATE_ACTION_ITEM_COLLECTION, UPDATE_ACTION_ITEM_COLLECTION, -} from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new action item collection @@ -23,11 +23,11 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createActionItemCollection( client: GraphQLClient, - actionItemCollection: ActionItemCollectionInput -): Promise> { + actionItemCollection: ActionItemCollectionInput, +): Promise> { const input = { title: actionItemCollection.title, - description: actionItemCollection.description || "", + description: actionItemCollection.description || '', hidden: actionItemCollection.hidden || false, productLine: actionItemCollection.productLine, }; @@ -54,7 +54,7 @@ export async function createActionItemCollection( export async function updateActionItemCollection( client: GraphQLClient, input: ActionItemCollectionInput, - actionItemCollectionId: string + actionItemCollectionId: string, ): Promise { await makeGraphQLRequest(client, UPDATE_ACTION_ITEM_COLLECTION, { input: { @@ -76,28 +76,27 @@ export async function updateActionItemCollection( */ export async function syncActionItemCollections( client: GraphQLClient, - inputs: ActionItemCollectionInput[] + inputs: ActionItemCollectionInput[], ): Promise { let encounteredError = false; // Fetch existing logger.info( - colors.magenta(`Syncing "${inputs.length}" action item collections...`) + colors.magenta(`Syncing "${inputs.length}" action item collections...`), ); // Fetch existing - const existingActionItemCollections = await fetchAllActionItemCollections( - client - ); + const existingActionItemCollections = + await fetchAllActionItemCollections(client); // Look up by title const collectionByTitle: Record = keyBy( existingActionItemCollections, - "title" + 'title', ); // Create new actionItems const newCollections = inputs.filter( - (input) => !collectionByTitle[input.title] + (input) => !collectionByTitle[input.title], ); // Create new actionItem collections @@ -106,15 +105,15 @@ export async function syncActionItemCollections( await createActionItemCollection(client, input); logger.info( colors.green( - `Successfully created action item collection "${input.title}"!` - ) + `Successfully created action item collection "${input.title}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to create action item collection "${input.title}"! - ${error.message}` - ) + `Failed to create action item collection "${input.title}"! - ${error.message}`, + ), ); } }); @@ -128,15 +127,15 @@ export async function syncActionItemCollections( await updateActionItemCollection(client, input, actionItemId); logger.info( colors.green( - `Successfully synced action item collection "${input.title}"!` - ) + `Successfully synced action item collection "${input.title}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync action item collection "${input.title}"! - ${error.message}` - ) + `Failed to sync action item collection "${input.title}"! - ${error.message}`, + ), ); } }); diff --git a/src/lib/graphql/syncActionItems.ts b/src/lib/graphql/syncActionItems.ts index c013a2c1..e12d15d9 100644 --- a/src/lib/graphql/syncActionItems.ts +++ b/src/lib/graphql/syncActionItems.ts @@ -1,17 +1,17 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy, uniq } from "lodash-es"; -import { ActionItemInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy, uniq } from 'lodash-es'; +import { ActionItemInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { ActionItemCollection, fetchAllActionItemCollections, -} from "./fetchAllActionItemCollections"; -import { ActionItem, fetchAllActionItems } from "./fetchAllActionItems"; -import { Attribute, fetchAllAttributes } from "./fetchAllAttributes"; -import { CREATE_ACTION_ITEMS, UPDATE_ACTION_ITEMS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './fetchAllActionItemCollections'; +import { ActionItem, fetchAllActionItems } from './fetchAllActionItems'; +import { Attribute, fetchAllAttributes } from './fetchAllAttributes'; +import { CREATE_ACTION_ITEMS, UPDATE_ACTION_ITEMS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new actionItem @@ -27,7 +27,7 @@ export async function createActionItems( actionItemCollectionByTitle: Record, // TODO: https://transcend.height.app/T-38961 - insert attributes // eslint-disable-next-line @typescript-eslint/no-unused-vars - attributeKeysByName: Record = {} + attributeKeysByName: Record = {}, ): Promise { // TODO: https://transcend.height.app/T-38961 - insert attributes // const getAttribute = (key: string): string => { @@ -62,7 +62,7 @@ export async function createActionItems( } : {}), collectionIds: actionItem.collections.map( - (collectionTitle) => actionItemCollectionByTitle[collectionTitle].id + (collectionTitle) => actionItemCollectionByTitle[collectionTitle].id, ), })), }); @@ -81,7 +81,7 @@ export async function updateActionItem( client: GraphQLClient, input: ActionItemInput, actionItemId: string, - attributeKeysByName: Record = {} + attributeKeysByName: Record = {}, ): Promise { const getAttribute = (key: string): string => { const existing = attributeKeysByName[key]; @@ -123,11 +123,11 @@ export async function updateActionItem( function actionItemToUniqueCode({ title, collections, -}: Pick): string { +}: Pick): string { return `${title}-${collections .map((c) => c.title) .sort() - .join("-")}`; + .join('-')}`; } /** @@ -139,8 +139,8 @@ function actionItemToUniqueCode({ function actionItemInputToUniqueCode({ title, collections, -}: Pick): string { - return `${title}-${collections.sort().join("-")}`; +}: Pick): string { + return `${title}-${collections.sort().join('-')}`; } /** @@ -152,7 +152,7 @@ function actionItemInputToUniqueCode({ */ export async function syncActionItems( client: GraphQLClient, - inputs: ActionItemInput[] + inputs: ActionItemInput[], ): Promise { let encounteredError = false; // Fetch existing @@ -160,7 +160,7 @@ export async function syncActionItems( // Determine if attributes are syncing const hasAttributes = inputs.some( - (input) => input.attributes && input.attributes.length > 0 + (input) => input.attributes && input.attributes.length > 0, ); // Fetch existing @@ -173,28 +173,28 @@ export async function syncActionItems( // Look up by title const actionItemCollectionByTitle: Record = - keyBy(existingActionItemCollections, "title"); + keyBy(existingActionItemCollections, 'title'); const actionItemByTitle: Record = keyBy( existingActionItems, - actionItemToUniqueCode + actionItemToUniqueCode, ); - const attributeKeysByName = keyBy(attributeKeys, "name"); + const attributeKeysByName = keyBy(attributeKeys, 'name'); const actionItemByCxId: Record = keyBy( existingActionItems.filter((x) => !!x.customerExperienceActionItemIds), - ({ customerExperienceActionItemIds }) => customerExperienceActionItemIds[0] + ({ customerExperienceActionItemIds }) => customerExperienceActionItemIds[0], ); // Ensure all collections exist const missingCollections = uniq( - inputs.flatMap((input) => input.collections) + inputs.flatMap((input) => input.collections), ).filter((collectionTitle) => !actionItemCollectionByTitle[collectionTitle]); if (missingCollections.length > 0) { logger.info( colors.red( `Missing action item collections: "${missingCollections.join( - '", "' - )}" - please create them first!` - ) + '", "', + )}" - please create them first!`, + ), ); return false; } @@ -203,30 +203,30 @@ export async function syncActionItems( const newActionItems = inputs.filter( (input) => !actionItemByTitle[actionItemInputToUniqueCode(input)] && - !actionItemByCxId[input.customerExperienceActionItemId!] + !actionItemByCxId[input.customerExperienceActionItemId!], ); // Create new actionItems if (newActionItems.length > 0) { try { logger.info( - colors.magenta(`Creating "${newActionItems.length}" actionItems...`) + colors.magenta(`Creating "${newActionItems.length}" actionItems...`), ); await createActionItems( client, newActionItems, actionItemCollectionByTitle, - attributeKeysByName + attributeKeysByName, ); logger.info( colors.green( - `Successfully created "${newActionItems.length}" actionItems!` - ) + `Successfully created "${newActionItems.length}" actionItems!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create action items! - ${error.message}`) + colors.red(`Failed to create action items! - ${error.message}`), ); } } @@ -243,14 +243,14 @@ export async function syncActionItems( try { await updateActionItem(client, input, actionItemId, attributeKeysByName); logger.info( - colors.green(`Successfully synced action item "${input.title}"!`) + colors.green(`Successfully synced action item "${input.title}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync action item "${input.title}"! - ${error.message}` - ) + `Failed to sync action item "${input.title}"! - ${error.message}`, + ), ); } }); diff --git a/src/lib/graphql/syncAgentFiles.ts b/src/lib/graphql/syncAgentFiles.ts index c2bd90fa..c83804d3 100644 --- a/src/lib/graphql/syncAgentFiles.ts +++ b/src/lib/graphql/syncAgentFiles.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { AgentFileInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { AgentFile, fetchAllAgentFiles } from "./fetchAllAgentFiles"; -import { CREATE_AGENT_FILE, UPDATE_AGENT_FILES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { AgentFileInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { AgentFile, fetchAllAgentFiles } from './fetchAllAgentFiles'; +import { CREATE_AGENT_FILE, UPDATE_AGENT_FILES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new agent file @@ -17,8 +17,8 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createAgentFile( client: GraphQLClient, - agentFile: AgentFileInput -): Promise> { + agentFile: AgentFileInput, +): Promise> { const input = { name: agentFile.name, description: agentFile.description, @@ -50,7 +50,7 @@ export async function createAgentFile( */ export async function updateAgentFiles( client: GraphQLClient, - agentFileIdPairs: [AgentFileInput, string][] + agentFileIdPairs: [AgentFileInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_AGENT_FILES, { input: { @@ -75,7 +75,7 @@ export async function updateAgentFiles( */ export async function syncAgentFiles( client: GraphQLClient, - inputs: AgentFileInput[] + inputs: AgentFileInput[], ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" agent files...`)); @@ -88,8 +88,8 @@ export async function syncAgentFiles( // Look up by name const agentFileByName: Record< string, - Pick - > = keyBy(existingAgentFiles, "name"); + Pick + > = keyBy(existingAgentFiles, 'name'); // Create new agent files const newAgentFiles = inputs.filter((input) => !agentFileByName[input.name]); @@ -100,14 +100,14 @@ export async function syncAgentFiles( const newAgentFile = await createAgentFile(client, agentFile); agentFileByName[newAgentFile.name] = newAgentFile; logger.info( - colors.green(`Successfully synced agent file "${agentFile.name}"!`) + colors.green(`Successfully synced agent file "${agentFile.name}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync agent file "${agentFile.name}"! - ${error.message}` - ) + `Failed to sync agent file "${agentFile.name}"! - ${error.message}`, + ), ); } }); @@ -117,17 +117,17 @@ export async function syncAgentFiles( logger.info(colors.magenta(`Updating "${inputs.length}" agent files!`)); await updateAgentFiles( client, - inputs.map((input) => [input, agentFileByName[input.name].id]) + inputs.map((input) => [input, agentFileByName[input.name].id]), ); logger.info( - colors.green(`Successfully synced "${inputs.length}" agent files!`) + colors.green(`Successfully synced "${inputs.length}" agent files!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" agent files! - ${error.message}` - ) + `Failed to sync "${inputs.length}" agent files! - ${error.message}`, + ), ); } diff --git a/src/lib/graphql/syncAgentFunctions.ts b/src/lib/graphql/syncAgentFunctions.ts index 53cc09aa..ca4b949e 100644 --- a/src/lib/graphql/syncAgentFunctions.ts +++ b/src/lib/graphql/syncAgentFunctions.ts @@ -1,15 +1,15 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { AgentFunctionInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { AgentFunctionInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { AgentFunction, fetchAllAgentFunctions, -} from "./fetchAllAgentFunctions"; -import { CREATE_AGENT_FUNCTION, UPDATE_AGENT_FUNCTIONS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './fetchAllAgentFunctions'; +import { CREATE_AGENT_FUNCTION, UPDATE_AGENT_FUNCTIONS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new agent function @@ -20,8 +20,8 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createAgentFunction( client: GraphQLClient, - agentFunction: AgentFunctionInput -): Promise> { + agentFunction: AgentFunctionInput, +): Promise> { const input = { name: agentFunction.name, description: agentFunction.description, @@ -50,7 +50,7 @@ export async function createAgentFunction( */ export async function updateAgentFunctions( client: GraphQLClient, - agentFunctionIdPairs: [AgentFunctionInput, string][] + agentFunctionIdPairs: [AgentFunctionInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_AGENT_FUNCTIONS, { input: { @@ -73,7 +73,7 @@ export async function updateAgentFunctions( */ export async function syncAgentFunctions( client: GraphQLClient, - inputs: AgentFunctionInput[] + inputs: AgentFunctionInput[], ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" agent functions...`)); @@ -86,12 +86,12 @@ export async function syncAgentFunctions( // Look up by name const agentFunctionByName: Record< string, - Pick - > = keyBy(existingAgentFunctions, "name"); + Pick + > = keyBy(existingAgentFunctions, 'name'); // Create new agent functions const newAgentFunctions = inputs.filter( - (input) => !agentFunctionByName[input.name] + (input) => !agentFunctionByName[input.name], ); // Create new agent functions @@ -101,15 +101,15 @@ export async function syncAgentFunctions( agentFunctionByName[newAgentFunction.name] = newAgentFunction; logger.info( colors.green( - `Successfully synced agent function "${agentFunction.name}"!` - ) + `Successfully synced agent function "${agentFunction.name}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync agent function "${agentFunction.name}"! - ${error.message}` - ) + `Failed to sync agent function "${agentFunction.name}"! - ${error.message}`, + ), ); } }); @@ -119,17 +119,17 @@ export async function syncAgentFunctions( logger.info(colors.magenta(`Updating "${inputs.length}" agent functions!`)); await updateAgentFunctions( client, - inputs.map((input) => [input, agentFunctionByName[input.name].id]) + inputs.map((input) => [input, agentFunctionByName[input.name].id]), ); logger.info( - colors.green(`Successfully synced "${inputs.length}" agent functions!`) + colors.green(`Successfully synced "${inputs.length}" agent functions!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" agent functions! - ${error.message}` - ) + `Failed to sync "${inputs.length}" agent functions! - ${error.message}`, + ), ); } diff --git a/src/lib/graphql/syncAgents.ts b/src/lib/graphql/syncAgents.ts index 974f39d0..9a1b1659 100644 --- a/src/lib/graphql/syncAgents.ts +++ b/src/lib/graphql/syncAgents.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { AgentInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { Agent, fetchAllAgents } from "./fetchAllAgents"; -import { CREATE_AGENT, UPDATE_AGENTS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { AgentInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { Agent, fetchAllAgents } from './fetchAllAgents'; +import { CREATE_AGENT, UPDATE_AGENTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new agent @@ -17,16 +17,16 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createAgent( client: GraphQLClient, - agent: AgentInput -): Promise> { + agent: AgentInput, +): Promise> { const input = { name: agent.name, description: agent.description, codeInterpreterEnabled: agent.codeInterpreterEnabled, retrievalEnabled: agent.retrievalEnabled, promptTitle: agent.prompt, - largeLanguageModelName: agent["large-language-model"].name, - largeLanguageModelClient: agent["large-language-model"].client, + largeLanguageModelName: agent['large-language-model'].name, + largeLanguageModelClient: agent['large-language-model'].client, // TODO: https://transcend.height.app/T-32760 - agentFunction, agentFile // TODO: https://transcend.height.app/T-31994 - owners and teams }; @@ -51,7 +51,7 @@ export async function createAgent( */ export async function updateAgents( client: GraphQLClient, - agentIdParis: [AgentInput, string][] + agentIdParis: [AgentInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_AGENTS, { input: { @@ -76,7 +76,7 @@ export async function updateAgents( */ export async function syncAgents( client: GraphQLClient, - inputs: AgentInput[] + inputs: AgentInput[], ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" agents...`)); @@ -89,8 +89,8 @@ export async function syncAgents( // Look up by name const agentByName: Record< string, - Pick - > = keyBy(existingAgents, "name"); + Pick + > = keyBy(existingAgents, 'name'); // Create new agents const newAgents = inputs.filter((input) => !agentByName[input.name]); @@ -104,7 +104,7 @@ export async function syncAgents( } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync agent "${agent.name}"! - ${error.message}`) + colors.red(`Failed to sync agent "${agent.name}"! - ${error.message}`), ); } }); @@ -114,15 +114,15 @@ export async function syncAgents( logger.info(colors.magenta(`Updating "${inputs.length}" agents!`)); await updateAgents( client, - inputs.map((input) => [input, agentByName[input.name].id]) + inputs.map((input) => [input, agentByName[input.name].id]), ); logger.info(colors.green(`Successfully synced "${inputs.length}" agents!`)); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" agents ! - ${error.message}` - ) + `Failed to sync "${inputs.length}" agents ! - ${error.message}`, + ), ); } diff --git a/src/lib/graphql/syncAttribute.ts b/src/lib/graphql/syncAttribute.ts index 609da1bd..f902ef43 100644 --- a/src/lib/graphql/syncAttribute.ts +++ b/src/lib/graphql/syncAttribute.ts @@ -1,18 +1,18 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { difference, groupBy, keyBy } from "lodash-es"; -import { AttributeInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { Attribute } from "./fetchAllAttributes"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference, groupBy, keyBy } from 'lodash-es'; +import { AttributeInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { Attribute } from './fetchAllAttributes'; import { CREATE_ATTRIBUTE, CREATE_ATTRIBUTE_VALUES, DELETE_ATTRIBUTE_VALUE, UPDATE_ATTRIBUTE, UPDATE_ATTRIBUTE_VALUES, -} from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Sync attribute @@ -32,7 +32,7 @@ export async function syncAttribute( existingAttribute?: Attribute; /** When true, delete extra attributes not specified in the list of values */ deleteExtraAttributeValues?: boolean; - } + }, ): Promise { // attribute key input const input = { @@ -72,15 +72,15 @@ export async function syncAttribute( } // upsert attribute values - const existingAttributeMap = keyBy(existingAttribute?.values || [], "name"); + const existingAttributeMap = keyBy(existingAttribute?.values || [], 'name'); const { existingValues = [], newValues = [] } = groupBy( attribute.values || [], (field) => - existingAttributeMap[field.name] ? "existingValues" : "newValues" + existingAttributeMap[field.name] ? 'existingValues' : 'newValues', ); const removedValues = difference( (existingAttribute?.values || []).map(({ name }) => name), - (attribute.values || []).map(({ name }) => name) + (attribute.values || []).map(({ name }) => name), ); // Create new attribute values @@ -108,7 +108,7 @@ export async function syncAttribute( })), }); logger.info( - colors.green(`Updated ${existingValues.length} attribute values`) + colors.green(`Updated ${existingValues.length} attribute values`), ); } @@ -123,10 +123,10 @@ export async function syncAttribute( }, { concurrency: 10, - } + }, ); logger.info( - colors.green(`Deleted ${removedValues.length} attribute values`) + colors.green(`Deleted ${removedValues.length} attribute values`), ); } } diff --git a/src/lib/graphql/syncBusinessEntities.ts b/src/lib/graphql/syncBusinessEntities.ts index 1fa10623..3b6ce91b 100644 --- a/src/lib/graphql/syncBusinessEntities.ts +++ b/src/lib/graphql/syncBusinessEntities.ts @@ -1,15 +1,15 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy } from "lodash-es"; -import { BusinessEntityInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy } from 'lodash-es'; +import { BusinessEntityInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { BusinessEntity, fetchAllBusinessEntities, -} from "./fetchAllBusinessEntities"; -import { CREATE_BUSINESS_ENTITY, UPDATE_BUSINESS_ENTITIES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './fetchAllBusinessEntities'; +import { CREATE_BUSINESS_ENTITY, UPDATE_BUSINESS_ENTITIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new business entity @@ -20,7 +20,7 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createBusinessEntity( client: GraphQLClient, - businessEntity: BusinessEntityInput + businessEntity: BusinessEntityInput, ): Promise { const input = { title: businessEntity.title, @@ -55,7 +55,7 @@ export async function createBusinessEntity( */ export async function updateBusinessEntities( client: GraphQLClient, - businessEntityIdParis: [BusinessEntityInput, string][] + businessEntityIdParis: [BusinessEntityInput, string][], ): Promise { const chunkedUpdates = chunk(businessEntityIdParis, 100); await mapSeries(chunkedUpdates, async (chunked) => { @@ -86,11 +86,11 @@ export async function updateBusinessEntities( */ export async function syncBusinessEntities( client: GraphQLClient, - inputs: BusinessEntityInput[] + inputs: BusinessEntityInput[], ): Promise { // Fetch existing logger.info( - colors.magenta(`Syncing "${inputs.length}" business entities...`) + colors.magenta(`Syncing "${inputs.length}" business entities...`), ); let encounteredError = false; @@ -99,11 +99,11 @@ export async function syncBusinessEntities( const existingBusinessEntities = await fetchAllBusinessEntities(client); // Look up by title - const businessEntityByTitle = keyBy(existingBusinessEntities, "title"); + const businessEntityByTitle = keyBy(existingBusinessEntities, 'title'); // Create new business entities const newBusinessEntities = inputs.filter( - (input) => !businessEntityByTitle[input.title] + (input) => !businessEntityByTitle[input.title], ); // Create new business entities @@ -111,20 +111,20 @@ export async function syncBusinessEntities( try { const newBusinessEntity = await createBusinessEntity( client, - businessEntity + businessEntity, ); businessEntityByTitle[newBusinessEntity.title] = newBusinessEntity; logger.info( colors.green( - `Successfully synced business entity "${businessEntity.title}"!` - ) + `Successfully synced business entity "${businessEntity.title}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync business entity "${businessEntity.title}"! - ${error.message}` - ) + `Failed to sync business entity "${businessEntity.title}"! - ${error.message}`, + ), ); } }); @@ -132,21 +132,21 @@ export async function syncBusinessEntities( // Update all business entities try { logger.info( - colors.magenta(`Updating "${inputs.length}" business entities!`) + colors.magenta(`Updating "${inputs.length}" business entities!`), ); await updateBusinessEntities( client, - inputs.map((input) => [input, businessEntityByTitle[input.title].id]) + inputs.map((input) => [input, businessEntityByTitle[input.title].id]), ); logger.info( - colors.green(`Successfully synced "${inputs.length}" business entities!`) + colors.green(`Successfully synced "${inputs.length}" business entities!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" business entities ! - ${error.message}` - ) + `Failed to sync "${inputs.length}" business entities ! - ${error.message}`, + ), ); } diff --git a/src/lib/graphql/syncCodePackages.ts b/src/lib/graphql/syncCodePackages.ts index 3e29b388..8d4ddf84 100644 --- a/src/lib/graphql/syncCodePackages.ts +++ b/src/lib/graphql/syncCodePackages.ts @@ -1,19 +1,19 @@ -import { CodePackageType } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy, uniq, uniqBy } from "lodash-es"; -import { CodePackageInput, RepositoryInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map, mapSeries } from "../bluebird-replace"; -import { CodePackage, fetchAllCodePackages } from "./fetchAllCodePackages"; -import { CREATE_CODE_PACKAGE, UPDATE_CODE_PACKAGES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; -import { syncRepositories } from "./syncRepositories"; -import { syncSoftwareDevelopmentKits } from "./syncSoftwareDevelopmentKits"; +import { CodePackageType } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy, uniq, uniqBy } from 'lodash-es'; +import { CodePackageInput, RepositoryInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; +import { CodePackage, fetchAllCodePackages } from './fetchAllCodePackages'; +import { CREATE_CODE_PACKAGE, UPDATE_CODE_PACKAGES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { syncRepositories } from './syncRepositories'; +import { syncSoftwareDevelopmentKits } from './syncSoftwareDevelopmentKits'; const CHUNK_SIZE = 100; -const LOOKUP_SPLIT_KEY = "%%%%"; +const LOOKUP_SPLIT_KEY = '%%%%'; /** * Create a new code package @@ -47,7 +47,7 @@ export async function createCodePackage( teamIds?: string[]; /** Names of teams */ teamNames?: string[]; - } + }, ): Promise { const { createCodePackage: { codePackage }, @@ -61,7 +61,7 @@ export async function createCodePackage( input, }); logger.info( - colors.green(`Successfully created code package "${input.name}"!`) + colors.green(`Successfully created code package "${input.name}"!`), ); return codePackage; } @@ -100,7 +100,7 @@ export async function updateCodePackages( teamIds?: string[]; /** Names of teams */ teamNames?: string[]; - }[] + }[], ): Promise { const { updateCodePackages: { codePackages }, @@ -116,7 +116,7 @@ export async function updateCodePackages( }, }); logger.info( - colors.green(`Successfully updated ${inputs.length} code packages!`) + colors.green(`Successfully updated ${inputs.length} code packages!`), ); return codePackages; } @@ -132,7 +132,7 @@ export async function updateCodePackages( export async function syncCodePackages( client: GraphQLClient, codePackages: CodePackageInput[], - concurrency = 20 + concurrency = 20, ): Promise { let encounteredError = false; const [ @@ -149,34 +149,34 @@ export async function syncCodePackages( softwareDevelopmentKits.map(({ name }) => ({ name, codePackageType: type, - })) + })), ), ({ name, codePackageType }) => - `${name}${LOOKUP_SPLIT_KEY}${codePackageType}` + `${name}${LOOKUP_SPLIT_KEY}${codePackageType}`, ), - concurrency + concurrency, ), // make sure all Repositories exist syncRepositories( client, - uniqBy(codePackages, "repositoryName").map( + uniqBy(codePackages, 'repositoryName').map( ({ repositoryName }) => ({ name: repositoryName, url: `https://github.com/${repositoryName}`, - } as RepositoryInput) - ) + }) as RepositoryInput, + ), ), ]); const softwareDevelopmentKitLookup = keyBy( existingSoftwareDevelopmentKits, ({ name, codePackageType }) => - `${name}${LOOKUP_SPLIT_KEY}${codePackageType}` + `${name}${LOOKUP_SPLIT_KEY}${codePackageType}`, ); const codePackagesLookup = keyBy( existingCodePackages, - ({ name, type }) => `${name}${LOOKUP_SPLIT_KEY}${type}` + ({ name, type }) => `${name}${LOOKUP_SPLIT_KEY}${type}`, ); // Determine which codePackages are new vs existing @@ -194,8 +194,8 @@ export async function syncCodePackages( try { logger.info( colors.magenta( - `Creating "${newCodePackages.length}" new code packages...` - ) + `Creating "${newCodePackages.length}" new code packages...`, + ), ); await map( newCodePackages, @@ -212,11 +212,11 @@ export async function syncCodePackages( ]; if (!sdk) { throw new Error( - `Failed to find SDK with name: "${name}"` + `Failed to find SDK with name: "${name}"`, ); } return sdk.id; - }) + }), ), } : {}), @@ -224,28 +224,28 @@ export async function syncCodePackages( }, { concurrency, - } + }, ); logger.info( colors.green( - `Successfully synced ${newCodePackages.length} code packages!` - ) + `Successfully synced ${newCodePackages.length} code packages!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create code packages! - ${error.message}`) + colors.red(`Failed to create code packages! - ${error.message}`), ); } // Update existing codePackages const existingCodePackageInputs = mapCodePackagesToExisting.filter( - (x): x is [CodePackageInput, string] => !!x[1] + (x): x is [CodePackageInput, string] => !!x[1], ); logger.info( colors.magenta( - `Updating "${existingCodePackageInputs.length}" code packages...` - ) + `Updating "${existingCodePackageInputs.length}" code packages...`, + ), ); const chunks = chunk(existingCodePackageInputs, CHUNK_SIZE); @@ -267,25 +267,25 @@ export async function syncCodePackages( ]; if (!sdk) { throw new Error( - `Failed to find SDK with name: "${name}"` + `Failed to find SDK with name: "${name}"`, ); } return sdk.id; - }) + }), ), } : {}), id, - }) - ) + }), + ), ); logger.info( - colors.green(`Successfully updated "${chunk.length}" code packages!`) + colors.green(`Successfully updated "${chunk.length}" code packages!`), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to update code packages! - ${error.message}`) + colors.red(`Failed to update code packages! - ${error.message}`), ); } }); diff --git a/src/lib/graphql/syncConfigurationToTranscend.ts b/src/lib/graphql/syncConfigurationToTranscend.ts index e5ac1eac..eaa2b390 100644 --- a/src/lib/graphql/syncConfigurationToTranscend.ts +++ b/src/lib/graphql/syncConfigurationToTranscend.ts @@ -1,46 +1,46 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { TranscendInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { fetchAllActions } from "./fetchAllActions"; -import { fetchAllAttributes } from "./fetchAllAttributes"; -import { fetchApiKeys } from "./fetchApiKeys"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { TranscendInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllActions } from './fetchAllActions'; +import { fetchAllAttributes } from './fetchAllAttributes'; +import { fetchApiKeys } from './fetchApiKeys'; import { ensureAllDataSubjectsExist, fetchAllDataSubjects, -} from "./fetchDataSubjects"; +} from './fetchDataSubjects'; import { fetchIdentifiersAndCreateMissing, Identifier, -} from "./fetchIdentifiers"; -import { syncAction } from "./syncAction"; -import { syncActionItemCollections } from "./syncActionItemCollections"; -import { syncActionItems } from "./syncActionItems"; -import { syncAgentFiles } from "./syncAgentFiles"; -import { syncAgentFunctions } from "./syncAgentFunctions"; -import { syncAgents } from "./syncAgents"; -import { syncAttribute } from "./syncAttribute"; -import { syncBusinessEntities } from "./syncBusinessEntities"; -import { syncConsentManager } from "./syncConsentManager"; -import { syncCookies } from "./syncCookies"; -import { syncDataCategories } from "./syncDataCategories"; -import { syncDataFlows } from "./syncDataFlows"; -import { syncDataSiloDependencies, syncDataSilos } from "./syncDataSilos"; -import { syncDataSubject } from "./syncDataSubject"; -import { syncEnricher } from "./syncEnrichers"; -import { syncIdentifier } from "./syncIdentifier"; -import { syncIntlMessages } from "./syncIntlMessages"; -import { syncPartitions } from "./syncPartitions"; -import { syncPolicies } from "./syncPolicies"; -import { syncPrivacyCenter } from "./syncPrivacyCenter"; -import { syncProcessingPurposes } from "./syncProcessingPurposes"; -import { syncPromptGroups } from "./syncPromptGroups"; -import { syncPromptPartials } from "./syncPromptPartials"; -import { syncPrompts } from "./syncPrompts"; -import { syncTeams } from "./syncTeams"; -import { syncTemplate } from "./syncTemplates"; -import { syncVendors } from "./syncVendors"; +} from './fetchIdentifiers'; +import { syncAction } from './syncAction'; +import { syncActionItemCollections } from './syncActionItemCollections'; +import { syncActionItems } from './syncActionItems'; +import { syncAgentFiles } from './syncAgentFiles'; +import { syncAgentFunctions } from './syncAgentFunctions'; +import { syncAgents } from './syncAgents'; +import { syncAttribute } from './syncAttribute'; +import { syncBusinessEntities } from './syncBusinessEntities'; +import { syncConsentManager } from './syncConsentManager'; +import { syncCookies } from './syncCookies'; +import { syncDataCategories } from './syncDataCategories'; +import { syncDataFlows } from './syncDataFlows'; +import { syncDataSiloDependencies, syncDataSilos } from './syncDataSilos'; +import { syncDataSubject } from './syncDataSubject'; +import { syncEnricher } from './syncEnrichers'; +import { syncIdentifier } from './syncIdentifier'; +import { syncIntlMessages } from './syncIntlMessages'; +import { syncPartitions } from './syncPartitions'; +import { syncPolicies } from './syncPolicies'; +import { syncPrivacyCenter } from './syncPrivacyCenter'; +import { syncProcessingPurposes } from './syncProcessingPurposes'; +import { syncPromptGroups } from './syncPromptGroups'; +import { syncPromptPartials } from './syncPromptPartials'; +import { syncPrompts } from './syncPrompts'; +import { syncTeams } from './syncTeams'; +import { syncTemplate } from './syncTemplates'; +import { syncVendors } from './syncVendors'; const CONCURRENCY = 10; @@ -70,7 +70,7 @@ export async function syncConfigurationToTranscend( deleteExtraAttributeValues?: boolean; /** classify data flow service if missing */ classifyService?: boolean; - } + }, ): Promise { let encounteredError = false; @@ -81,26 +81,26 @@ export async function syncConfigurationToTranscend( attributes, actions, identifiers, - "data-subjects": dataSubjects, - "business-entities": businessEntities, + 'data-subjects': dataSubjects, + 'business-entities': businessEntities, enrichers, cookies, - "consent-manager": consentManager, - "data-silos": dataSilos, - "data-flows": dataFlows, + 'consent-manager': consentManager, + 'data-silos': dataSilos, + 'data-flows': dataFlows, prompts, - "prompt-groups": promptGroups, - "prompt-partials": promptPartials, + 'prompt-groups': promptGroups, + 'prompt-partials': promptPartials, agents, - "agent-functions": agentFunctions, - "agent-files": agentFiles, + 'agent-functions': agentFunctions, + 'agent-files': agentFiles, vendors, - "data-categories": dataCategories, - "processing-purposes": processingPurposes, - "action-items": actionItems, - "action-item-collections": actionItemCollections, + 'data-categories': dataCategories, + 'processing-purposes': processingPurposes, + 'action-items': actionItems, + 'action-item-collections': actionItemCollections, teams, - "privacy-center": privacyCenter, + 'privacy-center': privacyCenter, messages, policies, partitions, @@ -113,7 +113,7 @@ export async function syncConfigurationToTranscend( ? fetchIdentifiersAndCreateMissing( input, client, - !publishToPrivacyCenter + !publishToPrivacyCenter, ) : ({} as Record), // Grab all data subjects in the organization @@ -123,7 +123,7 @@ export async function syncConfigurationToTranscend( // Grab API keys dataSilos && dataSilos - .map((dataSilo) => dataSilo["api-key-title"] || []) + .map((dataSilo) => dataSilo['api-key-title'] || []) .reduce((accumulator, lst) => accumulator + lst.length, 0) > 0 ? fetchApiKeys(input, client) : {}, @@ -131,14 +131,14 @@ export async function syncConfigurationToTranscend( // Sync consent manager if (consentManager) { - logger.info(colors.magenta("Syncing consent manager...")); + logger.info(colors.magenta('Syncing consent manager...')); try { await syncConsentManager(client, consentManager); - logger.info(colors.green("Successfully synced consent manager!")); + logger.info(colors.green('Successfully synced consent manager!')); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync consent manager! - ${error.message}`) + colors.red(`Failed to sync consent manager! - ${error.message}`), ); } } @@ -165,7 +165,7 @@ export async function syncConfigurationToTranscend( // Sync email templates if (templates) { logger.info( - colors.magenta(`Syncing "${templates.length}" email templates...`) + colors.magenta(`Syncing "${templates.length}" email templates...`), ); await map( templates, @@ -174,20 +174,20 @@ export async function syncConfigurationToTranscend( try { await syncTemplate(template, client); logger.info( - colors.green(`Successfully synced template "${template.title}"!`) + colors.green(`Successfully synced template "${template.title}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync template "${template.title}"! - ${error.message}` - ) + `Failed to sync template "${template.title}"! - ${error.message}`, + ), ); } }, { concurrency: CONCURRENCY, - } + }, ); logger.info(colors.green(`Synced "${templates.length}" email templates!`)); } @@ -196,7 +196,7 @@ export async function syncConfigurationToTranscend( if (businessEntities) { const businessEntitySuccess = await syncBusinessEntities( client, - businessEntities + businessEntities, ); encounteredError = encounteredError || !businessEntitySuccess; } @@ -211,7 +211,7 @@ export async function syncConfigurationToTranscend( if (dataCategories) { const dataCategoriesSuccess = await syncDataCategories( client, - dataCategories + dataCategories, ); encounteredError = encounteredError || !dataCategoriesSuccess; } @@ -220,7 +220,7 @@ export async function syncConfigurationToTranscend( if (processingPurposes) { const processingPurposesSuccess = await syncProcessingPurposes( client, - processingPurposes + processingPurposes, ); encounteredError = encounteredError || !processingPurposesSuccess; } @@ -241,7 +241,7 @@ export async function syncConfigurationToTranscend( if (agentFunctions) { const agentFunctionsSuccess = await syncAgentFunctions( client, - agentFunctions + agentFunctions, ); encounteredError = encounteredError || !agentFunctionsSuccess; } @@ -262,7 +262,7 @@ export async function syncConfigurationToTranscend( if (actionItemCollections) { const actionItemCollectionsSuccess = await syncActionItemCollections( client, - actionItemCollections + actionItemCollections, ); encounteredError = encounteredError || !actionItemCollectionsSuccess; } @@ -276,7 +276,7 @@ export async function syncConfigurationToTranscend( attributes, async (attribute) => { const existing = existingAttributes.find( - (attribute_) => attribute_.name === attribute.name + (attribute_) => attribute_.name === attribute.name, ); logger.info(colors.magenta(`Syncing attribute "${attribute.name}"...`)); @@ -286,20 +286,20 @@ export async function syncConfigurationToTranscend( deleteExtraAttributeValues, }); logger.info( - colors.green(`Successfully synced attribute "${attribute.name}"!`) + colors.green(`Successfully synced attribute "${attribute.name}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync attribute "${attribute.name}"! - ${error.message}` - ) + `Failed to sync attribute "${attribute.name}"! - ${error.message}`, + ), ); } }, { concurrency: CONCURRENCY, - } + }, ); logger.info(colors.green(`Synced "${attributes.length}" attributes!`)); } @@ -324,20 +324,20 @@ export async function syncConfigurationToTranscend( dataSubjectsByName, }); logger.info( - colors.green(`Successfully synced enricher "${enricher.title}"!`) + colors.green(`Successfully synced enricher "${enricher.title}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync enricher "${enricher.title}"! - ${error.message}` - ) + `Failed to sync enricher "${enricher.title}"! - ${error.message}`, + ), ); } }, { concurrency: CONCURRENCY, - } + }, ); logger.info(colors.green(`Synced "${enrichers.length}" enrichers!`)); } @@ -346,7 +346,7 @@ export async function syncConfigurationToTranscend( if (identifiers) { // Fetch existing logger.info( - colors.magenta(`Syncing "${identifiers.length}" identifiers...`) + colors.magenta(`Syncing "${identifiers.length}" identifiers...`), ); await map( identifiers, @@ -354,12 +354,12 @@ export async function syncConfigurationToTranscend( const existing = identifierByName[identifier.name]; if (!existing) { throw new Error( - `Failed to find identifier with name: ${identifier.type}. Should have been auto-created by cli.` + `Failed to find identifier with name: ${identifier.type}. Should have been auto-created by cli.`, ); } logger.info( - colors.magenta(`Syncing identifier "${identifier.type}"...`) + colors.magenta(`Syncing identifier "${identifier.type}"...`), ); try { await syncIdentifier(client, { @@ -369,20 +369,22 @@ export async function syncConfigurationToTranscend( skipPublish: !publishToPrivacyCenter, }); logger.info( - colors.green(`Successfully synced identifier "${identifier.type}"!`) + colors.green( + `Successfully synced identifier "${identifier.type}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync identifier "${identifier.type}"! - ${error.message}` - ) + `Failed to sync identifier "${identifier.type}"! - ${error.message}`, + ), ); } }, { concurrency: CONCURRENCY, - } + }, ); logger.info(colors.green(`Synced "${identifiers.length}" identifiers!`)); } @@ -396,11 +398,11 @@ export async function syncConfigurationToTranscend( actions, async (action) => { const existing = existingActions.find( - (act) => act.type === action.type + (act) => act.type === action.type, ); if (!existing) { throw new Error( - `Failed to find action with type: ${action.type}. Should have already existing in the organization.` + `Failed to find action with type: ${action.type}. Should have already existing in the organization.`, ); } @@ -412,20 +414,20 @@ export async function syncConfigurationToTranscend( skipPublish: !publishToPrivacyCenter, }); logger.info( - colors.green(`Successfully synced action "${action.type}"!`) + colors.green(`Successfully synced action "${action.type}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync action "${action.type}"! - ${error.message}` - ) + `Failed to sync action "${action.type}"! - ${error.message}`, + ), ); } }, { concurrency: CONCURRENCY, - } + }, ); logger.info(colors.green(`Synced "${actions.length}" actions!`)); } @@ -434,23 +436,23 @@ export async function syncConfigurationToTranscend( if (dataSubjects) { // Fetch existing logger.info( - colors.magenta(`Syncing "${dataSubjects.length}" data subjects...`) + colors.magenta(`Syncing "${dataSubjects.length}" data subjects...`), ); const existingDataSubjects = await fetchAllDataSubjects(client); await map( dataSubjects, async (dataSubject) => { const existing = existingDataSubjects.find( - (subj) => subj.type === dataSubject.type + (subj) => subj.type === dataSubject.type, ); if (!existing) { throw new Error( - `Failed to find data subject with type: ${dataSubject.type}. Should have already existing in the organization.` + `Failed to find data subject with type: ${dataSubject.type}. Should have already existing in the organization.`, ); } logger.info( - colors.magenta(`Syncing data subject "${dataSubject.type}"...`) + colors.magenta(`Syncing data subject "${dataSubject.type}"...`), ); try { await syncDataSubject(client, { @@ -460,21 +462,21 @@ export async function syncConfigurationToTranscend( }); logger.info( colors.green( - `Successfully synced data subject "${dataSubject.type}"!` - ) + `Successfully synced data subject "${dataSubject.type}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync data subject "${dataSubject.type}"! - ${error.message}` - ) + `Failed to sync data subject "${dataSubject.type}"! - ${error.message}`, + ), ); } }, { concurrency: CONCURRENCY, - } + }, ); logger.info(colors.green(`Synced "${dataSubjects.length}" data subjects!`)); } @@ -484,7 +486,7 @@ export async function syncConfigurationToTranscend( const syncedDataFlows = await syncDataFlows( client, dataFlows, - classifyService + classifyService, ); encounteredError = encounteredError || !syncedDataFlows; } @@ -518,15 +520,15 @@ export async function syncConfigurationToTranscend( dataSubjectsByName, apiKeysByTitle: apiKeyTitleMap, pageSize, - } + }, ); if (dataSilos) for (const dataSilo of dataSilos) { // Queue up dependency update - if (dataSilo["deletion-dependencies"]) { + if (dataSilo['deletion-dependencies']) { dependencyUpdates.push([ dataSiloTitleToId[dataSilo.title], - dataSilo["deletion-dependencies"], + dataSilo['deletion-dependencies'], ]); } } diff --git a/src/lib/graphql/syncConsentManager.ts b/src/lib/graphql/syncConsentManager.ts index 2f45a885..5184f7aa 100644 --- a/src/lib/graphql/syncConsentManager.ts +++ b/src/lib/graphql/syncConsentManager.ts @@ -1,22 +1,22 @@ import { InitialViewState, OnConsentExpiry, -} from "@transcend-io/airgap.js-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; +} from '@transcend-io/airgap.js-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; import { ConsentManageExperienceInput, ConsentManagerInput, -} from "../../codecs"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { fetchAllPurposes } from "./fetchAllPurposes"; +} from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllPurposes } from './fetchAllPurposes'; import { fetchConsentManagerExperiences, fetchConsentManagerId, -} from "./fetchConsentManagerId"; -import { fetchPrivacyCenterId } from "./fetchPrivacyCenterId"; +} from './fetchConsentManagerId'; +import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; import { CREATE_CONSENT_EXPERIENCE, CREATE_CONSENT_MANAGER, @@ -30,12 +30,12 @@ import { UPDATE_CONSENT_MANAGER_THEME, UPDATE_CONSENT_MANAGER_VERSION, UPDATE_TOGGLE_USP_API, -} from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; -import { fetchPartitions } from "./syncPartitions"; +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; +import { fetchPartitions } from './syncPartitions'; const PURPOSES_LINK = - "https://app.transcend.io/consent-manager/regional-experiences/purposes"; + 'https://app.transcend.io/consent-manager/regional-experiences/purposes'; /** * Sync consent manager experiences up to Transcend @@ -45,15 +45,15 @@ const PURPOSES_LINK = */ export async function syncConsentManagerExperiences( client: GraphQLClient, - experiences: ConsentManageExperienceInput[] + experiences: ConsentManageExperienceInput[], ): Promise { // Fetch existing experiences and const existingExperiences = await fetchConsentManagerExperiences(client); - const experienceLookup = keyBy(existingExperiences, "name"); + const experienceLookup = keyBy(existingExperiences, 'name'); // Fetch existing purposes const purposes = await fetchAllPurposes(client); - const purposeLookup = keyBy(purposes, "trackingType"); + const purposeLookup = keyBy(purposes, 'trackingType'); // Bulk update or create experiences await map( @@ -65,7 +65,7 @@ export async function syncConsentManagerExperiences( if (!existingPurpose) { throw new Error( `Invalid purpose trackingType provided at consentManager.experiences[${ind}].purposes[${ind2}]: ` + - `${purpose.trackingType}. See list of valid purposes ${PURPOSES_LINK}` + `${purpose.trackingType}. See list of valid purposes ${PURPOSES_LINK}`, ); } return existingPurpose.id; @@ -75,7 +75,7 @@ export async function syncConsentManagerExperiences( if (!existingPurpose) { throw new Error( `Invalid purpose trackingType provided at consentManager.experiences[${ind}].optedOutPurposes[${ind2}]: ` + - `${purpose.trackingType}. See list of valid purposes ${PURPOSES_LINK}` + `${purpose.trackingType}. See list of valid purposes ${PURPOSES_LINK}`, ); } return existingPurpose.id; @@ -104,7 +104,7 @@ export async function syncConsentManagerExperiences( }, }); logger.info( - colors.green(`Successfully synced consent experience "${exp.name}"!`) + colors.green(`Successfully synced consent experience "${exp.name}"!`), ); } else { // create new experience @@ -125,13 +125,15 @@ export async function syncConsentManagerExperiences( }, }); logger.info( - colors.green(`Successfully created consent experience "${exp.name}"!`) + colors.green( + `Successfully created consent experience "${exp.name}"!`, + ), ); } }, { concurrency: 10, - } + }, ); } @@ -143,7 +145,7 @@ export async function syncConsentManagerExperiences( */ export async function syncConsentManager( client: GraphQLClient, - consentManager: ConsentManagerInput + consentManager: ConsentManagerInput, ): Promise { let airgapBundleId: string; @@ -152,7 +154,7 @@ export async function syncConsentManager( airgapBundleId = await fetchConsentManagerId(client, 1); } catch (error) { // TODO: https://transcend.height.app/T-23778 - if (error.message.includes("AirgapBundle not found")) { + if (error.message.includes('AirgapBundle not found')) { const privacyCenterId = await fetchPrivacyCenterId(client); const { createConsentManager } = await makeGraphQLRequest<{ @@ -186,11 +188,11 @@ export async function syncConsentManager( if (consentManager.partition) { const partitions = await fetchPartitions(client); const partitionToUpdate = partitions.find( - (part) => part.name === consentManager.partition + (part) => part.name === consentManager.partition, ); if (!partitionToUpdate) { throw new Error( - `Partition "${consentManager.partition}" not found. Please create the partition first.` + `Partition "${consentManager.partition}" not found. Please create the partition first.`, ); } await makeGraphQLRequest(client, UPDATE_CONSENT_MANAGER_PARTITION, { diff --git a/src/lib/graphql/syncCookies.ts b/src/lib/graphql/syncCookies.ts index 2c0dd7a4..7fcdf38b 100644 --- a/src/lib/graphql/syncCookies.ts +++ b/src/lib/graphql/syncCookies.ts @@ -1,13 +1,13 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk } from "lodash-es"; -import { CookieInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { fetchConsentManagerId } from "./fetchConsentManagerId"; -import { UPDATE_OR_CREATE_COOKIES } from "./gqls"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk } from 'lodash-es'; +import { CookieInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { fetchConsentManagerId } from './fetchConsentManagerId'; +import { UPDATE_OR_CREATE_COOKIES } from './gqls'; // import { keyBy } from 'lodash-es'; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const MAX_PAGE_SIZE = 100; @@ -19,7 +19,7 @@ const MAX_PAGE_SIZE = 100; */ export async function updateOrCreateCookies( client: GraphQLClient, - cookieInputs: CookieInput[] + cookieInputs: CookieInput[], ): Promise { const airgapBundleId = await fetchConsentManagerId(client); @@ -64,7 +64,7 @@ export async function updateOrCreateCookies( */ export async function syncCookies( client: GraphQLClient, - cookies: CookieInput[] + cookies: CookieInput[], ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${cookies.length}" cookies...`)); @@ -73,14 +73,14 @@ export async function syncCookies( const notUnique = cookies.filter( (cookie) => cookies.filter( - (cook) => cookie.name === cook.name && cookie.isRegex === cook.isRegex - ).length > 1 + (cook) => cookie.name === cook.name && cookie.isRegex === cook.isRegex, + ).length > 1, ); if (notUnique.length > 0) { throw new Error( `Failed to upload cookies as there were non-unique entries found: ${notUnique .map(({ name }) => name) - .join(",")}` + .join(',')}`, ); } diff --git a/src/lib/graphql/syncDataCategories.ts b/src/lib/graphql/syncDataCategories.ts index 0b37f92c..2821bf2b 100644 --- a/src/lib/graphql/syncDataCategories.ts +++ b/src/lib/graphql/syncDataCategories.ts @@ -1,15 +1,15 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { DataCategoryInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { DataCategoryInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { DataSubCategory, fetchAllDataCategories, -} from "./fetchAllDataCategories"; -import { CREATE_DATA_SUB_CATEGORY, UPDATE_DATA_SUB_CATEGORIES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './fetchAllDataCategories'; +import { CREATE_DATA_SUB_CATEGORY, UPDATE_DATA_SUB_CATEGORIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new data category @@ -20,8 +20,8 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createDataCategory( client: GraphQLClient, - dataCategory: DataCategoryInput -): Promise> { + dataCategory: DataCategoryInput, +): Promise> { const input = { name: dataCategory.name, category: dataCategory.category, @@ -49,7 +49,7 @@ export async function createDataCategory( */ export async function updateDataCategories( client: GraphQLClient, - dataCategoryIdPairs: [DataCategoryInput, string][] + dataCategoryIdPairs: [DataCategoryInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_DATA_SUB_CATEGORIES, { input: { @@ -72,7 +72,7 @@ export async function updateDataCategories( */ export async function syncDataCategories( client: GraphQLClient, - inputs: DataCategoryInput[] + inputs: DataCategoryInput[], ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" data categories...`)); @@ -85,15 +85,15 @@ export async function syncDataCategories( // Look up by name const dataCategoryByName: Record< string, - Pick + Pick > = keyBy( existingDataCategories, - ({ name, category }) => `${name}:${category}` + ({ name, category }) => `${name}:${category}`, ); // Create new data categories const newDataCategories = inputs.filter( - (input) => !dataCategoryByName[`${input.name}:${input.category}`] + (input) => !dataCategoryByName[`${input.name}:${input.category}`], ); // Create new data categories @@ -105,15 +105,15 @@ export async function syncDataCategories( ] = newDataCategory; logger.info( colors.green( - `Successfully synced data category "${dataCategory.name}"!` - ) + `Successfully synced data category "${dataCategory.name}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync data category "${dataCategory.name}"! - ${error.message}` - ) + `Failed to sync data category "${dataCategory.name}"! - ${error.message}`, + ), ); } }); @@ -126,17 +126,17 @@ export async function syncDataCategories( inputs.map((input) => [ input, dataCategoryByName[`${input.name}:${input.category}`].id, - ]) + ]), ); logger.info( - colors.green(`Successfully synced "${inputs.length}" data categories!`) + colors.green(`Successfully synced "${inputs.length}" data categories!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" data categories ! - ${error.message}` - ) + `Failed to sync "${inputs.length}" data categories ! - ${error.message}`, + ), ); } diff --git a/src/lib/graphql/syncDataFlows.ts b/src/lib/graphql/syncDataFlows.ts index b781f6a7..795fa6de 100644 --- a/src/lib/graphql/syncDataFlows.ts +++ b/src/lib/graphql/syncDataFlows.ts @@ -1,14 +1,14 @@ -import { ConsentTrackerStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk } from "lodash-es"; -import { DataFlowInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { fetchAllDataFlows } from "./fetchAllDataFlows"; -import { fetchConsentManagerId } from "./fetchConsentManagerId"; -import { CREATE_DATA_FLOWS, UPDATE_DATA_FLOWS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { ConsentTrackerStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk } from 'lodash-es'; +import { DataFlowInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { fetchAllDataFlows } from './fetchAllDataFlows'; +import { fetchConsentManagerId } from './fetchConsentManagerId'; +import { CREATE_DATA_FLOWS, UPDATE_DATA_FLOWS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const MAX_PAGE_SIZE = 100; @@ -22,7 +22,7 @@ const MAX_PAGE_SIZE = 100; export async function updateDataFlows( client: GraphQLClient, dataFlowInputs: [DataFlowInput, string][], - classifyService = false + classifyService = false, ): Promise { const airgapBundleId = await fetchConsentManagerId(client); @@ -70,7 +70,7 @@ export async function updateDataFlows( export async function createDataFlows( client: GraphQLClient, dataFlowInputs: DataFlowInput[], - classifyService = false + classifyService = false, ): Promise { const airgapBundleId = await fetchConsentManagerId(client); // TODO: https://transcend.height.app/T-19841 - add with custom purposes @@ -117,7 +117,7 @@ export async function createDataFlows( export async function syncDataFlows( client: GraphQLClient, dataFlows: DataFlowInput[], - classifyService: boolean + classifyService: boolean, ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${dataFlows.length}" data flows...`)); @@ -127,8 +127,8 @@ export async function syncDataFlows( const notUnique = dataFlows.filter( (dataFlow) => dataFlows.filter( - (flow) => dataFlow.value === flow.value && dataFlow.type === flow.type - ).length > 1 + (flow) => dataFlow.value === flow.value && dataFlow.type === flow.type, + ).length > 1, ); // Throw error to prompt user to de-dupe before uploading @@ -136,13 +136,13 @@ export async function syncDataFlows( throw new Error( `Failed to upload data flows as there were non-unique entries found: ${notUnique .map(({ value }) => value) - .join(",")}` + .join(',')}`, ); } // Fetch existing data flows to determine whether we are creating a new data flow // or updating an existing data flow - logger.info(colors.magenta("Fetching data flows...")); + logger.info(colors.magenta('Fetching data flows...')); const [existingLiveDataFlows, existingInReviewDataFlows] = await Promise.all([ fetchAllDataFlows(client, ConsentTrackerStatus.Live), fetchAllDataFlows(client, ConsentTrackerStatus.NeedsReview), @@ -153,7 +153,7 @@ export async function syncDataFlows( const mapDataFlowsToExisting = dataFlows.map((dataFlow) => [ dataFlow, allDataFlows.find( - (flow) => dataFlow.value === flow.value && dataFlow.type === flow.type + (flow) => dataFlow.value === flow.value && dataFlow.type === flow.type, )?.id, ]); @@ -163,11 +163,11 @@ export async function syncDataFlows( .map(([flow]) => flow as DataFlowInput); try { logger.info( - colors.magenta(`Creating "${newDataFlows.length}" new data flows...`) + colors.magenta(`Creating "${newDataFlows.length}" new data flows...`), ); await createDataFlows(client, newDataFlows, classifyService); logger.info( - colors.green(`Successfully synced ${newDataFlows.length} data flows!`) + colors.green(`Successfully synced ${newDataFlows.length} data flows!`), ); } catch (error) { encounteredError = true; @@ -176,17 +176,17 @@ export async function syncDataFlows( // Update existing data flows const existingDataFlows = mapDataFlowsToExisting.filter( - (x): x is [DataFlowInput, string] => !!x[1] + (x): x is [DataFlowInput, string] => !!x[1], ); try { logger.info( - colors.magenta(`Updating "${existingDataFlows.length}" data flows...`) + colors.magenta(`Updating "${existingDataFlows.length}" data flows...`), ); await updateDataFlows(client, existingDataFlows, classifyService); logger.info( colors.green( - `Successfully updated "${existingDataFlows.length}" data flows!` - ) + `Successfully updated "${existingDataFlows.length}" data flows!`, + ), ); } catch (error) { encounteredError = true; diff --git a/src/lib/graphql/syncDataSilos.ts b/src/lib/graphql/syncDataSilos.ts index 2c47f458..da3b828b 100644 --- a/src/lib/graphql/syncDataSilos.ts +++ b/src/lib/graphql/syncDataSilos.ts @@ -6,24 +6,24 @@ import { PromptAVendorEmailSendType, RequestActionObjectResolver, SubDataPointDataSubCategoryGuessStatus, -} from "@transcend-io/privacy-types"; -import { apply } from "@transcend-io/type-utils"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy, sortBy } from "lodash-es"; +} from '@transcend-io/privacy-types'; +import { apply } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy, sortBy } from 'lodash-es'; import { DataCategoryInput, DataSiloInput, ProcessingPurposeInput, -} from "../../codecs"; -import { logger } from "../../logger"; -import { map, mapSeries } from "../bluebird-replace"; -import { ApiKey } from "./fetchApiKeys"; +} from '../../codecs'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; +import { ApiKey } from './fetchApiKeys'; import { convertToDataSubjectBlockList, DataSubject, -} from "./fetchDataSubjects"; +} from './fetchDataSubjects'; import { CREATE_DATA_SILOS, DATA_POINTS, @@ -33,8 +33,8 @@ import { SUB_DATA_POINTS_WITH_GUESSES, UPDATE_DATA_SILOS, UPDATE_OR_CREATE_DATA_POINT, -} from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface DataSiloAttributeValue { /** Key associated to value */ @@ -93,12 +93,12 @@ export async function fetchAllDataSilos( integrationNames?: string[]; /** GQL query for data silos */ gql?: string; - } + }, ): Promise { logger.info( colors.magenta( - `Fetching ${ids.length === 0 ? "all" : ids.length} Data Silos...` - ) + `Fetching ${ids.length === 0 ? 'all' : ids.length} Data Silos...`, + ), ); const dataSilos: TDataSilo[] = []; @@ -131,13 +131,13 @@ export async function fetchAllDataSilos( logger.info( colors.green( `Found a total of ${dataSilos.length} data silo${ - ids.length > 0 ? ` matching IDs ${ids.join(",")}` : "" + ids.length > 0 ? ` matching IDs ${ids.join(',')}` : '' }s${ integrationNames.length > 0 - ? ` matching integrationNames ${integrationNames.join(",")}` - : "" - }` - ) + ? ` matching integrationNames ${integrationNames.join(',')}` + : '' + }`, + ), ); return dataSilos.sort((a, b) => a.title.localeCompare(b.title)); @@ -266,7 +266,7 @@ export async function fetchAllSubDataPoints( pageSize: number; /** When true, metadata around guessed data categories should be included */ includeGuessedCategories?: boolean; - } + }, ): Promise { const subDataPoints: SubDataPoint[] = []; @@ -277,7 +277,7 @@ export async function fetchAllSubDataPoints( try { if (debug) { logger.log( - colors.magenta(`Pulling in subdatapoints for offset ${offset}`) + colors.magenta(`Pulling in subdatapoints for offset ${offset}`), ); } const { @@ -299,7 +299,7 @@ export async function fetchAllSubDataPoints( dataPoints: [dataPointId], }, offset, - } + }, ); subDataPoints.push(...nodes); @@ -309,20 +309,20 @@ export async function fetchAllSubDataPoints( if (debug) { logger.log( colors.green( - `Pulled in subdatapoints for offset ${offset} for dataPointId=${dataPointId}` - ) + `Pulled in subdatapoints for offset ${offset} for dataPointId=${dataPointId}`, + ), ); } } catch (error) { logger.error( colors.red( - `An error fetching subdatapoints for offset ${offset} for dataPointId=${dataPointId}` - ) + `An error fetching subdatapoints for offset ${offset} for dataPointId=${dataPointId}`, + ), ); throw error; } } while (shouldContinue); - return sortBy(subDataPoints, "name"); + return sortBy(subDataPoints, 'name'); } /** @@ -350,7 +350,7 @@ export async function fetchAllDataPoints( skipSubDatapoints?: boolean; /** When true, metadata around guessed data categories should be included */ includeGuessedCategories?: boolean; - } + }, ): Promise { const dataPoints: DataPointWithSubDataPoint[] = []; @@ -383,8 +383,8 @@ export async function fetchAllDataPoints( if (debug) { logger.info( colors.magenta( - `Fetched ${nodes.length} datapoints at offset: ${offset}` - ) + `Fetched ${nodes.length} datapoints at offset: ${offset}`, + ), ); } @@ -397,8 +397,8 @@ export async function fetchAllDataPoints( if (debug) { logger.info( colors.magenta( - `Fetching subdatapoints for ${node.name} for datapoint offset ${offset}` - ) + `Fetching subdatapoints for ${node.name} for datapoint offset ${offset}`, + ), ); } @@ -410,22 +410,22 @@ export async function fetchAllDataPoints( dataPoints.push({ ...node, subDataPoints: subDataPoints.sort((a, b) => - a.name.localeCompare(b.name) + a.name.localeCompare(b.name), ), }); if (debug) { logger.info( colors.green( - `Successfully fetched subdatapoints for ${node.name}` - ) + `Successfully fetched subdatapoints for ${node.name}`, + ), ); } } catch (error) { logger.error( colors.red( - `An error fetching subdatapoints for ${node.name} datapoint offset ${offset}` - ) + `An error fetching subdatapoints for ${node.name} datapoint offset ${offset}`, + ), ); throw error; } @@ -433,14 +433,14 @@ export async function fetchAllDataPoints( { concurrency: 5, - } + }, ); if (debug) { logger.info( colors.green( - `Fetched all subdatapoints for page of datapoints at offset: ${offset}` - ) + `Fetched all subdatapoints for page of datapoints at offset: ${offset}`, + ), ); } } @@ -587,7 +587,7 @@ export async function fetchEnrichedDataSilos( skipSubDatapoints?: boolean; /** When true, metadata around guessed data categories should be included */ includeGuessedCategories?: boolean; - } + }, ): Promise<[DataSiloEnriched, DataPointWithSubDataPoint[]][]> { const dataSilos: [DataSiloEnriched, DataPointWithSubDataPoint[]][] = []; @@ -605,8 +605,8 @@ export async function fetchEnrichedDataSilos( await mapSeries(silos, async (silo, index) => { logger.info( colors.magenta( - `[${index + 1}/${silos.length}] Fetching data silo - ${silo.title}` - ) + `[${index + 1}/${silos.length}] Fetching data silo - ${silo.title}`, + ), ); const dataPoints = await fetchAllDataPoints(client, silo.id, { @@ -621,8 +621,8 @@ export async function fetchEnrichedDataSilos( colors.green( `[${index + 1}/${ silos.length - }] Successfully fetched datapoint for - ${silo.title}` - ) + }] Successfully fetched datapoint for - ${silo.title}`, + ), ); } @@ -632,8 +632,8 @@ export async function fetchEnrichedDataSilos( logger.info( colors.green( - `Successfully fetched all ${silos.length} data silo configurations` - ) + `Successfully fetched all ${silos.length} data silo configurations`, + ), ); return dataSilos; @@ -661,7 +661,7 @@ export async function syncDataSilos( dataSubjectsByName: Record; /** API key title to API key */ apiKeysByTitle: Record; - } + }, ): Promise<{ /** Whether successfully updated */ success: boolean; @@ -681,20 +681,20 @@ export async function syncDataSilos( }); // Create a mapping of title -> existing silo, if it exists - const existingDataSiloByTitle = keyBy>( + const existingDataSiloByTitle = keyBy>( existingDataSilos, - "title" + 'title', ); // Create new silos that do not exist const newDataSiloInputs = dataSilos.filter( - ({ title }) => !existingDataSiloByTitle[title] + ({ title }) => !existingDataSiloByTitle[title], ); if (newDataSiloInputs.length > 0) { logger.info( colors.magenta( - `Creating "${newDataSiloInputs.length}" data silos that did not exist...` - ) + `Creating "${newDataSiloInputs.length}" data silos that did not exist...`, + ), ); // Batch the creation @@ -706,11 +706,11 @@ export async function syncDataSilos( /** Mutation result */ createDataSilos: { /** New data silos */ - dataSilos: Pick[]; + dataSilos: Pick[]; }; }>(client, CREATE_DATA_SILOS, { input: dependencyUpdateChunk.map((input) => ({ - name: input["outer-type"] || input.integrationName, + name: input['outer-type'] || input.integrationName, title: input.title, country: input.country, countrySubDivision: input.countrySubDivision, @@ -725,8 +725,8 @@ export async function syncDataSilos( logger.info( colors.green( - `Successfully created "${newDataSiloInputs.length}" data silos!` - ) + `Successfully created "${newDataSiloInputs.length}" data silos!`, + ), ); } @@ -737,14 +737,14 @@ export async function syncDataSilos( colors.magenta( `[Batch ${ind + 1}/${chunkedUpdates.length}] Syncing "${ dataSiloUpdateChunk.length - }" data silos` - ) + }" data silos`, + ), ); await makeGraphQLRequest<{ /** Mutation result */ updateDataSilos: { /** New data silos */ - dataSilos: Pick[]; + dataSilos: Pick[]; }; }>(client, UPDATE_DATA_SILOS, { input: { @@ -755,37 +755,37 @@ export async function syncDataSilos( url: input.url, headers: input.headers, description: input.description, - identifiers: input["identity-keys"], + identifiers: input['identity-keys'], isLive: !input.disabled, ownerEmails: input.owners, teamNames: input.teams, // clear out if not specified, otherwise the update needs to be applied after // all data silos are created - dependedOnDataSiloTitles: input["deletion-dependencies"] + dependedOnDataSiloTitles: input['deletion-dependencies'] ? undefined : [], - apiKeyId: input["api-key-title"] - ? apiKeysByTitle[input["api-key-title"]].id + apiKeyId: input['api-key-title'] + ? apiKeysByTitle[input['api-key-title']].id : undefined, - dataSubjectBlockListIds: input["data-subjects"] + dataSubjectBlockListIds: input['data-subjects'] ? convertToDataSubjectBlockList( - input["data-subjects"], - dataSubjectsByName + input['data-subjects'], + dataSubjectsByName, ) : undefined, attributes: input.attributes, businessEntityTitles: input.businessEntityTitles, // AVC settings - notifyEmailAddress: input["email-settings"]?.["notify-email-address"], + notifyEmailAddress: input['email-settings']?.['notify-email-address'], promptAVendorEmailSendFrequency: - input["email-settings"]?.["send-frequency"], - promptAVendorEmailSendType: input["email-settings"]?.["send-type"], + input['email-settings']?.['send-frequency'], + promptAVendorEmailSendType: input['email-settings']?.['send-type'], promptAVendorEmailIncludeIdentifiersAttachment: - input["email-settings"]?.["include-identifiers-attachment"], + input['email-settings']?.['include-identifiers-attachment'], promptAVendorEmailCompletionLinkType: - input["email-settings"]?.["completion-link-type"], + input['email-settings']?.['completion-link-type'], manualWorkRetryFrequency: - input["email-settings"]?.["manual-work-retry-frequency"], + input['email-settings']?.['manual-work-retry-frequency'], })), }, }); @@ -793,8 +793,8 @@ export async function syncDataSilos( colors.green( `[Batch ${ind + 1}/${chunkedUpdates.length}] Synced "${ dataSiloUpdateChunk.length - }" data silos!` - ) + }" data silos!`, + ), ); }); @@ -803,18 +803,18 @@ export async function syncDataSilos( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); const dataSilosWithDataPoints = dataSilos.filter( - ({ datapoints = [] }) => datapoints.length > 0 + ({ datapoints = [] }) => datapoints.length > 0, ); const totalDataPoints = dataSilos .map(({ datapoints = [] }) => datapoints.length) .reduce((accumulator, count) => accumulator + count, 0); logger.info( colors.magenta( - `Syncing "${totalDataPoints}" datapoints from "${dataSilosWithDataPoints.length}" data silos...` - ) + `Syncing "${totalDataPoints}" datapoints from "${dataSilosWithDataPoints.length}" data silos...`, + ), ); progressBar.start(totalDataPoints, 0); let total = 0; @@ -841,21 +841,21 @@ export async function syncDataSilos( categories: categories ? categories.map((category) => ({ ...category, - name: category.name || "Other", + name: category.name || 'Other', })) : undefined, purposes: purposes ? purposes.map((purpose) => ({ ...purpose, - name: purpose.name || "Other", + name: purpose.name || 'Other', })) : undefined, attributes, accessRequestVisibilityEnabled: - rest["access-request-visibility-enabled"], + rest['access-request-visibility-enabled'], erasureRequestRedactionEnabled: - rest["erasure-request-redaction-enabled"], - }) + rest['erasure-request-redaction-enabled'], + }), ) : undefined; @@ -875,27 +875,27 @@ export async function syncDataSilos( teamNames: datapoint.teams, } : {}), - ...(datapoint["data-collection-tag"] - ? { dataCollectionTag: datapoint["data-collection-tag"] } + ...(datapoint['data-collection-tag'] + ? { dataCollectionTag: datapoint['data-collection-tag'] } : {}), - querySuggestions: datapoint["privacy-action-queries"] - ? Object.entries(datapoint["privacy-action-queries"]).map( + querySuggestions: datapoint['privacy-action-queries'] + ? Object.entries(datapoint['privacy-action-queries']).map( ([key, value]) => ({ requestType: key, suggestedQuery: value, - }) + }), ) : undefined, - enabledActions: datapoint["privacy-actions"] || [], // clear out when not specified + enabledActions: datapoint['privacy-actions'] || [], // clear out when not specified subDataPoints: fields, }; // Ensure no duplicate sub-datapoints are provided const subDataPointsToUpdate = (payload.subDataPoints || []).map( - ({ name }) => name + ({ name }) => name, ); const duplicateDataPoints = subDataPointsToUpdate.filter( - (name, index) => subDataPointsToUpdate.indexOf(name) !== index + (name, index) => subDataPointsToUpdate.indexOf(name) !== index, ); if (duplicateDataPoints.length > 0) { logger.info( @@ -903,9 +903,9 @@ export async function syncDataSilos( `\nCannot update datapoint "${ datapoint.key }" as it has duplicate sub-datapoints with the same name: \n${duplicateDataPoints.join( - "\n" - )}` - ) + '\n', + )}`, + ), ); encounteredError = true; } else { @@ -913,13 +913,13 @@ export async function syncDataSilos( await makeGraphQLRequest( client, UPDATE_OR_CREATE_DATA_POINT, - payload + payload, ); } catch (error) { logger.info( colors.red( - `\nFailed to update datapoint "${datapoint.key}" for data silo "${title}"! - \n${error.message}` - ) + `\nFailed to update datapoint "${datapoint.key}" for data silo "${title}"! - \n${error.message}`, + ), ); encounteredError = true; } @@ -931,7 +931,7 @@ export async function syncDataSilos( }, { concurrency: 10, - } + }, ); progressBar.stop(); @@ -944,8 +944,8 @@ export async function syncDataSilos( dataSilos.length }" data silos and "${totalDataPoints}" datapoints in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return { success: !encounteredError, @@ -962,13 +962,13 @@ export async function syncDataSilos( */ export async function syncDataSiloDependencies( client: GraphQLClient, - dependencyUpdates: [string, string[]][] + dependencyUpdates: [string, string[]][], ): Promise { let encounteredError = false; logger.info( colors.magenta( - `Syncing "${dependencyUpdates.length}" data silo dependencies...` - ) + `Syncing "${dependencyUpdates.length}" data silo dependencies...`, + ), ); // Batch the updates @@ -976,15 +976,15 @@ export async function syncDataSiloDependencies( await mapSeries(chunkedUpdates, async (dependencyUpdateChunk, ind) => { logger.info( colors.magenta( - `[Batch ${ind}/${dependencyUpdateChunk.length}] Updating "${dependencyUpdateChunk.length}" data silos...` - ) + `[Batch ${ind}/${dependencyUpdateChunk.length}] Updating "${dependencyUpdateChunk.length}" data silos...`, + ), ); try { await makeGraphQLRequest<{ /** Mutation result */ updateDataSilos: { /** New data silos */ - dataSilos: Pick[]; + dataSilos: Pick[]; }; }>(client, UPDATE_DATA_SILOS, { input: { @@ -992,23 +992,23 @@ export async function syncDataSiloDependencies( ([id, dependedOnDataSiloTitles]) => ({ id, dependedOnDataSiloTitles, - }) + }), ), }, }); logger.info( colors.green( `[Batch ${ind + 1}/${dependencyUpdateChunk.length}] ` + - `Synced "${dependencyUpdateChunk.length}" data silos!` - ) + `Synced "${dependencyUpdateChunk.length}" data silos!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( `[Batch ${ind + 1}/${dependencyUpdateChunk.length}] ` + - `Failed to update "${dependencyUpdateChunk.length}" silos! - ${error.message}` - ) + `Failed to update "${dependencyUpdateChunk.length}" silos! - ${error.message}`, + ), ); } }); diff --git a/src/lib/graphql/syncDataSubject.ts b/src/lib/graphql/syncDataSubject.ts index e548d91c..3f55d609 100644 --- a/src/lib/graphql/syncDataSubject.ts +++ b/src/lib/graphql/syncDataSubject.ts @@ -1,7 +1,7 @@ -import { GraphQLClient } from "graphql-request"; -import { DataSubjectInput } from "../../codecs"; -import { TOGGLE_DATA_SUBJECT, UPDATE_DATA_SUBJECT } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { GraphQLClient } from 'graphql-request'; +import { DataSubjectInput } from '../../codecs'; +import { TOGGLE_DATA_SUBJECT, UPDATE_DATA_SUBJECT } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Sync the data subjects @@ -22,7 +22,7 @@ export async function syncDataSubject( dataSubjectId: string; /** When true, skip publishing to privacy center */ skipPublish?: boolean; - } + }, ): Promise { await makeGraphQLRequest(client, UPDATE_DATA_SUBJECT, { input: { @@ -35,7 +35,7 @@ export async function syncDataSubject( }, }); - if (typeof dataSubject.active === "boolean") { + if (typeof dataSubject.active === 'boolean') { await makeGraphQLRequest(client, TOGGLE_DATA_SUBJECT, { input: { id: dataSubjectId, diff --git a/src/lib/graphql/syncEnrichers.ts b/src/lib/graphql/syncEnrichers.ts index f6e557bf..634dff1b 100644 --- a/src/lib/graphql/syncEnrichers.ts +++ b/src/lib/graphql/syncEnrichers.ts @@ -4,13 +4,13 @@ import { IsoCountrySubdivisionCode, PreflightRequestStatus, RequestAction, -} from "@transcend-io/privacy-types"; -import { GraphQLClient } from "graphql-request"; -import { EnricherInput } from "../../codecs"; -import { DataSubject } from "./fetchDataSubjects"; -import { Identifier } from "./fetchIdentifiers"; -import { CREATE_ENRICHER, ENRICHERS, UPDATE_ENRICHER } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from '@transcend-io/privacy-types'; +import { GraphQLClient } from 'graphql-request'; +import { EnricherInput } from '../../codecs'; +import { DataSubject } from './fetchDataSubjects'; +import { Identifier } from './fetchIdentifiers'; +import { CREATE_ENRICHER, ENRICHERS, UPDATE_ENRICHER } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface Enricher { /** ID of enricher */ @@ -65,7 +65,7 @@ const PAGE_SIZE = 20; */ export async function fetchAllEnrichers( client: GraphQLClient, - title?: string + title?: string, ): Promise { const enrichers: Enricher[] = []; let offset = 0; @@ -113,16 +113,16 @@ export async function syncEnricher( identifierByName: Record; /** Lookup data subject by name */ dataSubjectsByName: Record; - } + }, ): Promise { // Whether to continue looping const matches = await fetchAllEnrichers(client, enricher.title); const existingEnricher = matches.find( - ({ title }) => title === enricher.title + ({ title }) => title === enricher.title, ); // Map to data subject Ids - const dataSubjectIds = enricher["data-subjects"]?.map((subject) => { + const dataSubjectIds = enricher['data-subjects']?.map((subject) => { const existing = dataSubjectsByName[subject]; if (!existing) { throw new Error(`Failed to find a data subject with name: ${subject}`); @@ -131,9 +131,9 @@ export async function syncEnricher( }); // If enricher exists, update it, else create new - const inputIdentifier = enricher["input-identifier"]; + const inputIdentifier = enricher['input-identifier']; const actionUpdates = - enricher["privacy-actions"] || Object.values(RequestAction); + enricher['privacy-actions'] || Object.values(RequestAction); if (existingEnricher) { await makeGraphQLRequest(client, UPDATE_ENRICHER, { input: { @@ -144,19 +144,19 @@ export async function syncEnricher( testRegex: enricher.testRegex, lookerQueryTitle: enricher.lookerQueryTitle, expirationDuration: - typeof enricher.expirationDuration === "number" + typeof enricher.expirationDuration === 'number' ? enricher.expirationDuration.toString() : undefined, transitionRequestStatus: enricher.transitionRequestStatus, phoneNumbers: enricher.phoneNumbers, regionList: enricher.regionList, dataSubjectIds, - description: enricher.description || "", + description: enricher.description || '', inputIdentifier: inputIdentifier ? identifierByName[inputIdentifier].id : undefined, - identifiers: enricher["output-identifiers"].map( - (id) => identifierByName[id].id + identifiers: enricher['output-identifiers'].map( + (id) => identifierByName[id].id, ), ...(existingEnricher.type === EnricherType.Sombra ? {} @@ -173,17 +173,17 @@ export async function syncEnricher( testRegex: enricher.testRegex, lookerQueryTitle: enricher.lookerQueryTitle, expirationDuration: - typeof enricher.expirationDuration === "number" + typeof enricher.expirationDuration === 'number' ? enricher.expirationDuration.toString() : undefined, transitionRequestStatus: enricher.transitionRequestStatus, phoneNumbers: enricher.phoneNumbers, dataSubjectIds, regionList: enricher.regionList, - description: enricher.description || "", + description: enricher.description || '', inputIdentifier: identifierByName[inputIdentifier].id, - identifiers: enricher["output-identifiers"].map( - (id) => identifierByName[id].id + identifiers: enricher['output-identifiers'].map( + (id) => identifierByName[id].id, ), actions: actionUpdates, }, diff --git a/src/lib/graphql/syncIdentifier.ts b/src/lib/graphql/syncIdentifier.ts index 35edd820..dbf53249 100644 --- a/src/lib/graphql/syncIdentifier.ts +++ b/src/lib/graphql/syncIdentifier.ts @@ -1,8 +1,8 @@ -import { GraphQLClient } from "graphql-request"; -import { IdentifierInput } from "../../codecs"; -import type { DataSubject } from "./fetchDataSubjects"; -import { UPDATE_IDENTIFIER } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import { GraphQLClient } from 'graphql-request'; +import { IdentifierInput } from '../../codecs'; +import type { DataSubject } from './fetchDataSubjects'; +import { UPDATE_IDENTIFIER } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Sync the consent manager @@ -26,7 +26,7 @@ export async function syncIdentifier( identifierId: string; /** When true, skip publishing to privacy center */ skipPublish?: boolean; - } + }, ): Promise { await makeGraphQLRequest(client, UPDATE_IDENTIFIER, { input: { diff --git a/src/lib/graphql/syncIntlMessages.ts b/src/lib/graphql/syncIntlMessages.ts index d879db54..4d048f37 100644 --- a/src/lib/graphql/syncIntlMessages.ts +++ b/src/lib/graphql/syncIntlMessages.ts @@ -1,11 +1,11 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk } from "lodash-es"; -import { IntlMessageInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { UPDATE_INTL_MESSAGES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk } from 'lodash-es'; +import { IntlMessageInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { UPDATE_INTL_MESSAGES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const MAX_PAGE_SIZE = 100; @@ -17,13 +17,13 @@ const MAX_PAGE_SIZE = 100; */ export async function updateIntlMessages( client: GraphQLClient, - messageInputs: IntlMessageInput[] + messageInputs: IntlMessageInput[], ): Promise { // Batch update messages await mapSeries(chunk(messageInputs, MAX_PAGE_SIZE), async (page) => { await makeGraphQLRequest(client, UPDATE_INTL_MESSAGES, { messages: page.map((message) => ({ - ...(message.id.includes(".") ? {} : { id: message.id }), + ...(message.id.includes('.') ? {} : { id: message.id }), defaultMessage: message.defaultMessage, targetReactIntlId: message.targetReactIntlId, translations: message.translations @@ -46,30 +46,30 @@ export async function updateIntlMessages( */ export async function syncIntlMessages( client: GraphQLClient, - messages: IntlMessageInput[] + messages: IntlMessageInput[], ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${messages.length}" messages...`)); // Ensure no duplicates are being uploaded const notUnique = messages.filter( - (message) => messages.filter((pol) => message.id === pol.id).length > 1 + (message) => messages.filter((pol) => message.id === pol.id).length > 1, ); if (notUnique.length > 0) { throw new Error( `Failed to upload messages as there were non-unique entries found: ${notUnique .map(({ id }) => id) - .join(",")}` + .join(',')}`, ); } try { logger.info( - colors.magenta(`Upserting "${messages.length}" new messages...`) + colors.magenta(`Upserting "${messages.length}" new messages...`), ); await updateIntlMessages(client, messages); logger.info( - colors.green(`Successfully synced ${messages.length} messages!`) + colors.green(`Successfully synced ${messages.length} messages!`), ); } catch (error) { encounteredError = true; diff --git a/src/lib/graphql/syncPartitions.ts b/src/lib/graphql/syncPartitions.ts index cee07491..88c34c44 100644 --- a/src/lib/graphql/syncPartitions.ts +++ b/src/lib/graphql/syncPartitions.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { difference } from "lodash-es"; -import { PartitionInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { fetchConsentManagerId } from "./fetchConsentManagerId"; -import { CONSENT_PARTITIONS, CREATE_CONSENT_PARTITION } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { difference } from 'lodash-es'; +import { PartitionInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { fetchConsentManagerId } from './fetchConsentManagerId'; +import { CONSENT_PARTITIONS, CREATE_CONSENT_PARTITION } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const PAGE_SIZE = 50; @@ -26,7 +26,7 @@ export interface TranscendPartition { * @returns Partition list */ export async function fetchPartitions( - client: GraphQLClient + client: GraphQLClient, ): Promise { const partitions: TranscendPartition[] = []; let offset = 0; @@ -63,7 +63,7 @@ export async function fetchPartitions( */ export async function syncPartitions( client: GraphQLClient, - partitionInputs: PartitionInput[] + partitionInputs: PartitionInput[], ): Promise { // Grab the bundleId associated with this API key const airgapBundleId = await fetchConsentManagerId(client); @@ -71,7 +71,7 @@ export async function syncPartitions( const partitions = await fetchPartitions(client); const newPartitionNames = difference( partitionInputs.map(({ name }) => name), - partitions.map(({ name }) => name) + partitions.map(({ name }) => name), ); await mapSeries(newPartitionNames, async (name) => { try { @@ -82,13 +82,13 @@ export async function syncPartitions( }, }); logger.info( - colors.green(`Successfully created consent partition: ${name}!`) + colors.green(`Successfully created consent partition: ${name}!`), ); } catch (error) { logger.error( colors.red( - `Failed to create consent partition: ${name}! - ${error.message}` - ) + `Failed to create consent partition: ${name}! - ${error.message}`, + ), ); encounteredError = true; } diff --git a/src/lib/graphql/syncPolicies.ts b/src/lib/graphql/syncPolicies.ts index c60cda73..4ebb1846 100644 --- a/src/lib/graphql/syncPolicies.ts +++ b/src/lib/graphql/syncPolicies.ts @@ -1,13 +1,13 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy } from "lodash-es"; -import { PolicyInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { fetchAllPolicies } from "./fetchAllPolicies"; -import { fetchPrivacyCenterId } from "./fetchPrivacyCenterId"; -import { UPDATE_POLICIES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy } from 'lodash-es'; +import { PolicyInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { fetchAllPolicies } from './fetchAllPolicies'; +import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; +import { UPDATE_POLICIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const MAX_PAGE_SIZE = 100; @@ -19,7 +19,7 @@ const MAX_PAGE_SIZE = 100; */ export async function updatePolicies( client: GraphQLClient, - policyInputs: [PolicyInput, string | undefined][] + policyInputs: [PolicyInput, string | undefined][], ): Promise { const privacyCenterId = await fetchPrivacyCenterId(client); @@ -62,20 +62,20 @@ export async function updatePolicies( */ export async function syncPolicies( client: GraphQLClient, - policies: PolicyInput[] + policies: PolicyInput[], ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${policies.length}" policies...`)); // Ensure no duplicates are being uploaded const notUnique = policies.filter( - (policy) => policies.filter((pol) => policy.title === pol.title).length > 1 + (policy) => policies.filter((pol) => policy.title === pol.title).length > 1, ); if (notUnique.length > 0) { throw new Error( `Failed to upload policies as there were non-unique entries found: ${notUnique .map(({ title }) => title) - .join(",")}` + .join(',')}`, ); } @@ -83,19 +83,19 @@ export async function syncPolicies( const existingPolicies = await fetchAllPolicies(client); const policiesById = keyBy( existingPolicies, - ({ title }) => title.defaultMessage + ({ title }) => title.defaultMessage, ); try { logger.info( - colors.magenta(`Upserting "${policies.length}" new policies...`) + colors.magenta(`Upserting "${policies.length}" new policies...`), ); await updatePolicies( client, - policies.map((policy) => [policy, policiesById[policy.title]?.id]) + policies.map((policy) => [policy, policiesById[policy.title]?.id]), ); logger.info( - colors.green(`Successfully synced ${policies.length} policies!`) + colors.green(`Successfully synced ${policies.length} policies!`), ); } catch (error) { encounteredError = true; diff --git a/src/lib/graphql/syncPrivacyCenter.ts b/src/lib/graphql/syncPrivacyCenter.ts index ac8c90ca..4c6ea353 100644 --- a/src/lib/graphql/syncPrivacyCenter.ts +++ b/src/lib/graphql/syncPrivacyCenter.ts @@ -1,10 +1,10 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { PrivacyCenterInput } from "../../codecs"; -import { logger } from "../../logger"; -import { fetchPrivacyCenterId } from "./fetchPrivacyCenterId"; -import { UPDATE_PRIVACY_CENTER } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { PrivacyCenterInput } from '../../codecs'; +import { logger } from '../../logger'; +import { fetchPrivacyCenterId } from './fetchPrivacyCenterId'; +import { UPDATE_PRIVACY_CENTER } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Sync the privacy center @@ -15,10 +15,10 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function syncPrivacyCenter( client: GraphQLClient, - privacyCenter: PrivacyCenterInput + privacyCenter: PrivacyCenterInput, ): Promise { let encounteredError = false; - logger.info(colors.magenta("Syncing privacy center...")); + logger.info(colors.magenta('Syncing privacy center...')); // Grab the privacy center ID const privacyCenterId = await fetchPrivacyCenterId(client); @@ -54,11 +54,11 @@ export async function syncPrivacyCenter( : {}), }, }); - logger.info(colors.green("Successfully synced privacy center!")); + logger.info(colors.green('Successfully synced privacy center!')); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create privacy center! - ${error.message}`) + colors.red(`Failed to create privacy center! - ${error.message}`), ); } diff --git a/src/lib/graphql/syncProcessingPurposes.ts b/src/lib/graphql/syncProcessingPurposes.ts index 52a334cc..b20526dd 100644 --- a/src/lib/graphql/syncProcessingPurposes.ts +++ b/src/lib/graphql/syncProcessingPurposes.ts @@ -1,18 +1,18 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { ProcessingPurposeInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { ProcessingPurposeInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; import { fetchAllProcessingPurposes, ProcessingPurposeSubCategory, -} from "./fetchAllProcessingPurposes"; +} from './fetchAllProcessingPurposes'; import { CREATE_PROCESSING_PURPOSE_SUB_CATEGORY, UPDATE_PROCESSING_PURPOSE_SUB_CATEGORIES, -} from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new processing purpose @@ -23,8 +23,8 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createProcessingPurpose( client: GraphQLClient, - processingPurpose: ProcessingPurposeInput -): Promise> { + processingPurpose: ProcessingPurposeInput, +): Promise> { const input = { name: processingPurpose.name, purpose: processingPurpose.purpose, @@ -52,7 +52,7 @@ export async function createProcessingPurpose( */ export async function updateProcessingPurposes( client: GraphQLClient, - processingPurposeIdPairs: [ProcessingPurposeInput, string][] + processingPurposeIdPairs: [ProcessingPurposeInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_PROCESSING_PURPOSE_SUB_CATEGORIES, { input: { @@ -62,7 +62,7 @@ export async function updateProcessingPurposes( description: processingPurpose.description, // TODO: https://transcend.height.app/T-31994 - add teams, owners attributes: processingPurpose.attributes, - }) + }), ), }, }); @@ -77,11 +77,11 @@ export async function updateProcessingPurposes( */ export async function syncProcessingPurposes( client: GraphQLClient, - inputs: ProcessingPurposeInput[] + inputs: ProcessingPurposeInput[], ): Promise { // Fetch existing logger.info( - colors.magenta(`Syncing "${inputs.length}" processing purposes...`) + colors.magenta(`Syncing "${inputs.length}" processing purposes...`), ); let encounteredError = false; @@ -92,15 +92,15 @@ export async function syncProcessingPurposes( // Look up by name const processingPurposeByName: Record< string, - Pick + Pick > = keyBy( existingProcessingPurposes, - ({ name, purpose }) => `${name}:${purpose}` + ({ name, purpose }) => `${name}:${purpose}`, ); // Create new processing purposes const newProcessingPurposes = inputs.filter( - (input) => !processingPurposeByName[`${input.name}:${input.purpose}`] + (input) => !processingPurposeByName[`${input.name}:${input.purpose}`], ); // Create new processing purposes @@ -108,22 +108,22 @@ export async function syncProcessingPurposes( try { const newProcessingPurpose = await createProcessingPurpose( client, - processingPurpose + processingPurpose, ); processingPurposeByName[ `${newProcessingPurpose.name}:${newProcessingPurpose.purpose}` ] = newProcessingPurpose; logger.info( colors.green( - `Successfully synced processing purpose "${processingPurpose.name}"!` - ) + `Successfully synced processing purpose "${processingPurpose.name}"!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync processing purpose "${processingPurpose.name}"! - ${error.message}` - ) + `Failed to sync processing purpose "${processingPurpose.name}"! - ${error.message}`, + ), ); } }); @@ -131,26 +131,26 @@ export async function syncProcessingPurposes( // Update all processing purposes try { logger.info( - colors.magenta(`Updating "${inputs.length}" processing purposes!`) + colors.magenta(`Updating "${inputs.length}" processing purposes!`), ); await updateProcessingPurposes( client, inputs.map((input) => [ input, processingPurposeByName[`${input.name}:${input.purpose}`].id, - ]) + ]), ); logger.info( colors.green( - `Successfully synced "${inputs.length}" processing purposes!` - ) + `Successfully synced "${inputs.length}" processing purposes!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" processing purposes ! - ${error.message}` - ) + `Failed to sync "${inputs.length}" processing purposes ! - ${error.message}`, + ), ); } diff --git a/src/lib/graphql/syncPromptGroups.ts b/src/lib/graphql/syncPromptGroups.ts index 659249a3..178ba8d1 100644 --- a/src/lib/graphql/syncPromptGroups.ts +++ b/src/lib/graphql/syncPromptGroups.ts @@ -1,13 +1,13 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { PromptGroupInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { fetchAllPromptGroups } from "./fetchPromptGroups"; -import { fetchAllPrompts } from "./fetchPrompts"; -import { CREATE_PROMPT_GROUP, UPDATE_PROMPT_GROUPS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { PromptGroupInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllPromptGroups } from './fetchPromptGroups'; +import { fetchAllPrompts } from './fetchPrompts'; +import { CREATE_PROMPT_GROUP, UPDATE_PROMPT_GROUPS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; export interface EditPromptGroupInput { /** Title of prompt group */ @@ -27,7 +27,7 @@ export interface EditPromptGroupInput { */ export async function createPromptGroup( client: GraphQLClient, - input: EditPromptGroupInput + input: EditPromptGroupInput, ): Promise { const { createPromptGroup: { promptGroup }, @@ -44,7 +44,7 @@ export async function createPromptGroup( input, }); logger.info( - colors.green(`Successfully created prompt group "${input.title}"!`) + colors.green(`Successfully created prompt group "${input.title}"!`), ); return promptGroup.id; } @@ -57,7 +57,7 @@ export async function createPromptGroup( */ export async function updatePromptGroups( client: GraphQLClient, - input: [EditPromptGroupInput, string][] + input: [EditPromptGroupInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_PROMPT_GROUPS, { input: { @@ -68,7 +68,7 @@ export async function updatePromptGroups( }, }); logger.info( - colors.green(`Successfully updated ${input.length} prompt groups!`) + colors.green(`Successfully updated ${input.length} prompt groups!`), ); } @@ -83,18 +83,18 @@ export async function updatePromptGroups( export async function syncPromptGroups( client: GraphQLClient, promptGroups: PromptGroupInput[], - concurrency = 20 + concurrency = 20, ): Promise { let encounteredError = false; logger.info( - colors.magenta(`Syncing "${promptGroups.length}" prompt groups...`) + colors.magenta(`Syncing "${promptGroups.length}" prompt groups...`), ); // Index existing prompt groups const existing = await fetchAllPromptGroups(client); const existingPrompts = await fetchAllPrompts(client); - const promptByTitle = keyBy(existingPrompts, "title"); - const promptGroupByTitle = keyBy(existing, "title"); + const promptByTitle = keyBy(existingPrompts, 'title'); + const promptGroupByTitle = keyBy(existing, 'title'); // Determine which promptGroups are new vs existing const mapPromptGroupsToExisting = promptGroups.map((promptInput) => [ @@ -109,8 +109,8 @@ export async function syncPromptGroups( try { logger.info( colors.magenta( - `Creating "${newPromptGroups.length}" new prompt groups...` - ) + `Creating "${newPromptGroups.length}" new prompt groups...`, + ), ); await map( newPromptGroups, @@ -128,29 +128,29 @@ export async function syncPromptGroups( }, { concurrency, - } + }, ); logger.info( colors.green( - `Successfully synced ${newPromptGroups.length} prompt groups!` - ) + `Successfully synced ${newPromptGroups.length} prompt groups!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create prompt groups! - ${error.message}`) + colors.red(`Failed to create prompt groups! - ${error.message}`), ); } // Update existing promptGroups const existingPromptGroups = mapPromptGroupsToExisting.filter( - (x): x is [PromptGroupInput, string] => !!x[1] + (x): x is [PromptGroupInput, string] => !!x[1], ); try { logger.info( colors.magenta( - `Updating "${existingPromptGroups.length}" prompt groups...` - ) + `Updating "${existingPromptGroups.length}" prompt groups...`, + ), ); await updatePromptGroups( client, @@ -166,17 +166,17 @@ export async function syncPromptGroups( }), }, id, - ]) + ]), ); logger.info( colors.green( - `Successfully updated "${existingPromptGroups.length}" prompt groups!` - ) + `Successfully updated "${existingPromptGroups.length}" prompt groups!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create prompt groups! - ${error.message}`) + colors.red(`Failed to create prompt groups! - ${error.message}`), ); } diff --git a/src/lib/graphql/syncPromptPartials.ts b/src/lib/graphql/syncPromptPartials.ts index 2668f030..dcdbe2af 100644 --- a/src/lib/graphql/syncPromptPartials.ts +++ b/src/lib/graphql/syncPromptPartials.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { PromptPartialInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { fetchAllPromptPartials } from "./fetchPromptPartials"; -import { CREATE_PROMPT_PARTIAL, UPDATE_PROMPT_PARTIALS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { PromptPartialInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllPromptPartials } from './fetchPromptPartials'; +import { CREATE_PROMPT_PARTIAL, UPDATE_PROMPT_PARTIALS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Create a new prompt partial @@ -22,7 +22,7 @@ export async function createPromptPartial( title: string; /** Prompt content */ content: string; - } + }, ): Promise { const { createPromptPartial: { promptPartial }, @@ -39,7 +39,7 @@ export async function createPromptPartial( input, }); logger.info( - colors.green(`Successfully created prompt partial "${input.title}"!`) + colors.green(`Successfully created prompt partial "${input.title}"!`), ); return promptPartial.id; } @@ -52,7 +52,7 @@ export async function createPromptPartial( */ export async function updatePromptPartials( client: GraphQLClient, - input: [PromptPartialInput, string][] + input: [PromptPartialInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_PROMPT_PARTIALS, { input: { @@ -63,7 +63,7 @@ export async function updatePromptPartials( }, }); logger.info( - colors.green(`Successfully updated ${input.length} prompt partials!`) + colors.green(`Successfully updated ${input.length} prompt partials!`), ); } @@ -78,16 +78,16 @@ export async function updatePromptPartials( export async function syncPromptPartials( client: GraphQLClient, promptPartials: PromptPartialInput[], - concurrency = 20 + concurrency = 20, ): Promise { let encounteredError = false; logger.info( - colors.magenta(`Syncing "${promptPartials.length}" prompt partials...`) + colors.magenta(`Syncing "${promptPartials.length}" prompt partials...`), ); // Index existing prompt partials const existing = await fetchAllPromptPartials(client); - const promptPartialByTitle = keyBy(existing, "title"); + const promptPartialByTitle = keyBy(existing, 'title'); // Determine which promptPartials are new vs existing const mapPromptPartialsToExisting = promptPartials.map((promptInput) => [ @@ -102,8 +102,8 @@ export async function syncPromptPartials( try { logger.info( colors.magenta( - `Creating "${newPromptPartials.length}" new prompt partials...` - ) + `Creating "${newPromptPartials.length}" new prompt partials...`, + ), ); await map( newPromptPartials, @@ -112,45 +112,45 @@ export async function syncPromptPartials( }, { concurrency, - } + }, ); logger.info( colors.green( - `Successfully synced ${newPromptPartials.length} prompt partials!` - ) + `Successfully synced ${newPromptPartials.length} prompt partials!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create prompt partials! - ${error.message}`) + colors.red(`Failed to create prompt partials! - ${error.message}`), ); } // Update existing promptPartials const existingPromptPartials = mapPromptPartialsToExisting.filter( - (x): x is [PromptPartialInput, string] => !!x[1] + (x): x is [PromptPartialInput, string] => !!x[1], ); try { logger.info( colors.magenta( - `Updating "${existingPromptPartials.length}" prompt partials...` - ) + `Updating "${existingPromptPartials.length}" prompt partials...`, + ), ); await updatePromptPartials(client, existingPromptPartials); logger.info( colors.green( - `Successfully updated "${existingPromptPartials.length}" prompt partials!` - ) + `Successfully updated "${existingPromptPartials.length}" prompt partials!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create prompt partials! - ${error.message}`) + colors.red(`Failed to create prompt partials! - ${error.message}`), ); } logger.info( - colors.green(`Synced "${promptPartials.length}" prompt partials!`) + colors.green(`Synced "${promptPartials.length}" prompt partials!`), ); // Return true upon success diff --git a/src/lib/graphql/syncPrompts.ts b/src/lib/graphql/syncPrompts.ts index 4864b8cf..ba666e5f 100644 --- a/src/lib/graphql/syncPrompts.ts +++ b/src/lib/graphql/syncPrompts.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { PromptInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { fetchAllPrompts } from "./fetchPrompts"; -import { CREATE_PROMPT, UPDATE_PROMPTS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { PromptInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { fetchAllPrompts } from './fetchPrompts'; +import { CREATE_PROMPT, UPDATE_PROMPTS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Create a new prompt @@ -22,7 +22,7 @@ export async function createPrompt( title: string; /** Prompt content */ content: string; - } + }, ): Promise { const { createPrompt: { prompt }, @@ -51,7 +51,7 @@ export async function createPrompt( */ export async function updatePrompts( client: GraphQLClient, - input: [PromptInput, string][] + input: [PromptInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_PROMPTS, { input: { @@ -75,14 +75,14 @@ export async function updatePrompts( export async function syncPrompts( client: GraphQLClient, prompts: PromptInput[], - concurrency = 20 + concurrency = 20, ): Promise { let encounteredError = false; logger.info(colors.magenta(`Syncing "${prompts.length}" prompts...`)); // Index existing prompts const existing = await fetchAllPrompts(client); - const promptByTitle = keyBy(existing, "title"); + const promptByTitle = keyBy(existing, 'title'); // Determine which prompts are new vs existing const mapPromptsToExisting = prompts.map((promptInput) => [ @@ -96,7 +96,7 @@ export async function syncPrompts( .map(([promptInput]) => promptInput as PromptInput); try { logger.info( - colors.magenta(`Creating "${newPrompts.length}" new prompts...`) + colors.magenta(`Creating "${newPrompts.length}" new prompts...`), ); await map( newPrompts, @@ -105,10 +105,10 @@ export async function syncPrompts( }, { concurrency, - } + }, ); logger.info( - colors.green(`Successfully synced ${newPrompts.length} prompts!`) + colors.green(`Successfully synced ${newPrompts.length} prompts!`), ); } catch (error) { encounteredError = true; @@ -117,15 +117,15 @@ export async function syncPrompts( // Update existing prompts const existingPrompts = mapPromptsToExisting.filter( - (x): x is [PromptInput, string] => !!x[1] + (x): x is [PromptInput, string] => !!x[1], ); try { logger.info( - colors.magenta(`Updating "${existingPrompts.length}" prompts...`) + colors.magenta(`Updating "${existingPrompts.length}" prompts...`), ); await updatePrompts(client, existingPrompts); logger.info( - colors.green(`Successfully updated "${existingPrompts.length}" prompts!`) + colors.green(`Successfully updated "${existingPrompts.length}" prompts!`), ); } catch (error) { encounteredError = true; diff --git a/src/lib/graphql/syncRepositories.ts b/src/lib/graphql/syncRepositories.ts index 61a8aac9..b0d08aed 100644 --- a/src/lib/graphql/syncRepositories.ts +++ b/src/lib/graphql/syncRepositories.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy } from "lodash-es"; -import { RepositoryInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map, mapSeries } from "../bluebird-replace"; -import { fetchAllRepositories, Repository } from "./fetchAllRepositories"; -import { CREATE_REPOSITORY, UPDATE_REPOSITORIES } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy } from 'lodash-es'; +import { RepositoryInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; +import { fetchAllRepositories, Repository } from './fetchAllRepositories'; +import { CREATE_REPOSITORY, UPDATE_REPOSITORIES } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const CHUNK_SIZE = 100; @@ -34,7 +34,7 @@ export async function createRepository( teamIds?: string[]; /** Team names */ teamNames?: string[]; - } + }, ): Promise { const { createRepository: { repository }, @@ -77,7 +77,7 @@ export async function updateRepositories( teamIds?: string[]; /** Team names */ teamNames?: string[]; - }[] + }[], ): Promise { const { updateRepositories: { repositories }, @@ -93,7 +93,7 @@ export async function updateRepositories( }, }); logger.info( - colors.green(`Successfully updated ${inputs.length} repositories!`) + colors.green(`Successfully updated ${inputs.length} repositories!`), ); return repositories; } @@ -109,7 +109,7 @@ export async function updateRepositories( export async function syncRepositories( client: GraphQLClient, repositories: RepositoryInput[], - concurrency = 20 + concurrency = 20, ): Promise<{ /** The repositories that were upserted */ repositories: Repository[]; @@ -121,7 +121,7 @@ export async function syncRepositories( // Index existing repositories const existing = await fetchAllRepositories(client); - const repositoryByName = keyBy(existing, "name"); + const repositoryByName = keyBy(existing, 'name'); // Determine which repositories are new vs existing const mapRepositoriesToExisting = repositories.map((repoInput) => [ @@ -135,7 +135,9 @@ export async function syncRepositories( .map(([repoInput]) => repoInput as RepositoryInput); try { logger.info( - colors.magenta(`Creating "${newRepositories.length}" new repositories...`) + colors.magenta( + `Creating "${newRepositories.length}" new repositories...`, + ), ); await map( newRepositories, @@ -145,27 +147,27 @@ export async function syncRepositories( }, { concurrency, - } + }, ); logger.info( colors.green( - `Successfully synced ${newRepositories.length} repositories!` - ) + `Successfully synced ${newRepositories.length} repositories!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to create repositories! - ${error.message}`) + colors.red(`Failed to create repositories! - ${error.message}`), ); } // Update existing repositories const existingRepositories = mapRepositoriesToExisting.filter( - (x): x is [RepositoryInput, string] => !!x[1] + (x): x is [RepositoryInput, string] => !!x[1], ); const chunks = chunk(existingRepositories, CHUNK_SIZE); logger.info( - colors.magenta(`Updating "${existingRepositories.length}" repositories...`) + colors.magenta(`Updating "${existingRepositories.length}" repositories...`), ); await mapSeries(chunks, async (chunk) => { @@ -175,18 +177,18 @@ export async function syncRepositories( chunk.map(([input, id]) => ({ ...input, id, - })) + })), ); repos.push(...updatedRepos); logger.info( colors.green( - `Successfully updated "${existingRepositories.length}" repositories!` - ) + `Successfully updated "${existingRepositories.length}" repositories!`, + ), ); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to update repositories! - ${error.message}`) + colors.red(`Failed to update repositories! - ${error.message}`), ); } diff --git a/src/lib/graphql/syncSoftwareDevelopmentKits.ts b/src/lib/graphql/syncSoftwareDevelopmentKits.ts index 227fede5..62059d8c 100644 --- a/src/lib/graphql/syncSoftwareDevelopmentKits.ts +++ b/src/lib/graphql/syncSoftwareDevelopmentKits.ts @@ -1,19 +1,19 @@ -import { CodePackageType } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { chunk, keyBy } from "lodash-es"; -import { SoftwareDevelopmentKitInput } from "../../codecs"; -import { logger } from "../../logger"; -import { map, mapSeries } from "../bluebird-replace"; +import { CodePackageType } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, keyBy } from 'lodash-es'; +import { SoftwareDevelopmentKitInput } from '../../codecs'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; import { fetchAllSoftwareDevelopmentKits, SoftwareDevelopmentKit, -} from "./fetchAllSoftwareDevelopmentKits"; +} from './fetchAllSoftwareDevelopmentKits'; import { CREATE_SOFTWARE_DEVELOPMENT_KIT, UPDATE_SOFTWARE_DEVELOPMENT_KITS, -} from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +} from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; const CHUNK_SIZE = 100; @@ -51,7 +51,7 @@ export async function createSoftwareDevelopmentKit( teamIds?: string[]; /** Team names */ teamNames?: string[]; - } + }, ): Promise { const { createSoftwareDevelopmentKit: { softwareDevelopmentKit }, @@ -66,8 +66,8 @@ export async function createSoftwareDevelopmentKit( }); logger.info( colors.green( - `Successfully created software development kit "${input.name}"!` - ) + `Successfully created software development kit "${input.name}"!`, + ), ); return softwareDevelopmentKit; } @@ -106,7 +106,7 @@ export async function updateSoftwareDevelopmentKits( teamIds?: string[]; /** Team names */ teamNames?: string[]; - }[] + }[], ): Promise { const { updateSoftwareDevelopmentKits: { softwareDevelopmentKits }, @@ -123,8 +123,8 @@ export async function updateSoftwareDevelopmentKits( }); logger.info( colors.green( - `Successfully updated ${inputs.length} software development kits!` - ) + `Successfully updated ${inputs.length} software development kits!`, + ), ); return softwareDevelopmentKits; } @@ -140,7 +140,7 @@ export async function updateSoftwareDevelopmentKits( export async function syncSoftwareDevelopmentKits( client: GraphQLClient, softwareDevelopmentKits: SoftwareDevelopmentKitInput[], - concurrency = 20 + concurrency = 20, ): Promise<{ /** The SDKs that were upserted */ softwareDevelopmentKits: SoftwareDevelopmentKit[]; @@ -149,13 +149,13 @@ export async function syncSoftwareDevelopmentKits( }> { let encounteredError = false; const sdks: SoftwareDevelopmentKit[] = []; - logger.info(colors.magenta("Syncing software development kits...")); + logger.info(colors.magenta('Syncing software development kits...')); // Index existing software development kits const existing = await fetchAllSoftwareDevelopmentKits(client); const softwareDevelopmentKitByTitle = keyBy( existing, - ({ name, codePackageType }) => JSON.stringify({ name, codePackageType }) + ({ name, codePackageType }) => JSON.stringify({ name, codePackageType }), ); // Determine which software development kits are new vs existing @@ -168,7 +168,7 @@ export async function syncSoftwareDevelopmentKits( codePackageType: sdkInput.codePackageType, }) ]?.id, - ] + ], ); // Create the new software development kits @@ -178,8 +178,8 @@ export async function syncSoftwareDevelopmentKits( try { logger.info( colors.magenta( - `Creating "${newSoftwareDevelopmentKits.length}" new software development kits...` - ) + `Creating "${newSoftwareDevelopmentKits.length}" new software development kits...`, + ), ); await map( newSoftwareDevelopmentKits, @@ -189,32 +189,32 @@ export async function syncSoftwareDevelopmentKits( }, { concurrency, - } + }, ); logger.info( colors.green( - `Successfully synced ${newSoftwareDevelopmentKits.length} software development kits!` - ) + `Successfully synced ${newSoftwareDevelopmentKits.length} software development kits!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to create software development kits! - ${error.message}` - ) + `Failed to create software development kits! - ${error.message}`, + ), ); } // Update existing software development kits const existingSoftwareDevelopmentKits = mapSoftwareDevelopmentKitsToExisting.filter( - (x): x is [SoftwareDevelopmentKitInput, string] => !!x[1] + (x): x is [SoftwareDevelopmentKitInput, string] => !!x[1], ); const chunks = chunk(existingSoftwareDevelopmentKits, CHUNK_SIZE); logger.info( colors.magenta( - `Updating "${existingSoftwareDevelopmentKits.length}" software development kits...` - ) + `Updating "${existingSoftwareDevelopmentKits.length}" software development kits...`, + ), ); await mapSeries(chunks, async (chunk) => { @@ -225,27 +225,27 @@ export async function syncSoftwareDevelopmentKits( chunk.map(([{ codePackageType, ...input }, id]) => ({ ...input, id, - })) + })), ); sdks.push(...updatedSdks); logger.info( colors.green( - `Successfully updated "${existingSoftwareDevelopmentKits.length}" software development kits!` - ) + `Successfully updated "${existingSoftwareDevelopmentKits.length}" software development kits!`, + ), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to update software development kits! - ${error.message}` - ) + `Failed to update software development kits! - ${error.message}`, + ), ); } logger.info( colors.green( - `Synced "${softwareDevelopmentKits.length}" software development kits!` - ) + `Synced "${softwareDevelopmentKits.length}" software development kits!`, + ), ); }); diff --git a/src/lib/graphql/syncTeams.ts b/src/lib/graphql/syncTeams.ts index f22199bc..7681bee0 100644 --- a/src/lib/graphql/syncTeams.ts +++ b/src/lib/graphql/syncTeams.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { TeamInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { fetchAllTeams, Team } from "./fetchAllTeams"; -import { CREATE_TEAM, UPDATE_TEAM } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { TeamInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { fetchAllTeams, Team } from './fetchAllTeams'; +import { CREATE_TEAM, UPDATE_TEAM } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new team @@ -17,14 +17,14 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createTeam( client: GraphQLClient, - team: TeamInput -): Promise> { + team: TeamInput, +): Promise> { const input = { name: team.name, description: team.description, - ssoTitle: team["sso-title"], - ssoDepartment: team["sso-department"], - ssoGroup: team["sso-group"], + ssoTitle: team['sso-title'], + ssoDepartment: team['sso-department'], + ssoGroup: team['sso-group'], scopes: team.scopes, userEmails: team.users, }; @@ -52,8 +52,8 @@ export async function createTeam( export async function updateTeam( client: GraphQLClient, input: TeamInput, - teamId: string -): Promise> { + teamId: string, +): Promise> { const { updateTeam } = await makeGraphQLRequest<{ /** Update team mutation */ updateTeam: { @@ -65,9 +65,9 @@ export async function updateTeam( id: teamId, name: input.name, description: input.description, - ssoTitle: input["sso-title"], - ssoDepartment: input["sso-department"], - ssoGroup: input["sso-group"], + ssoTitle: input['sso-title'], + ssoDepartment: input['sso-department'], + ssoGroup: input['sso-group'], scopes: input.scopes, userEmails: input.users, }, @@ -84,7 +84,7 @@ export async function updateTeam( */ export async function syncTeams( client: GraphQLClient, - inputs: TeamInput[] + inputs: TeamInput[], ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" teams...`)); @@ -95,9 +95,9 @@ export async function syncTeams( const existingTeams = await fetchAllTeams(client); // Look up by name - const teamsByName: Record> = keyBy( + const teamsByName: Record> = keyBy( existingTeams, - "name" + 'name', ); // Create new teams @@ -113,7 +113,7 @@ export async function syncTeams( } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync team "${team.name}"! - ${error.message}`) + colors.red(`Failed to sync team "${team.name}"! - ${error.message}`), ); } }); @@ -124,14 +124,14 @@ export async function syncTeams( const newTeam = await updateTeam( client, input, - teamsByName[input.name].id + teamsByName[input.name].id, ); teamsByName[newTeam.name] = newTeam; logger.info(colors.green(`Successfully updated team "${input.name}"!`)); } catch (error) { encounteredError = true; logger.info( - colors.red(`Failed to sync team "${input.name}"! - ${error.message}`) + colors.red(`Failed to sync team "${input.name}"! - ${error.message}`), ); } }); diff --git a/src/lib/graphql/syncVendors.ts b/src/lib/graphql/syncVendors.ts index cd9e60c4..9b25814e 100644 --- a/src/lib/graphql/syncVendors.ts +++ b/src/lib/graphql/syncVendors.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { keyBy } from "lodash-es"; -import { VendorInput } from "../../codecs"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { fetchAllVendors, Vendor } from "./fetchAllVendors"; -import { CREATE_VENDOR, UPDATE_VENDORS } from "./gqls"; -import { makeGraphQLRequest } from "./makeGraphQLRequest"; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy } from 'lodash-es'; +import { VendorInput } from '../../codecs'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { fetchAllVendors, Vendor } from './fetchAllVendors'; +import { CREATE_VENDOR, UPDATE_VENDORS } from './gqls'; +import { makeGraphQLRequest } from './makeGraphQLRequest'; /** * Input to create a new vendor @@ -17,8 +17,8 @@ import { makeGraphQLRequest } from "./makeGraphQLRequest"; */ export async function createVendor( client: GraphQLClient, - vendor: VendorInput -): Promise> { + vendor: VendorInput, +): Promise> { const input = { title: vendor.title, description: vendor.description, @@ -52,7 +52,7 @@ export async function createVendor( */ export async function updateVendors( client: GraphQLClient, - vendorIdParis: [VendorInput, string][] + vendorIdParis: [VendorInput, string][], ): Promise { await makeGraphQLRequest(client, UPDATE_VENDORS, { input: { @@ -83,7 +83,7 @@ export async function updateVendors( */ export async function syncVendors( client: GraphQLClient, - inputs: VendorInput[] + inputs: VendorInput[], ): Promise { // Fetch existing logger.info(colors.magenta(`Syncing "${inputs.length}" vendors...`)); @@ -94,9 +94,9 @@ export async function syncVendors( const existingVendors = await fetchAllVendors(client); // Look up by title - const vendorByTitle: Record> = keyBy( + const vendorByTitle: Record> = keyBy( existingVendors, - "title" + 'title', ); // Create new vendors @@ -108,14 +108,14 @@ export async function syncVendors( const newVendor = await createVendor(client, vendor); vendorByTitle[newVendor.title] = newVendor; logger.info( - colors.green(`Successfully synced vendor "${vendor.title}"!`) + colors.green(`Successfully synced vendor "${vendor.title}"!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync vendor "${vendor.title}"! - ${error.message}` - ) + `Failed to sync vendor "${vendor.title}"! - ${error.message}`, + ), ); } }); @@ -125,17 +125,17 @@ export async function syncVendors( logger.info(colors.magenta(`Updating "${inputs.length}" vendors!`)); await updateVendors( client, - inputs.map((input) => [input, vendorByTitle[input.title].id]) + inputs.map((input) => [input, vendorByTitle[input.title].id]), ); logger.info( - colors.green(`Successfully synced "${inputs.length}" vendors!`) + colors.green(`Successfully synced "${inputs.length}" vendors!`), ); } catch (error) { encounteredError = true; logger.info( colors.red( - `Failed to sync "${inputs.length}" vendors ! - ${error.message}` - ) + `Failed to sync "${inputs.length}" vendors ! - ${error.message}`, + ), ); } diff --git a/src/lib/helpers/inquirer.ts b/src/lib/helpers/inquirer.ts index 911d90ca..151b8c37 100644 --- a/src/lib/helpers/inquirer.ts +++ b/src/lib/helpers/inquirer.ts @@ -1,7 +1,7 @@ -import { ObjByString } from "@transcend-io/type-utils"; -import inquirer from "inquirer"; -import autoCompletePrompt from "inquirer-autocomplete-prompt"; -import { fuzzySearch } from "../requests"; +import { ObjByString } from '@transcend-io/type-utils'; +import inquirer from 'inquirer'; +import autoCompletePrompt from 'inquirer-autocomplete-prompt'; +import { fuzzySearch } from '../requests'; /** * Inquirer confirm text @@ -20,9 +20,9 @@ export async function inquirerConfirmBoolean({ response: boolean; }>([ { - name: "response", + name: 'response', message, - type: "confirm", + type: 'confirm', }, ]); return response; @@ -45,9 +45,9 @@ export async function inquirerConfirmText({ response: string; }>([ { - name: "response", + name: 'response', message, - type: "text", + type: 'text', validate: (x) => x.trim().length > 0, }, ]); @@ -72,19 +72,19 @@ export async function inquirerAutoComplete({ /** Values to select */ values: string[]; }): Promise { - inquirer.registerPrompt("autocomplete", autoCompletePrompt); + inquirer.registerPrompt('autocomplete', autoCompletePrompt); const { response } = await inquirer.prompt<{ /** confirmation */ response: string; }>([ { - name: "response", + name: 'response', message, - type: "autocomplete", + type: 'autocomplete', default: defaultValue, source: (answersSoFar: ObjByString, input: string) => input - ? values.filter((x) => typeof x === "string" && fuzzySearch(input, x)) + ? values.filter((x) => typeof x === 'string' && fuzzySearch(input, x)) : values, }, ]); diff --git a/src/lib/helpers/parseVariablesFromString.ts b/src/lib/helpers/parseVariablesFromString.ts index e58df764..a187a7f0 100644 --- a/src/lib/helpers/parseVariablesFromString.ts +++ b/src/lib/helpers/parseVariablesFromString.ts @@ -5,16 +5,16 @@ * @returns Variables as object */ export function parseVariablesFromString( - variables: string + variables: string, ): Record { // Parse out the variables - const splitVariables = variables.split(",").filter((x) => !!x); + const splitVariables = variables.split(',').filter((x) => !!x); const variables_: Record = {}; for (const variable of splitVariables) { - const [k, v] = variable.split(":"); + const [k, v] = variable.split(':'); if (!k || !v) { throw new Error( - `Invalid variable: ${variable}. Expected format: key:value` + `Invalid variable: ${variable}. Expected format: key:value`, ); } variables_[k] = v; diff --git a/src/lib/manual-enrichment/enrichPrivacyRequest.ts b/src/lib/manual-enrichment/enrichPrivacyRequest.ts index e8999237..91b1475a 100644 --- a/src/lib/manual-enrichment/enrichPrivacyRequest.ts +++ b/src/lib/manual-enrichment/enrichPrivacyRequest.ts @@ -1,12 +1,12 @@ -import colors from "colors"; -import type { Got } from "got"; -import * as t from "io-ts"; -import { uniq } from "lodash-es"; -import { logger } from "../../logger"; -import { splitCsvToList } from "../requests/splitCsvToList"; +import colors from 'colors'; +import type { Got } from 'got'; +import * as t from 'io-ts'; +import { uniq } from 'lodash-es'; +import { logger } from '../../logger'; +import { splitCsvToList } from '../requests/splitCsvToList'; const ADMIN_URL = - "https://app.transcend.io/privacy-requests/incoming-requests/"; + 'https://app.transcend.io/privacy-requests/incoming-requests/'; /** * Minimal set required to mark as completed */ @@ -28,12 +28,12 @@ export async function enrichPrivacyRequest( sombra: Got, { id: rawId, ...rest }: EnrichPrivacyRequest, enricherId: string, - index?: number + index?: number, ): Promise { if (!rawId) { // error const message = `Request ID must be provided to enricher request.${ - index ? ` Found error in row: ${index}` : "" + index ? ` Found error in row: ${index}` : '' }`; logger.error(colors.red(message)); throw new Error(message); @@ -50,7 +50,7 @@ export async function enrichPrivacyRequest( ? accumulator : Object.assign(accumulator, { [key]: uniq(splitCsvToList(value)).map((value_) => ({ - value: key === "email" ? value_.toLowerCase() : value_, + value: key === 'email' ? value_.toLowerCase() : value_, })), }); }, {}); @@ -58,10 +58,10 @@ export async function enrichPrivacyRequest( // Make the GraphQL request try { await sombra - .post("v1/enrich-identifiers", { + .post('v1/enrich-identifiers', { headers: { - "x-transcend-request-id": id, - "x-transcend-enricher-id": enricherId, + 'x-transcend-request-id': id, + 'x-transcend-enricher-id': enricherId, }, json: { enrichedIdentifiers, @@ -70,19 +70,19 @@ export async function enrichPrivacyRequest( .json(); logger.error( - colors.green(`Successfully enriched request: ${ADMIN_URL}${id}`) + colors.green(`Successfully enriched request: ${ADMIN_URL}${id}`), ); return true; } catch (error) { // skip if already enriched if ( - typeof error.response.body === "string" && - error.response.body.includes("Cannot update a resolved RequestEnricher") + typeof error.response.body === 'string' && + error.response.body.includes('Cannot update a resolved RequestEnricher') ) { logger.warn( colors.magenta( - `Skipped enrichment for request: ${ADMIN_URL}${id}, request is no longer in the enriching phase.` - ) + `Skipped enrichment for request: ${ADMIN_URL}${id}, request is no longer in the enriching phase.`, + ), ); return false; } @@ -90,8 +90,8 @@ export async function enrichPrivacyRequest( // error logger.error( colors.red( - `Failed to enricher identifiers for request with id: ${ADMIN_URL}${id} - ${error.message} - ${error.response.body}` - ) + `Failed to enricher identifiers for request with id: ${ADMIN_URL}${id} - ${error.message} - ${error.response.body}`, + ), ); throw error; } diff --git a/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts b/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts index 053e9184..cf022bec 100644 --- a/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts +++ b/src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts @@ -1,10 +1,10 @@ -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { groupBy, uniq } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { writeCsv } from "../cron/writeCsv"; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { groupBy, uniq } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { writeCsv } from '../cron/writeCsv'; import { buildTranscendGraphQLClient, createSombraGotInstance, @@ -14,7 +14,7 @@ import { PrivacyRequest, RequestEnricher, RequestIdentifier, -} from "../graphql"; +} from '../graphql'; export interface PrivacyRequestWithIdentifiers extends PrivacyRequest { /** Request Enrichers */ @@ -57,9 +57,9 @@ export async function pullManualEnrichmentIdentifiersToCsv({ logger.info( colors.magenta( `Pulling manual enrichment requests, filtered for actions: ${requestActions.join( - "," - )}` - ) + ',', + )}`, + ), ); // Pull all privacy requests @@ -82,7 +82,7 @@ export async function pullManualEnrichmentIdentifiersToCsv({ // Check if manual enrichment exists for that request const hasManualEnrichment = requestEnrichers.filter( - ({ status }) => status === "ACTION_REQUIRED" + ({ status }) => status === 'ACTION_REQUIRED', ); // Save request to queue @@ -92,7 +92,7 @@ export async function pullManualEnrichmentIdentifiersToCsv({ sombra, { requestId: request.id, - } + }, ); savedRequests.push({ ...request, @@ -103,7 +103,7 @@ export async function pullManualEnrichmentIdentifiersToCsv({ }, { concurrency, - } + }, ); const data = savedRequests.map( @@ -116,17 +116,17 @@ export async function pullManualEnrichmentIdentifiersToCsv({ ...request, // flatten identifiers ...Object.fromEntries( - Object.entries(groupBy(requestIdentifiers, "name")).map( - ([key, values]) => [key, values.map(({ value }) => value).join(",")] - ) + Object.entries(groupBy(requestIdentifiers, 'name')).map( + ([key, values]) => [key, values.map(({ value }) => value).join(',')], + ), ), // flatten attributes ...Object.fromEntries( - Object.entries(groupBy(attributeValues, "attributeKey.name")).map( - ([key, values]) => [key, values.map(({ name }) => name).join(",")] - ) + Object.entries(groupBy(attributeValues, 'attributeKey.name')).map( + ([key, values]) => [key, values.map(({ name }) => name).join(',')], + ), ), - }) + }), ); // Write out to CSV @@ -135,8 +135,8 @@ export async function pullManualEnrichmentIdentifiersToCsv({ logger.info( colors.green( - `Successfully wrote ${savedRequests.length} requests to file "${file}"` - ) + `Successfully wrote ${savedRequests.length} requests to file "${file}"`, + ), ); return savedRequests; diff --git a/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts b/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts index 67720d5f..58e5f10c 100644 --- a/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts +++ b/src/lib/manual-enrichment/pushManualEnrichmentIdentifiersFromCsv.ts @@ -1,18 +1,18 @@ -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, makeGraphQLRequest, UPDATE_PRIVACY_REQUEST, -} from "../graphql"; -import { readCsv } from "../requests"; +} from '../graphql'; +import { readCsv } from '../requests'; import { enrichPrivacyRequest, EnrichPrivacyRequest, -} from "./enrichPrivacyRequest"; +} from './enrichPrivacyRequest'; /** * Push a CSV of enriched requests back into Transcend @@ -54,7 +54,7 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ // Notify Transcend logger.info( - colors.magenta(`Enriching "${activeResults.length}" privacy requests.`) + colors.magenta(`Enriching "${activeResults.length}" privacy requests.`), ); let successCount = 0; @@ -75,7 +75,7 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ }); logger.info( - colors.magenta(`Mark request as silent mode - ${request.id}`) + colors.magenta(`Mark request as silent mode - ${request.id}`), ); } @@ -83,7 +83,7 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ sombra, request, enricherId, - index + index, ); if (result) { successCount += 1; @@ -94,13 +94,13 @@ export async function pushManualEnrichmentIdentifiersFromCsv({ errorCount += 1; } }, - { concurrency } + { concurrency }, ); logger.info( colors.green( - `Successfully notified Transcend! \n Success count: ${successCount}.` - ) + `Successfully notified Transcend! \n Success count: ${successCount}.`, + ), ); if (skippedCount > 0) { diff --git a/src/lib/mergeTranscendInputs.ts b/src/lib/mergeTranscendInputs.ts index 71c2c8ee..f05ea2a2 100644 --- a/src/lib/mergeTranscendInputs.ts +++ b/src/lib/mergeTranscendInputs.ts @@ -1,5 +1,5 @@ -import { getEntries } from "@transcend-io/type-utils"; -import { TranscendInput } from "../codecs"; +import { getEntries } from '@transcend-io/type-utils'; +import { TranscendInput } from '../codecs'; /** * Combine a set of TranscendInput yaml files into a single yaml diff --git a/src/lib/oneTrust/helpers/convertToEmptyStrings.ts b/src/lib/oneTrust/helpers/convertToEmptyStrings.ts index 5cbff11a..067e325a 100644 --- a/src/lib/oneTrust/helpers/convertToEmptyStrings.ts +++ b/src/lib/oneTrust/helpers/convertToEmptyStrings.ts @@ -36,7 +36,7 @@ export function convertToEmptyStrings(input: T): any { // Handle null/undefined if (input === null || input === undefined) { - return ""; + return ''; } // Handle arrays @@ -45,15 +45,15 @@ export function convertToEmptyStrings(input: T): any { } // Handle objects - if (typeof input === "object") { + if (typeof input === 'object') { return Object.fromEntries( Object.entries(input).map>(([key, value]) => [ key, convertToEmptyStrings(value), - ]) + ]), ); } // Handle primitives - return ""; + return ''; } diff --git a/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts b/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts index 9e7c38f4..5dde3ee2 100644 --- a/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts +++ b/src/lib/oneTrust/helpers/parseCliSyncOtArguments.ts @@ -1,11 +1,11 @@ -import colors from "colors"; -import yargs from "yargs-parser"; +import colors from 'colors'; +import yargs from 'yargs-parser'; import { OneTrustFileFormat, OneTrustPullResource, OneTrustPullSource, -} from "../../../enums"; -import { logger } from "../../../logger"; +} from '../../../enums'; +import { logger } from '../../../logger'; const VALID_RESOURCES = Object.values(OneTrustPullResource); @@ -48,21 +48,21 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { source, } = yargs(process.argv.slice(2), { string: [ - "file", - "hostname", - "oneTrustAuth", - "resource", - "dryRun", - "transcendAuth", - "transcendUrl", - "source", + 'file', + 'hostname', + 'oneTrustAuth', + 'resource', + 'dryRun', + 'transcendAuth', + 'transcendUrl', + 'source', ], - boolean: ["debug", "dryRun"], + boolean: ['debug', 'dryRun'], default: { resource: OneTrustPullResource.Assessments, debug: false, dryRun: false, - transcendUrl: "https://api.transcend.io", + transcendUrl: 'https://api.transcend.io', source: OneTrustPullSource.OneTrust, }, }); @@ -71,16 +71,16 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (!dryRun && !transcendAuth) { logger.error( colors.red( - 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}' - ) + 'Must specify a "transcendAuth" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}', + ), ); return process.exit(1); } if (!dryRun && !transcendUrl) { logger.error( colors.red( - 'Must specify a "transcendUrl" parameter to sync resources to Transcend. e.g. --transcendUrl=https://api.transcend.io' - ) + 'Must specify a "transcendUrl" parameter to sync resources to Transcend. e.g. --transcendUrl=https://api.transcend.io', + ), ); return process.exit(1); } @@ -89,19 +89,19 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (dryRun && !file) { logger.error( colors.red( - 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json' - ) + 'Must set a "file" parameter when "dryRun" is "true". e.g. --file=./oneTrustAssessments.json', + ), ); return process.exit(1); } if (file) { - const splitFile = file.split("."); + const splitFile = file.split('.'); if (splitFile.length < 2) { logger.error( colors.red( - 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.' - ) + 'The "file" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.', + ), ); return process.exit(1); } @@ -110,8 +110,8 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { colors.red( `Expected the format of the "file" parameters '${file}' to be '${ OneTrustFileFormat.Json - }', but got '${splitFile.at(-1)}'.` - ) + }', but got '${splitFile.at(-1)}'.`, + ), ); return process.exit(1); } @@ -123,8 +123,8 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (!hostname) { logger.error( colors.red( - 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com' - ) + 'Missing required parameter "hostname". e.g. --hostname=customer.my.onetrust.com', + ), ); return process.exit(1); } @@ -132,8 +132,8 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (!oneTrustAuth) { logger.error( colors.red( - 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN' - ) + 'Missing required parameter "oneTrustAuth". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN', + ), ); return process.exit(1); } @@ -142,8 +142,8 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (!file) { logger.error( colors.red( - 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json' - ) + 'Must specify a "file" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json', + ), ); return process.exit(1); } @@ -152,9 +152,9 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { if (dryRun) { logger.error( colors.red( - "Cannot read and write to a file simultaneously." + - ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.` - ) + 'Cannot read and write to a file simultaneously.' + + ` Emit the "source" parameter or set it to ${OneTrustPullSource.OneTrust} if "dryRun" is enabled.`, + ), ); return process.exit(1); } @@ -164,9 +164,9 @@ export const parseCliSyncOtArguments = (): OneTrustCliArguments => { logger.error( colors.red( `Received invalid resource value: "${resource}". Allowed: ${VALID_RESOURCES.join( - "," - )}` - ) + ',', + )}`, + ), ); return process.exit(1); } diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts index 885e33d5..ee2a9126 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToDisk.ts @@ -1,8 +1,8 @@ -import fs from "node:fs"; -import { OneTrustEnrichedAssessment } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { logger } from "../../../logger"; -import { oneTrustAssessmentToJson } from "./oneTrustAssessmentToJson"; +import fs from 'node:fs'; +import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { logger } from '../../../logger'; +import { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson'; /** * Write the assessment to disk at the specified file path. @@ -29,8 +29,8 @@ export const syncOneTrustAssessmentToDisk = ({ colors.magenta( `Writing enriched assessment ${ index + 1 - } of ${total} to file "${file}"...` - ) + } of ${total} to file "${file}"...`, + ), ); if (index === 0) { @@ -41,7 +41,7 @@ export const syncOneTrustAssessmentToDisk = ({ index, total, wrap: false, - }) + }), ); } else { fs.appendFileSync( @@ -51,7 +51,7 @@ export const syncOneTrustAssessmentToDisk = ({ index, total, wrap: false, - }) + }), ); } }; diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts index 80127ee8..e18c1d26 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts @@ -1,13 +1,13 @@ -import { OneTrustEnrichedAssessment } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { ImportOnetrustAssessmentsInput } from "../../../codecs"; -import { logger } from "../../../logger"; +import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { ImportOnetrustAssessmentsInput } from '../../../codecs'; +import { logger } from '../../../logger'; import { IMPORT_ONE_TRUST_ASSESSMENT_FORMS, makeGraphQLRequest, -} from "../../graphql"; -import { oneTrustAssessmentToJson } from "./oneTrustAssessmentToJson"; +} from '../../graphql'; +import { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson'; export interface AssessmentForm { /** ID of Assessment Form */ @@ -40,9 +40,9 @@ export const syncOneTrustAssessmentToTranscend = async ({ logger.info( colors.magenta( `Writing enriched assessment ${index + 1} ${ - total ? `of ${total} ` : " " - }to Transcend...` - ) + total ? `of ${total} ` : ' ' + }to Transcend...`, + ), ); // convert the OneTrust assessment object into a json record @@ -71,10 +71,10 @@ export const syncOneTrustAssessmentToTranscend = async ({ logger.error( colors.red( `Failed to sync assessment ${index + 1} ${ - total ? `of ${total} ` : " " + total ? `of ${total} ` : ' ' }to Transcend.\n` + - `\tAssessment Title: ${assessment.name}. Template Title: ${assessment.template.name}\n` - ) + `\tAssessment Title: ${assessment.name}. Template Title: ${assessment.template.name}\n`, + ), ); } }; diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts index 3d9b69a4..e2d3ae23 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromFile.ts @@ -1,11 +1,11 @@ -import { createReadStream } from "node:fs"; -import { OneTrustEnrichedAssessment } from "@transcend-io/privacy-types"; -import { decodeCodec } from "@transcend-io/type-utils"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import JSONStream from "JSONStream"; -import { logger } from "../../../logger"; -import { syncOneTrustAssessmentToTranscend } from "./syncOneTrustAssessmentToTranscend"; +import { createReadStream } from 'node:fs'; +import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import JSONStream from 'JSONStream'; +import { logger } from '../../../logger'; +import { syncOneTrustAssessmentToTranscend } from './syncOneTrustAssessmentToTranscend'; /** * Reads assessments from a file and syncs them to Transcend. @@ -26,12 +26,12 @@ export const syncOneTrustAssessmentsFromFile = ({ return new Promise((resolve, reject) => { // Create a readable stream from the file const fileStream = createReadStream(file, { - encoding: "utf-8", + encoding: 'utf-8', highWaterMark: 64 * 1024, // 64KB chunks }); // Create a JSONStream parser to parse the array of OneTrust assessments from the file - const parser = JSONStream.parse("*"); // '*' matches each element in the root array + const parser = JSONStream.parse('*'); // '*' matches each element in the root array let index = 0; @@ -39,7 +39,7 @@ export const syncOneTrustAssessmentsFromFile = ({ fileStream.pipe(parser); // Handle each parsed assessment object - parser.on("data", async (assessment) => { + parser.on('data', async (assessment) => { try { // Pause the stream while processing to avoid overwhelming memory parser.pause(); @@ -47,7 +47,7 @@ export const syncOneTrustAssessmentsFromFile = ({ // Decode and validate the assessment const parsedAssessment = decodeCodec( OneTrustEnrichedAssessment, - assessment + assessment, ); // Sync the assessment to transcend @@ -65,29 +65,29 @@ export const syncOneTrustAssessmentsFromFile = ({ // if failed to parse a line, report error and continue logger.error( colors.red( - `Failed to parse the assessment ${index} from file '${file}': ${error.message}.` - ) + `Failed to parse the assessment ${index} from file '${file}': ${error.message}.`, + ), ); } }); // Handle completion - parser.on("end", () => { + parser.on('end', () => { logger.info(`Finished processing ${index} assessments from file ${file}`); resolve(); }); // Handle stream or parsing errors - parser.on("error", (error) => { + parser.on('error', (error) => { logger.error( - colors.red(`Error parsing file '${file}': ${error.message}`) + colors.red(`Error parsing file '${file}': ${error.message}`), ); reject(error); }); - fileStream.on("error", (error) => { + fileStream.on('error', (error) => { logger.error( - colors.red(`Error reading file '${file}': ${error.message}`) + colors.red(`Error reading file '${file}': ${error.message}`), ); reject(error); }); diff --git a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts index d33d0897..c06eb645 100644 --- a/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts +++ b/src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts @@ -4,22 +4,22 @@ import { OneTrustEnrichedAssessment, OneTrustGetRiskResponse, OneTrustGetUserResponse, -} from "@transcend-io/privacy-types"; -import colors from "colors"; -import type { Got } from "got"; -import { GraphQLClient } from "graphql-request"; -import { uniq } from "lodash-es"; -import { logger } from "../../../logger"; -import { map, mapSeries } from "../../bluebird-replace"; +} from '@transcend-io/privacy-types'; +import colors from 'colors'; +import type { Got } from 'got'; +import { GraphQLClient } from 'graphql-request'; +import { uniq } from 'lodash-es'; +import { logger } from '../../../logger'; +import { map, mapSeries } from '../../bluebird-replace'; import { getListOfOneTrustAssessments, getOneTrustAssessment, getOneTrustRisk, getOneTrustUser, -} from "../endpoints"; -import { enrichOneTrustAssessment } from "./enrichOneTrustAssessment"; -import { syncOneTrustAssessmentToDisk } from "./syncOneTrustAssessmentToDisk"; -import { syncOneTrustAssessmentToTranscend } from "./syncOneTrustAssessmentToTranscend"; +} from '../endpoints'; +import { enrichOneTrustAssessment } from './enrichOneTrustAssessment'; +import { syncOneTrustAssessmentToDisk } from './syncOneTrustAssessmentToDisk'; +import { syncOneTrustAssessmentToTranscend } from './syncOneTrustAssessmentToTranscend'; export interface AssessmentForm { /** ID of Assessment Form */ @@ -49,7 +49,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ file?: string; }): Promise => { // fetch the list of all assessments in the OneTrust organization - logger.info("Getting list of all assessments from OneTrust..."); + logger.info('Getting list of all assessments from OneTrust...'); const assessments = await getListOfOneTrustAssessments({ oneTrust }); // a cache of OneTrust users so we avoid requesting already fetched users @@ -62,7 +62,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ length: Math.ceil(assessments.length / BATCH_SIZE), }, (_, index) => - assessments.slice(index * BATCH_SIZE, (index + 1) * BATCH_SIZE) + assessments.slice(index * BATCH_SIZE, (index + 1) * BATCH_SIZE), ); // process each batch and sync the batch right away so it's garbage collected and we don't run out of memory @@ -75,7 +75,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ async (assessment, index) => { const assessmentNumber = BATCH_SIZE * batch + index + 1; logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching details...` + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching details...`, ); const { templateName, assessmentId } = assessment; const assessmentDetails = await getOneTrustAssessment({ @@ -87,7 +87,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ let creator = oneTrustCachedUsers[creatorId]; if (!creator) { logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching creator...` + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching creator...`, ); try { creator = await getOneTrustUser({ @@ -99,8 +99,8 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ logger.warn( colors.yellow( `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch form creator.` + - `\tcreatorId: ${creatorId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}` - ) + `\tcreatorId: ${creatorId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`, + ), ); } } @@ -110,7 +110,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ let approversDetails: OneTrustGetUserResponse[][] = []; if (approvers.length > 0) { logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching approvers...` + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching approvers...`, ); approversDetails = await map( approvers.map(({ id }) => id), @@ -126,13 +126,13 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ logger.warn( colors.yellow( `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch a form approver.` + - `\tapproverId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}` - ) + `\tapproverId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`, + ), ); return []; } }, - { concurrency: 5 } + { concurrency: 5 }, ); } @@ -140,12 +140,12 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ const { respondents } = assessmentDetails; // if a user is an internal respondents, their 'name' field can't be an email. const internalRespondents = respondents.filter( - (r) => !r.name.includes("@") + (r) => !r.name.includes('@'), ); let respondentsDetails: OneTrustGetUserResponse[][] = []; if (internalRespondents.length > 0) { logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching respondents...` + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching respondents...`, ); respondentsDetails = await map( internalRespondents.map(({ id }) => id), @@ -161,13 +161,13 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ logger.warn( colors.yellow( `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch a respondent.` + - `\trespondentId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}` - ) + `\trespondentId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`, + ), ); return []; } }, - { concurrency: 5 } + { concurrency: 5 }, ); } @@ -176,20 +176,20 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ const riskIds = uniq( assessmentDetails.sections.flatMap((s: OneTrustAssessmentSection) => s.questions.flatMap((q: OneTrustAssessmentQuestion) => - (q.risks ?? []).flatMap((r) => r.riskId) - ) - ) + (q.risks ?? []).flatMap((r) => r.riskId), + ), + ), ); if (riskIds.length > 0) { logger.info( - `[assessment ${assessmentNumber} of ${assessments.length}]: fetching risks...` + `[assessment ${assessmentNumber} of ${assessments.length}]: fetching risks...`, ); riskDetails = await map( riskIds, (riskId) => getOneTrustRisk({ oneTrust, riskId: riskId }), { concurrency: 5, - } + }, ); } @@ -205,7 +205,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ batchEnrichedAssessments.push(enrichedAssessment); }, - { concurrency: BATCH_SIZE } + { concurrency: BATCH_SIZE }, ); // sync assessments in series to avoid concurrency bugs @@ -232,7 +232,7 @@ export const syncOneTrustAssessmentsFromOneTrust = async ({ index: globalIndex, }); } - } + }, ); }); }; diff --git a/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts b/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts index 91024c53..a5de0dd5 100644 --- a/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts +++ b/src/lib/oneTrust/helpers/tests/convertToEmptyStrings.test.ts @@ -1,51 +1,51 @@ -import { describe, expect, it } from "vitest"; -import { convertToEmptyStrings } from "../convertToEmptyStrings"; +import { describe, expect, it } from 'vitest'; +import { convertToEmptyStrings } from '../convertToEmptyStrings'; -describe("buildDefaultCodecWrapper", () => { - it("should correctly build a default codec for null", () => { +describe('buildDefaultCodecWrapper', () => { + it('should correctly build a default codec for null', () => { const result = convertToEmptyStrings(null); - expect(result).to.equal(""); + expect(result).to.equal(''); }); - it("should correctly build a default codec for number", () => { + it('should correctly build a default codec for number', () => { const result = convertToEmptyStrings(0); - expect(result).to.equal(""); + expect(result).to.equal(''); }); - it("should correctly build a default codec for boolean", () => { + it('should correctly build a default codec for boolean', () => { const result = convertToEmptyStrings(false); - expect(result).to.equal(""); + expect(result).to.equal(''); }); - it("should correctly build a default codec for undefined", () => { + it('should correctly build a default codec for undefined', () => { const result = convertToEmptyStrings(); - expect(result).to.equal(""); + expect(result).to.equal(''); }); - it("should correctly build a default codec for string", () => { - const result = convertToEmptyStrings("1"); - expect(result).to.equal(""); + it('should correctly build a default codec for string', () => { + const result = convertToEmptyStrings('1'); + expect(result).to.equal(''); }); - it("should correctly build a default codec for an object", () => { - const result = convertToEmptyStrings({ name: "joe" }); - expect(result).to.deep.equal({ name: "" }); + it('should correctly build a default codec for an object', () => { + const result = convertToEmptyStrings({ name: 'joe' }); + expect(result).to.deep.equal({ name: '' }); }); - it("should correctly build a default codec for an array of primitive types", () => { - const result = convertToEmptyStrings(["name", 0, false]); - expect(result).to.deep.equal(["", "", ""]); + it('should correctly build a default codec for an array of primitive types', () => { + const result = convertToEmptyStrings(['name', 0, false]); + expect(result).to.deep.equal(['', '', '']); }); - it("should correctly build a default codec for an array of object", () => { + it('should correctly build a default codec for an array of object', () => { const result = convertToEmptyStrings([ - { name: "john", age: 52 }, - { name: "jane", age: 15, isAdult: true }, + { name: 'john', age: 52 }, + { name: 'jane', age: 15, isAdult: true }, ]); // should default to the array with object if the union contains an array of objects expect(result).to.deep.equal([ - { name: "", age: "" }, - { name: "", age: "", isAdult: "" }, + { name: '', age: '' }, + { name: '', age: '', isAdult: '' }, ]); }); }); diff --git a/src/lib/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts b/src/lib/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts index 3a822364..e9e464a3 100644 --- a/src/lib/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts +++ b/src/lib/preference-management/checkIfPendingPreferenceUpdatesAreNoOp.ts @@ -2,8 +2,8 @@ import { PreferenceQueryResponseItem, PreferenceStorePurposeResponse, PreferenceTopicType, -} from "@transcend-io/privacy-types"; -import { PreferenceTopic } from "../graphql"; +} from '@transcend-io/privacy-types'; +import { PreferenceTopic } from '../graphql'; /** * Check if the pending set of updates are exactly the same as the current consent record. @@ -21,7 +21,7 @@ export function checkIfPendingPreferenceUpdatesAreNoOp({ /** The pending updates */ pendingUpdates: Record< string, - Omit + Omit >; /** The preference topic configurations */ preferenceTopics: PreferenceTopic[]; @@ -31,7 +31,7 @@ export function checkIfPendingPreferenceUpdatesAreNoOp({ ([purposeName, { preferences = [], enabled }]) => { // Ensure the purpose exists const currentPurpose = currentConsentRecord.purposes.find( - (existingPurpose) => existingPurpose.purpose === purposeName + (existingPurpose) => existingPurpose.purpose === purposeName, ); // Ensure purpose.enabled is in sync @@ -53,7 +53,7 @@ export function checkIfPendingPreferenceUpdatesAreNoOp({ // Determine type of preference topic const preferenceTopic = preferenceTopics.find( - (x) => x.slug === topic && x.purpose.trackingType === purposeName + (x) => x.slug === topic && x.purpose.trackingType === purposeName, ); if (!preferenceTopic) { throw new Error(`Could not find preference topic for ${topic}`); @@ -80,18 +80,18 @@ export function checkIfPendingPreferenceUpdatesAreNoOp({ return ( sortedCurrentValues.length === sortedNewValues.length && sortedCurrentValues.every( - (x, index) => x === sortedNewValues[index] + (x, index) => x === sortedNewValues[index], ) ); } default: { throw new Error( - `Unknown preference topic type: ${preferenceTopic.type}` + `Unknown preference topic type: ${preferenceTopic.type}`, ); } } - }) + }), ); - } + }, ); } diff --git a/src/lib/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts b/src/lib/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts index 06bd39b7..276d170f 100644 --- a/src/lib/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts +++ b/src/lib/preference-management/checkIfPendingPreferenceUpdatesCauseConflict.ts @@ -2,8 +2,8 @@ import { PreferenceQueryResponseItem, PreferenceStorePurposeResponse, PreferenceTopicType, -} from "@transcend-io/privacy-types"; -import { PreferenceTopic } from "../graphql"; +} from '@transcend-io/privacy-types'; +import { PreferenceTopic } from '../graphql'; /** * Check if the pending set of updates will result in a change of @@ -22,7 +22,7 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ /** The pending updates */ pendingUpdates: Record< string, - Omit + Omit >; /** The preference topic configurations */ preferenceTopics: PreferenceTopic[]; @@ -32,7 +32,7 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ ([purposeName, { preferences = [], enabled }]) => { // Ensure the purpose exists const currentPurpose = currentConsentRecord.purposes.find( - (existingPurpose) => existingPurpose.purpose === purposeName + (existingPurpose) => existingPurpose.purpose === purposeName, ); // If no purpose exists, then it is not a conflict @@ -49,7 +49,7 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ return !!preferences.find(({ topic, choice }) => { // find matching topic const currentPreference = (currentPurpose.preferences || []).find( - (existingPreference) => existingPreference.topic === topic + (existingPreference) => existingPreference.topic === topic, ); // if no topic exists, no conflict @@ -59,7 +59,7 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ // Determine type of preference topic const preferenceTopic = preferenceTopics.find( - (x) => x.slug === topic && x.purpose.trackingType === purposeName + (x) => x.slug === topic && x.purpose.trackingType === purposeName, ); if (!preferenceTopic) { throw new Error(`Could not find preference topic for ${topic}`); @@ -84,17 +84,17 @@ export function checkIfPendingPreferenceUpdatesCauseConflict({ return ( sortedCurrentValues.length !== sortedNewValues.length || !sortedCurrentValues.every( - (x, index) => x === sortedNewValues[index] + (x, index) => x === sortedNewValues[index], ) ); } default: { throw new Error( - `Unknown preference topic type: ${preferenceTopic.type}` + `Unknown preference topic type: ${preferenceTopic.type}`, ); } } }); - } + }, ); } diff --git a/src/lib/preference-management/getPreferenceUpdatesFromRow.ts b/src/lib/preference-management/getPreferenceUpdatesFromRow.ts index 7d9a5801..da4b3ac3 100644 --- a/src/lib/preference-management/getPreferenceUpdatesFromRow.ts +++ b/src/lib/preference-management/getPreferenceUpdatesFromRow.ts @@ -1,11 +1,11 @@ import { PreferenceStorePurposeResponse, PreferenceTopicType, -} from "@transcend-io/privacy-types"; -import { apply } from "@transcend-io/type-utils"; -import { PreferenceTopic } from "../graphql"; -import { splitCsvToList } from "../requests"; -import { PurposeRowMapping } from "./codecs"; +} from '@transcend-io/privacy-types'; +import { apply } from '@transcend-io/type-utils'; +import { PreferenceTopic } from '../graphql'; +import { splitCsvToList } from '../requests'; +import { PurposeRowMapping } from './codecs'; /** * Parse an arbitrary object to the Transcend PUT /v1/preference update shape @@ -42,7 +42,7 @@ export function getPreferenceUpdatesFromRow({ purposeSlugs: string[]; /** The preference topics */ preferenceTopics: PreferenceTopic[]; -}): Record> { +}): Record> { // Create a result object to store the parsed preferences const result: Record> = {}; @@ -54,14 +54,14 @@ export function getPreferenceUpdatesFromRow({ // Ensure the purpose is valid if (!purposeSlugs.includes(purpose)) { throw new Error( - `Invalid purpose slug: ${purpose}, expected: ${purposeSlugs.join(", ")}` + `Invalid purpose slug: ${purpose}, expected: ${purposeSlugs.join(', ')}`, ); } // CHeck if parsing a preference or just the top level purpose if (preference) { const preferenceTopic = preferenceTopics.find( - (x) => x.slug === preference && x.purpose.trackingType === purpose + (x) => x.slug === preference && x.purpose.trackingType === purpose, ); if (!preferenceTopic) { const allowedTopics = preferenceTopics @@ -70,8 +70,8 @@ export function getPreferenceUpdatesFromRow({ throw new Error( `Invalid preference slug: ${preference} for purpose: ${purpose}. ` + `Allowed preference slugs for purpose are: ${allowedTopics.join( - "," - )}` + ',', + )}`, ); } @@ -89,14 +89,14 @@ export function getPreferenceUpdatesFromRow({ const rawValue = row[columnName]; const rawMapping = valueMapping[rawValue]; const trimmedMapping = - typeof rawMapping === "string" ? rawMapping.trim() || null : null; + typeof rawMapping === 'string' ? rawMapping.trim() || null : null; // handle each type of preference switch (preferenceTopic.type) { case PreferenceTopicType.Boolean: { - if (typeof rawMapping !== "boolean") { + if (typeof rawMapping !== 'boolean') { throw new TypeError( - `Invalid value for boolean preference: ${preference}, expected boolean, got: ${rawValue}` + `Invalid value for boolean preference: ${preference}, expected boolean, got: ${rawValue}`, ); } result[purpose].preferences.push({ @@ -108,9 +108,9 @@ export function getPreferenceUpdatesFromRow({ break; } case PreferenceTopicType.Select: { - if (typeof rawMapping !== "string" && rawMapping !== null) { + if (typeof rawMapping !== 'string' && rawMapping !== null) { throw new Error( - `Invalid value for select preference: ${preference}, expected string or null, got: ${rawValue}` + `Invalid value for select preference: ${preference}, expected string or null, got: ${rawValue}`, ); } @@ -124,7 +124,7 @@ export function getPreferenceUpdatesFromRow({ `Invalid value for select preference: ${preference}, expected one of: ` + `${preferenceTopic.preferenceOptionValues .map(({ slug }) => slug) - .join(", ")}, got: ${rawValue}` + .join(', ')}, got: ${rawValue}`, ); } @@ -138,9 +138,9 @@ export function getPreferenceUpdatesFromRow({ break; } case PreferenceTopicType.MultiSelect: { - if (typeof rawValue !== "string") { + if (typeof rawValue !== 'string') { throw new TypeError( - `Invalid value for multi select preference: ${preference}, expected string, got: ${rawValue}` + `Invalid value for multi select preference: ${preference}, expected string, got: ${rawValue}`, ); } // Update preferences @@ -150,12 +150,12 @@ export function getPreferenceUpdatesFromRow({ selectValues: splitCsvToList(rawValue) .map((value) => { const result = valueMapping[value]; - if (typeof result !== "string") { + if (typeof result !== 'string') { throw new TypeError( `Invalid value for multi select preference: ${preference}, ` + `expected one of: ${preferenceTopic.preferenceOptionValues .map(({ slug }) => slug) - .join(", ")}, got: ${value}` + .join(', ')}, got: ${value}`, ); } return result; @@ -182,9 +182,9 @@ export function getPreferenceUpdatesFromRow({ // Ensure that enabled is provided return apply(result, (x, purposeName) => { - if (typeof x.enabled !== "boolean") { + if (typeof x.enabled !== 'boolean') { throw new TypeError( - `No mapping provided for purpose.enabled=true/false value: ${purposeName}` + `No mapping provided for purpose.enabled=true/false value: ${purposeName}`, ); } return { diff --git a/src/lib/preference-management/getPreferencesForIdentifiers.ts b/src/lib/preference-management/getPreferencesForIdentifiers.ts index 3f523344..5fd84c25 100644 --- a/src/lib/preference-management/getPreferencesForIdentifiers.ts +++ b/src/lib/preference-management/getPreferencesForIdentifiers.ts @@ -1,12 +1,12 @@ -import { PreferenceQueryResponseItem } from "@transcend-io/privacy-types"; -import { decodeCodec } from "@transcend-io/type-utils"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import type { Got } from "got"; -import * as t from "io-ts"; -import { chunk } from "lodash-es"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { PreferenceQueryResponseItem } from '@transcend-io/privacy-types'; +import { decodeCodec } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import type { Got } from 'got'; +import * as t from 'io-ts'; +import { chunk } from 'lodash-es'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; const PreferenceRecordsQueryResponse = t.intersection([ t.type({ @@ -19,10 +19,10 @@ const PreferenceRecordsQueryResponse = t.intersection([ ]); const MSGS = [ - "ENOTFOUND", - "ETIMEDOUT", - "504 Gateway Time-out", - "Task timed out after", + 'ENOTFOUND', + 'ETIMEDOUT', + '504 Gateway Time-out', + 'Task timed out after', ]; /** @@ -48,7 +48,7 @@ export async function getPreferencesForIdentifiers( partitionKey: string; /** Whether to skip logging */ skipLogging?: boolean; - } + }, ): Promise { const results: PreferenceQueryResponseItem[] = []; const groupedIdentifiers = chunk(identifiers, 100); @@ -57,7 +57,7 @@ export async function getPreferencesForIdentifiers( const t0 = Date.now(); const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); if (!skipLogging) { progressBar.start(identifiers.length, 0); @@ -90,28 +90,28 @@ export async function getPreferencesForIdentifiers( break; // Exit loop if successful } catch (error) { attempts += 1; - const message = error?.response?.body || error?.message || ""; + const message = error?.response?.body || error?.message || ''; if ( attempts >= maxAttempts || !MSGS.some((errorMessage) => message.includes(errorMessage)) ) { throw new Error( - `Received an error from server after ${attempts} attempts: ${message}` + `Received an error from server after ${attempts} attempts: ${message}`, ); } logger.warn( colors.yellow( `[RETRYING FAILED REQUEST - Attempt ${attempts}] ` + - `Failed to fetch ${group.length} user preferences from partition ${partitionKey}: ${message}` - ) + `Failed to fetch ${group.length} user preferences from partition ${partitionKey}: ${message}`, + ), ); } } }, { concurrency: 40, - } + }, ); progressBar.stop(); @@ -121,7 +121,7 @@ export async function getPreferencesForIdentifiers( if (!skipLogging) { // Log completion time logger.info( - colors.green(`Completed download in "${totalTime / 1000}" seconds.`) + colors.green(`Completed download in "${totalTime / 1000}" seconds.`), ); } diff --git a/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts b/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts index bd7240f1..c0845f58 100644 --- a/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts @@ -1,12 +1,12 @@ -import { PreferenceTopicType } from "@transcend-io/privacy-types"; -import colors from "colors"; -import inquirer from "inquirer"; -import { difference, uniq } from "lodash-es"; -import { logger } from "../../logger"; -import { mapSeries } from "../bluebird-replace"; -import { PreferenceTopic } from "../graphql"; -import { splitCsvToList } from "../requests"; -import { FileMetadataState } from "./codecs"; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import { difference, uniq } from 'lodash-es'; +import { logger } from '../../logger'; +import { mapSeries } from '../bluebird-replace'; +import { PreferenceTopic } from '../graphql'; +import { splitCsvToList } from '../requests'; +import { FileMetadataState } from './codecs'; /** * Parse out the purpose.enabled and preference values from a CSV file @@ -30,7 +30,7 @@ export async function parsePreferenceAndPurposeValuesFromCsv( preferenceTopics: PreferenceTopic[]; /** Force workflow triggers */ forceTriggerWorkflows: boolean; - } + }, ): Promise { // Determine columns to map const columnNames = uniq(preferences.flatMap((x) => Object.keys(x))); @@ -44,7 +44,7 @@ export async function parsePreferenceAndPurposeValuesFromCsv( if (forceTriggerWorkflows) { return currentState; } - throw new Error("No other columns to process"); + throw new Error('No other columns to process'); } // The purpose and preferences to map to @@ -63,8 +63,8 @@ export async function parsePreferenceAndPurposeValuesFromCsv( if (purposeMapping) { logger.info( colors.magenta( - `Column "${col}" is associated with purpose "${purposeMapping.purpose}"` - ) + `Column "${col}" is associated with purpose "${purposeMapping.purpose}"`, + ), ); } else { const { purposeName } = await inquirer.prompt<{ @@ -72,14 +72,14 @@ export async function parsePreferenceAndPurposeValuesFromCsv( purposeName: string; }>([ { - name: "purposeName", + name: 'purposeName', message: `Choose the purpose that column ${col} is associated with`, - type: "list", + type: 'list', default: purposeNames.find((x) => x.startsWith(purposeSlugs[0])), choices: purposeNames, }, ]); - const [purposeSlug, preferenceSlug] = purposeName.split("->"); + const [purposeSlug, preferenceSlug] = purposeName.split('->'); purposeMapping = { purpose: purposeSlug, preference: preferenceSlug || null, @@ -92,8 +92,8 @@ export async function parsePreferenceAndPurposeValuesFromCsv( if (purposeMapping.valueMapping[value] !== undefined) { logger.info( colors.magenta( - `Value "${value}" is associated with purpose value "${purposeMapping.valueMapping[value]}"` - ) + `Value "${value}" is associated with purpose value "${purposeMapping.valueMapping[value]}"`, + ), ); return; } @@ -104,10 +104,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( purposeValue: boolean; }>([ { - name: "purposeValue", + name: 'purposeValue', message: `Choose the purpose value for value "${value}" associated with purpose "${purposeMapping.purpose}"`, - type: "confirm", - default: value !== "false", + type: 'confirm', + default: value !== 'false', }, ]); purposeMapping.valueMapping[value] = purposeValue; @@ -116,18 +116,18 @@ export async function parsePreferenceAndPurposeValuesFromCsv( // if preference is not null, this column is for a specific preference if (purposeMapping.preference !== null) { const preferenceTopic = preferenceTopics.find( - (x) => x.slug === purposeMapping.preference + (x) => x.slug === purposeMapping.preference, ); if (!preferenceTopic) { logger.error( colors.red( - `Preference topic "${purposeMapping.preference}" not found` - ) + `Preference topic "${purposeMapping.preference}" not found`, + ), ); return; } const preferenceOptions = preferenceTopic.preferenceOptionValues.map( - ({ slug }) => slug + ({ slug }) => slug, ); if (preferenceTopic.type === PreferenceTopicType.Boolean) { @@ -136,10 +136,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( preferenceValue: boolean; }>([ { - name: "preferenceValue", + name: 'preferenceValue', message: `Choose the preference value for "${preferenceTopic.slug}" value "${value}" associated with purpose "${purposeMapping.purpose}"`, - type: "confirm", - default: value !== "false", + type: 'confirm', + default: value !== 'false', }, ]); purposeMapping.valueMapping[value] = preferenceValue; @@ -152,10 +152,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( preferenceValue: boolean; }>([ { - name: "preferenceValue", + name: 'preferenceValue', message: `Choose the preference value for "${preferenceTopic.slug}" value "${value}" associated with purpose "${purposeMapping.purpose}"`, - type: "list", + type: 'list', choices: preferenceOptions, default: preferenceOptions.find((x) => x === value), }, @@ -177,10 +177,10 @@ export async function parsePreferenceAndPurposeValuesFromCsv( preferenceValue: boolean; }>([ { - name: "preferenceValue", + name: 'preferenceValue', message: `Choose the preference value for "${preferenceTopic.slug}" value "${parsedValue}" associated with purpose "${purposeMapping.purpose}"`, - type: "list", + type: 'list', choices: preferenceOptions, default: preferenceOptions.find((x) => x === parsedValue), }, @@ -191,7 +191,7 @@ export async function parsePreferenceAndPurposeValuesFromCsv( } throw new Error( - `Unknown preference topic type: ${preferenceTopic.type}` + `Unknown preference topic type: ${preferenceTopic.type}`, ); } }); diff --git a/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts b/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts index 3487eedc..df9357ce 100644 --- a/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts @@ -1,9 +1,9 @@ -import colors from "colors"; -import inquirer from "inquirer"; -import { difference, groupBy, uniq } from "lodash-es"; -import { logger } from "../../logger"; -import { inquirerConfirmBoolean } from "../helpers"; -import { FileMetadataState } from "./codecs"; +import colors from 'colors'; +import inquirer from 'inquirer'; +import { difference, groupBy, uniq } from 'lodash-es'; +import { logger } from '../../logger'; +import { inquirerConfirmBoolean } from '../helpers'; +import { FileMetadataState } from './codecs'; /** * Parse identifiers from a CSV list of preferences @@ -17,7 +17,7 @@ import { FileMetadataState } from "./codecs"; */ export async function parsePreferenceIdentifiersFromCsv( preferences: Record[], - currentState: FileMetadataState + currentState: FileMetadataState, ): Promise<{ /** The updated state */ currentState: FileMetadataState; @@ -40,13 +40,13 @@ export async function parsePreferenceIdentifiersFromCsv( identifierName: string; }>([ { - name: "identifierName", + name: 'identifierName', message: - "Choose the column that will be used as the identifier to upload consent preferences by", - type: "list", + 'Choose the column that will be used as the identifier to upload consent preferences by', + type: 'list', default: remainingColumnsForIdentifier.find((col) => - col.toLowerCase().includes("email") + col.toLowerCase().includes('email'), ) || remainingColumnsForIdentifier[0], choices: remainingColumnsForIdentifier, }, @@ -54,7 +54,9 @@ export async function parsePreferenceIdentifiersFromCsv( currentState.identifierColumn = identifierName; } logger.info( - colors.magenta(`Using identifier column "${currentState.identifierColumn}"`) + colors.magenta( + `Using identifier column "${currentState.identifierColumn}"`, + ), ); // Validate that the identifier column is present for all rows and unique @@ -66,13 +68,13 @@ export async function parsePreferenceIdentifiersFromCsv( const message = `The identifier column "${ currentState.identifierColumn }" is missing a value for the following rows: ${identifierColumnsMissing.join( - ", " + ', ', )}`; logger.warn(colors.yellow(message)); // Ask user if they would like to skip rows missing an identifier const skip = await inquirerConfirmBoolean({ - message: "Would you like to skip rows missing an identifier?", + message: 'Would you like to skip rows missing an identifier?', }); if (!skip) { throw new Error(message); @@ -81,24 +83,24 @@ export async function parsePreferenceIdentifiersFromCsv( // Filter out rows missing an identifier const previous = preferences.length; preferences = preferences.filter( - (pref) => pref[currentState.identifierColumn!] + (pref) => pref[currentState.identifierColumn!], ); logger.info( colors.yellow( - `Skipped ${previous - preferences.length} rows missing an identifier` - ) + `Skipped ${previous - preferences.length} rows missing an identifier`, + ), ); } logger.info( colors.magenta( - `The identifier column "${currentState.identifierColumn}" is present for all rows` - ) + `The identifier column "${currentState.identifierColumn}" is present for all rows`, + ), ); // Validate that all identifiers are unique const rowsByUserId = groupBy(preferences, currentState.identifierColumn); const duplicateIdentifiers = Object.entries(rowsByUserId).filter( - ([, rows]) => rows.length > 1 + ([, rows]) => rows.length > 1, ); if (duplicateIdentifiers.length > 0) { const message = `The identifier column "${ @@ -106,13 +108,13 @@ export async function parsePreferenceIdentifiersFromCsv( }" has duplicate values for the following rows: ${duplicateIdentifiers .slice(0, 10) .map(([userId, rows]) => `${userId} (${rows.length})`) - .join("\n")}`; + .join('\n')}`; logger.warn(colors.yellow(message)); // Ask user if they would like to take the most recent update // for each duplicate identifier const skip = await inquirerConfirmBoolean({ - message: "Would you like to automatically take the latest update?", + message: 'Would you like to automatically take the latest update?', }); if (!skip) { throw new Error(message); @@ -122,7 +124,7 @@ export async function parsePreferenceIdentifiersFromCsv( const sorted = rows.sort( (a, b) => new Date(b[currentState.timestampColum!]).getTime() - - new Date(a[currentState.timestampColum!]).getTime() + new Date(a[currentState.timestampColum!]).getTime(), ); return sorted[0]; }) diff --git a/src/lib/preference-management/parsePreferenceManagementCsv.ts b/src/lib/preference-management/parsePreferenceManagementCsv.ts index 13415db4..c0d629f5 100644 --- a/src/lib/preference-management/parsePreferenceManagementCsv.ts +++ b/src/lib/preference-management/parsePreferenceManagementCsv.ts @@ -1,19 +1,19 @@ -import { PersistedState } from "@transcend-io/persisted-state"; -import colors from "colors"; -import type { Got } from "got"; -import * as t from "io-ts"; -import { keyBy } from "lodash-es"; -import { logger } from "../../logger"; -import { PreferenceTopic } from "../graphql"; -import { readCsv } from "../requests"; -import { checkIfPendingPreferenceUpdatesAreNoOp } from "./checkIfPendingPreferenceUpdatesAreNoOp"; -import { checkIfPendingPreferenceUpdatesCauseConflict } from "./checkIfPendingPreferenceUpdatesCauseConflict"; -import { FileMetadataState, PreferenceState } from "./codecs"; -import { getPreferencesForIdentifiers } from "./getPreferencesForIdentifiers"; -import { getPreferenceUpdatesFromRow } from "./getPreferenceUpdatesFromRow"; -import { parsePreferenceAndPurposeValuesFromCsv } from "./parsePreferenceAndPurposeValuesFromCsv"; -import { parsePreferenceIdentifiersFromCsv } from "./parsePreferenceIdentifiersFromCsv"; -import { parsePreferenceTimestampsFromCsv } from "./parsePreferenceTimestampsFromCsv"; +import { PersistedState } from '@transcend-io/persisted-state'; +import colors from 'colors'; +import type { Got } from 'got'; +import * as t from 'io-ts'; +import { keyBy } from 'lodash-es'; +import { logger } from '../../logger'; +import { PreferenceTopic } from '../graphql'; +import { readCsv } from '../requests'; +import { checkIfPendingPreferenceUpdatesAreNoOp } from './checkIfPendingPreferenceUpdatesAreNoOp'; +import { checkIfPendingPreferenceUpdatesCauseConflict } from './checkIfPendingPreferenceUpdatesCauseConflict'; +import { FileMetadataState, PreferenceState } from './codecs'; +import { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers'; +import { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow'; +import { parsePreferenceAndPurposeValuesFromCsv } from './parsePreferenceAndPurposeValuesFromCsv'; +import { parsePreferenceIdentifiersFromCsv } from './parsePreferenceIdentifiersFromCsv'; +import { parsePreferenceTimestampsFromCsv } from './parsePreferenceTimestampsFromCsv'; /** * Parse a file into the cache @@ -48,13 +48,13 @@ export async function parsePreferenceManagementCsvWithCache( /** Wheather to force workflow triggers */ forceTriggerWorkflows: boolean; }, - cache: PersistedState + cache: PersistedState, ): Promise { // Start the timer const t0 = Date.now(); // Get the current metadata - const fileMetadata = cache.getValue("fileMetadata"); + const fileMetadata = cache.getValue('fileMetadata'); // Read in the file logger.info(colors.magenta(`Reading in file: "${file}"`)); @@ -74,20 +74,20 @@ export async function parsePreferenceManagementCsvWithCache( // Validate that all timestamps are present in the file currentState = await parsePreferenceTimestampsFromCsv( preferences, - currentState + currentState, ); fileMetadata[file] = currentState; - await cache.setValue(fileMetadata, "fileMetadata"); + await cache.setValue(fileMetadata, 'fileMetadata'); // Validate that all identifiers are present and unique const result = await parsePreferenceIdentifiersFromCsv( preferences, - currentState + currentState, ); currentState = result.currentState; preferences = result.preferences; fileMetadata[file] = currentState; - await cache.setValue(fileMetadata, "fileMetadata"); + await cache.setValue(fileMetadata, 'fileMetadata'); // Ensure all other columns are mapped to purpose and preference // slug values @@ -98,14 +98,14 @@ export async function parsePreferenceManagementCsvWithCache( preferenceTopics, purposeSlugs, forceTriggerWorkflows, - } + }, ); fileMetadata[file] = currentState; - await cache.setValue(fileMetadata, "fileMetadata"); + await cache.setValue(fileMetadata, 'fileMetadata'); // Grab existing preference store records const identifiers = preferences.map( - (pref) => pref[currentState.identifierColumn!] + (pref) => pref[currentState.identifierColumn!], ); const existingConsentRecords = skipExistingRecordCheck ? [] @@ -113,7 +113,7 @@ export async function parsePreferenceManagementCsvWithCache( identifiers: identifiers.map((x) => ({ value: x })), partitionKey, }); - const consentRecordByIdentifier = keyBy(existingConsentRecords, "userId"); + const consentRecordByIdentifier = keyBy(existingConsentRecords, 'userId'); // Clear out previous updates currentState.pendingConflictUpdates = {}; @@ -138,7 +138,7 @@ export async function parsePreferenceManagementCsvWithCache( if (forceTriggerWorkflows && !currentConsentRecord) { throw new Error( `No existing consent record found for user with id: ${userId}. - When 'forceTriggerWorkflows' is set all the user identifiers should contain a consent record` + When 'forceTriggerWorkflows' is set all the user identifiers should contain a consent record`, ); } // Check if the update can be skipped @@ -179,11 +179,11 @@ export async function parsePreferenceManagementCsvWithCache( // Read in the file fileMetadata[file] = currentState; - await cache.setValue(fileMetadata, "fileMetadata"); + await cache.setValue(fileMetadata, 'fileMetadata'); const t1 = Date.now(); logger.info( colors.green( - `Successfully pre-processed file: "${file}" in ${(t1 - t0) / 1000}s` - ) + `Successfully pre-processed file: "${file}" in ${(t1 - t0) / 1000}s`, + ), ); } diff --git a/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts b/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts index e59ca765..09b2fb48 100644 --- a/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts +++ b/src/lib/preference-management/parsePreferenceTimestampsFromCsv.ts @@ -1,10 +1,10 @@ -import colors from "colors"; -import inquirer from "inquirer"; -import { difference, uniq } from "lodash-es"; -import { logger } from "../../logger"; -import { FileMetadataState } from "./codecs"; +import colors from 'colors'; +import inquirer from 'inquirer'; +import { difference, uniq } from 'lodash-es'; +import { logger } from '../../logger'; +import { FileMetadataState } from './codecs'; -export const NONE_PREFERENCE_MAP = "[NONE]"; +export const NONE_PREFERENCE_MAP = '[NONE]'; /** * Parse timestamps from a CSV list of preferences @@ -20,7 +20,7 @@ export const NONE_PREFERENCE_MAP = "[NONE]"; */ export async function parsePreferenceTimestampsFromCsv( preferences: Record[], - currentState: FileMetadataState + currentState: FileMetadataState, ): Promise { // Determine columns to map const columnNames = uniq(preferences.flatMap((x) => Object.keys(x))); @@ -38,16 +38,16 @@ export async function parsePreferenceTimestampsFromCsv( timestampName: string; }>([ { - name: "timestampName", + name: 'timestampName', message: - "Choose the column that will be used as the timestamp of last preference update", - type: "list", + 'Choose the column that will be used as the timestamp of last preference update', + type: 'list', default: remainingColumnsForTimestamp.find((col) => - col.toLowerCase().includes("date") + col.toLowerCase().includes('date'), ) || remainingColumnsForTimestamp.find((col) => - col.toLowerCase().includes("time") + col.toLowerCase().includes('time'), ) || remainingColumnsForTimestamp[0], choices: [...remainingColumnsForTimestamp, NONE_PREFERENCE_MAP], @@ -56,7 +56,7 @@ export async function parsePreferenceTimestampsFromCsv( currentState.timestampColum = timestampName; } logger.info( - colors.magenta(`Using timestamp column "${currentState.timestampColum}"`) + colors.magenta(`Using timestamp column "${currentState.timestampColum}"`), ); // Validate that all rows have valid timestamp @@ -70,14 +70,14 @@ export async function parsePreferenceTimestampsFromCsv( `The timestamp column "${ currentState.timestampColum }" is missing a value for the following rows: ${timestampColumnsMissing.join( - "\n" - )}` + '\n', + )}`, ); } logger.info( colors.magenta( - `The timestamp column "${currentState.timestampColum}" is present for all row` - ) + `The timestamp column "${currentState.timestampColum}" is present for all row`, + ), ); } return currentState; diff --git a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts index baa9f464..7d6bec73 100644 --- a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts +++ b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesAreNoOp.test.ts @@ -1,15 +1,15 @@ -import { PreferenceTopicType } from "@transcend-io/privacy-types"; -import { describe, expect, it } from "vitest"; -import { PreferenceTopic } from "../../graphql"; -import { checkIfPendingPreferenceUpdatesAreNoOp } from "../index"; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import { describe, expect, it } from 'vitest'; +import { PreferenceTopic } from '../../graphql'; +import { checkIfPendingPreferenceUpdatesAreNoOp } from '../index'; const DEFAULT_VALUES = { - userId: "test@transcend.io", - timestamp: "2024-11-30T00:00:15.327Z", - partition: "d9c0b9ca-2253-4418-89d2-88776d654223", + userId: 'test@transcend.io', + timestamp: '2024-11-30T00:00:15.327Z', + partition: 'd9c0b9ca-2253-4418-89d2-88776d654223', system: { - decryptionStatus: "DECRYPTED" as const, - updatedAt: "2024-11-30T00:00:16.506Z", + decryptionStatus: 'DECRYPTED' as const, + updatedAt: '2024-11-30T00:00:16.506Z', }, consentManagement: { usp: null, @@ -22,122 +22,122 @@ const DEFAULT_VALUES = { const PREFERENCE_TOPICS: PreferenceTopic[] = [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference1", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, title: { - defaultMessage: "Boolean Preference 1", - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3", + defaultMessage: 'Boolean Preference 1', + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3', }, displayDescription: { - defaultMessage: "This is a boolean preference for testing.", - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", + defaultMessage: 'This is a boolean preference for testing.', + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', }, - defaultConfiguration: "", + defaultConfiguration: '', showInPrivacyCenter: true, }, { - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference2", + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, title: { - defaultMessage: "Boolean Preference 2", - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", + defaultMessage: 'Boolean Preference 2', + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', }, displayDescription: { - defaultMessage: "This is another boolean preference for testing.", - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b5", + defaultMessage: 'This is another boolean preference for testing.', + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b5', }, - defaultConfiguration: "", + defaultConfiguration: '', showInPrivacyCenter: true, }, { - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "MultiSelectPreference", + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', type: PreferenceTopicType.MultiSelect, title: { - defaultMessage: "Multi Select Preference", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b0", + defaultMessage: 'Multi Select Preference', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b0', }, displayDescription: { - defaultMessage: "This is a multi-select preference for testing.", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", + defaultMessage: 'This is a multi-select preference for testing.', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', }, - defaultConfiguration: "", + defaultConfiguration: '', showInPrivacyCenter: true, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Value 1", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", + defaultMessage: 'Value 1', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Value 2", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2", + defaultMessage: 'Value 2', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2', }, }, ], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, { - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "SingleSelectPreference", + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', type: PreferenceTopicType.Select, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Value 1", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", + defaultMessage: 'Value 1', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Value 2", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2", + defaultMessage: 'Value 2', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2', }, }, ], title: { - defaultMessage: "Single Select Preference", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b0", + defaultMessage: 'Single Select Preference', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b0', }, displayDescription: { - defaultMessage: "This is a single-select preference for testing.", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", + defaultMessage: 'This is a single-select preference for testing.', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', }, - defaultConfiguration: "", + defaultConfiguration: '', showInPrivacyCenter: true, purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ]; -describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { - it("should return true for simple purpose comparison", () => { +describe('checkIfPendingPreferenceUpdatesAreNoOp', () => { + it('should return true for simple purpose comparison', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, preferences: [], }, @@ -149,18 +149,18 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); - it("should return false for simple purpose comparison", () => { + it('should return false for simple purpose comparison', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: true, preferences: [], }, @@ -172,22 +172,22 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return true for simple purpose comparison with extra preference", () => { + it('should return true for simple purpose comparison with extra preference', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, @@ -202,18 +202,18 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); - it("should return false for simple purpose comparison with extra preference in update", () => { + it('should return false for simple purpose comparison with extra preference in update', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, }, ], @@ -223,7 +223,7 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, @@ -232,36 +232,36 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return true for preferences being same", () => { + it('should return true for preferences being same', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -273,57 +273,57 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); - it("should return false for boolean preference changing", () => { + it('should return false for boolean preference changing', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: false, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -335,57 +335,57 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return false for single select preference changing", () => { + it('should return false for single select preference changing', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value2", + selectValue: 'Value2', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -397,57 +397,57 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return false for multi select preference changing", () => { + it('should return false for multi select preference changing', () => { expect( checkIfPendingPreferenceUpdatesAreNoOp({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value2"], + selectValues: ['Value2'], }, }, ], @@ -459,28 +459,28 @@ describe("checkIfPendingPreferenceUpdatesAreNoOp", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); }); diff --git a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts index 11fffaf9..f2735fbf 100644 --- a/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts +++ b/src/lib/preference-management/tests/checkIfPendingPreferenceUpdatesCauseConflict.test.ts @@ -1,15 +1,15 @@ -import { PreferenceTopicType } from "@transcend-io/privacy-types"; -import { describe, expect, it } from "vitest"; -import { PreferenceTopic } from "../../graphql"; -import { checkIfPendingPreferenceUpdatesCauseConflict } from "../index"; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import { describe, expect, it } from 'vitest'; +import { PreferenceTopic } from '../../graphql'; +import { checkIfPendingPreferenceUpdatesCauseConflict } from '../index'; const DEFAULT_VALUES = { - userId: "test@transcend.io", - timestamp: "2024-11-30T00:00:15.327Z", - partition: "d9c0b9ca-2253-4418-89d2-88776d654223", + userId: 'test@transcend.io', + timestamp: '2024-11-30T00:00:15.327Z', + partition: 'd9c0b9ca-2253-4418-89d2-88776d654223', system: { - decryptionStatus: "DECRYPTED" as const, - updatedAt: "2024-11-30T00:00:16.506Z", + decryptionStatus: 'DECRYPTED' as const, + updatedAt: '2024-11-30T00:00:16.506Z', }, consentManagement: { usp: null, @@ -22,125 +22,125 @@ const DEFAULT_VALUES = { const PREFERENCE_TOPICS: PreferenceTopic[] = [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference1", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, title: { - defaultMessage: "Boolean Preference 1", - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3", + defaultMessage: 'Boolean Preference 1', + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3', }, displayDescription: { - defaultMessage: "This is a boolean preference for testing purposes.", - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", + defaultMessage: 'This is a boolean preference for testing purposes.', + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', }, { - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference2", + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, title: { - defaultMessage: "Boolean Preference 2", - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", + defaultMessage: 'Boolean Preference 2', + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', }, displayDescription: { defaultMessage: - "This is another boolean preference for testing purposes.", - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b5", + 'This is another boolean preference for testing purposes.', + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b5', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', }, { - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "MultiSelectPreference", + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', type: PreferenceTopicType.MultiSelect, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Value 1", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", + defaultMessage: 'Value 1', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Value 2", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2", + defaultMessage: 'Value 2', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2', }, }, ], title: { - defaultMessage: "Multi Select Preference", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3", + defaultMessage: 'Multi Select Preference', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3', }, displayDescription: { - defaultMessage: "This is a multi-select preference for testing purposes.", - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", + defaultMessage: 'This is a multi-select preference for testing purposes.', + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, { - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "SingleSelectPreference", + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', type: PreferenceTopicType.Select, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Value 1", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1", + defaultMessage: 'Value 1', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b1', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Value 2", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2", + defaultMessage: 'Value 2', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b2', }, }, ], title: { - defaultMessage: "Single Select Preference", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3", + defaultMessage: 'Single Select Preference', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3', }, displayDescription: { defaultMessage: - "This is a single-select preference for testing purposes.", - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4", + 'This is a single-select preference for testing purposes.', + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b4', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', // This preference is associated with the Marketing purpose purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ]; -describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { - it("should return false for simple purpose comparison", () => { +describe('checkIfPendingPreferenceUpdatesCauseConflict', () => { + it('should return false for simple purpose comparison', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, preferences: [], }, @@ -152,18 +152,18 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return false if purpose missing", () => { + it('should return false if purpose missing', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, preferences: [], }, @@ -175,18 +175,18 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return true for simple purpose comparison", () => { + it('should return true for simple purpose comparison', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: true, preferences: [], }, @@ -198,22 +198,22 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); - it("should return true for simple purpose comparison with extra preference", () => { + it('should return true for simple purpose comparison with extra preference', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, @@ -228,18 +228,18 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return false for simple purpose comparison with extra preference in update", () => { + it('should return false for simple purpose comparison with extra preference in update', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "SalesOutreach", + purpose: 'SalesOutreach', enabled: false, }, ], @@ -249,7 +249,7 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, @@ -258,36 +258,36 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return false for preferences being same", () => { + it('should return false for preferences being same', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -299,57 +299,57 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(false); }); - it("should return true for boolean preference changing", () => { + it('should return true for boolean preference changing', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: false, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -361,57 +361,57 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); - it("should return true for single select preference changing", () => { + it('should return true for single select preference changing', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value2", + selectValue: 'Value2', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -423,57 +423,57 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); - it("should return true for multi select preference changing", () => { + it('should return true for multi select preference changing', () => { expect( checkIfPendingPreferenceUpdatesCauseConflict({ currentConsentRecord: { ...DEFAULT_VALUES, purposes: [ { - purpose: "Marketing", + purpose: 'Marketing', enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value2"], + selectValues: ['Value2'], }, }, ], @@ -485,28 +485,28 @@ describe("checkIfPendingPreferenceUpdatesCauseConflict", () => { enabled: false, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], }, }, preferenceTopics: PREFERENCE_TOPICS, - }) + }), ).to.equal(true); }); }); diff --git a/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts b/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts index 587d3353..1c4a3c90 100644 --- a/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts +++ b/src/lib/preference-management/tests/getPreferenceUpdatesFromRow.test.ts @@ -1,60 +1,60 @@ -import { PreferenceTopicType } from "@transcend-io/privacy-types"; -import { describe, expect, it } from "vitest"; -import { getPreferenceUpdatesFromRow } from "../index"; +import { PreferenceTopicType } from '@transcend-io/privacy-types'; +import { describe, expect, it } from 'vitest'; +import { getPreferenceUpdatesFromRow } from '../index'; -describe("getPreferenceUpdatesFromRow", () => { - it("should parse boolean updates", () => { +describe('getPreferenceUpdatesFromRow', () => { + it('should parse boolean updates', () => { expect( getPreferenceUpdatesFromRow({ row: { - my_purpose: "true", - has_topic_1: "true", - has_topic_2: "false", + my_purpose: 'true', + has_topic_1: 'true', + has_topic_2: 'false', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference1", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], title: { - defaultMessage: "Marketing Preferences", - id: "12345678-1234-1234-1234-123456789012", + defaultMessage: 'Marketing Preferences', + id: '12345678-1234-1234-1234-123456789012', }, displayDescription: { - defaultMessage: "Enable marketing tracking", - id: "12345678-1234-1234-1234-123456789013", + defaultMessage: 'Enable marketing tracking', + id: '12345678-1234-1234-1234-123456789013', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, { - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference2", + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', title: { - defaultMessage: "Advertising Preferences", - id: "12345678-1234-1234-1234-123456789014", + defaultMessage: 'Advertising Preferences', + id: '12345678-1234-1234-1234-123456789014', }, displayDescription: { - defaultMessage: "Enable advertising tracking", - id: "12345678-1234-1234-1234-123456789015", + defaultMessage: 'Enable advertising tracking', + id: '12345678-1234-1234-1234-123456789015', }, }, ], columnToPurposeName: { my_purpose: { - purpose: "Marketing", + purpose: 'Marketing', preference: null, valueMapping: { true: true, @@ -62,35 +62,35 @@ describe("getPreferenceUpdatesFromRow", () => { }, }, has_topic_1: { - purpose: "Marketing", - preference: "BooleanPreference1", + purpose: 'Marketing', + preference: 'BooleanPreference1', valueMapping: { true: true, false: false, }, }, has_topic_2: { - purpose: "Marketing", - preference: "BooleanPreference2", + purpose: 'Marketing', + preference: 'BooleanPreference2', valueMapping: { true: true, false: false, }, }, }, - }) + }), ).to.deep.equal({ Marketing: { enabled: true, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "BooleanPreference2", + topic: 'BooleanPreference2', choice: { booleanValue: false, }, @@ -100,53 +100,53 @@ describe("getPreferenceUpdatesFromRow", () => { }); }); - it("should parse a single select", () => { + it('should parse a single select', () => { expect( getPreferenceUpdatesFromRow({ row: { - my_purpose: "true", - has_topic_3: "Option 1", + my_purpose: 'true', + has_topic_3: 'Option 1', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "SingleSelectPreference", - defaultConfiguration: "", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', + defaultConfiguration: '', title: { - defaultMessage: "Single Select Preference", - id: "12345678-1234-1234-1234-123456789010", + defaultMessage: 'Single Select Preference', + id: '12345678-1234-1234-1234-123456789010', }, displayDescription: { - defaultMessage: "Choose one option", - id: "12345678-1234-1234-1234-123456789011", + defaultMessage: 'Choose one option', + id: '12345678-1234-1234-1234-123456789011', }, showInPrivacyCenter: true, type: PreferenceTopicType.Select, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Option 1", - id: "12345678-1234-1234-1234-123456789016", + defaultMessage: 'Option 1', + id: '12345678-1234-1234-1234-123456789016', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Option 2", - id: "12345678-1234-1234-1234-123456789017", + defaultMessage: 'Option 2', + id: '12345678-1234-1234-1234-123456789017', }, }, ], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ], columnToPurposeName: { my_purpose: { - purpose: "Marketing", + purpose: 'Marketing', preference: null, valueMapping: { true: true, @@ -154,23 +154,23 @@ describe("getPreferenceUpdatesFromRow", () => { }, }, has_topic_3: { - purpose: "Marketing", - preference: "SingleSelectPreference", + purpose: 'Marketing', + preference: 'SingleSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, }, - }) + }), ).to.deep.equal({ Marketing: { enabled: true, preferences: [ { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, ], @@ -178,54 +178,54 @@ describe("getPreferenceUpdatesFromRow", () => { }); }); - it("should parse a multi select example", () => { + it('should parse a multi select example', () => { expect( getPreferenceUpdatesFromRow({ row: { - my_purpose: "true", - has_topic_4: "Option 2,Option 1", - has_topic_5: "Option 1", + my_purpose: 'true', + has_topic_4: 'Option 2,Option 1', + has_topic_5: 'Option 1', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "MultiSelectPreference", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', type: PreferenceTopicType.MultiSelect, - defaultConfiguration: "", + defaultConfiguration: '', title: { - defaultMessage: "Multi Select Preference", - id: "12345678-1234-1234-1234-123456789020", + defaultMessage: 'Multi Select Preference', + id: '12345678-1234-1234-1234-123456789020', }, displayDescription: { - defaultMessage: "Choose multiple options", - id: "12345678-1234-1234-1234-123456789021", + defaultMessage: 'Choose multiple options', + id: '12345678-1234-1234-1234-123456789021', }, showInPrivacyCenter: true, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Option 1", - id: "12345678-1234-1234-1234-123456789018", + defaultMessage: 'Option 1', + id: '12345678-1234-1234-1234-123456789018', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Option 2", - id: "12345678-1234-1234-1234-123456789019", + defaultMessage: 'Option 2', + id: '12345678-1234-1234-1234-123456789019', }, }, ], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ], columnToPurposeName: { my_purpose: { - purpose: "Marketing", + purpose: 'Marketing', preference: null, valueMapping: { true: true, @@ -233,37 +233,37 @@ describe("getPreferenceUpdatesFromRow", () => { }, }, has_topic_4: { - purpose: "Marketing", - preference: "MultiSelectPreference", + purpose: 'Marketing', + preference: 'MultiSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, has_topic_5: { - purpose: "Marketing", - preference: "MultiSelectPreference", + purpose: 'Marketing', + preference: 'MultiSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, }, - }) + }), ).to.deep.equal({ Marketing: { enabled: true, preferences: [ { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1"], + selectValues: ['Value1'], }, }, ], @@ -271,128 +271,128 @@ describe("getPreferenceUpdatesFromRow", () => { }); }); - it("should parse boolean, single select, multi select example", () => { + it('should parse boolean, single select, multi select example', () => { expect( getPreferenceUpdatesFromRow({ row: { - my_purpose: "true", - has_topic_1: "true", - has_topic_2: "false", - has_topic_3: "Option 1", - has_topic_4: "Option 2,Option 1", + my_purpose: 'true', + has_topic_1: 'true', + has_topic_2: 'false', + has_topic_3: 'Option 1', + has_topic_4: 'Option 2,Option 1', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference1", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], title: { - defaultMessage: "Boolean Preference 1", - id: "12345678-1234-1234-1234-123456789022", + defaultMessage: 'Boolean Preference 1', + id: '12345678-1234-1234-1234-123456789022', }, displayDescription: { - defaultMessage: "Enable this preference", - id: "12345678-1234-1234-1234-123456789023", + defaultMessage: 'Enable this preference', + id: '12345678-1234-1234-1234-123456789023', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, { - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference2", + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], title: { - defaultMessage: "Boolean Preference 2", - id: "12345678-1234-1234-1234-123456789024", + defaultMessage: 'Boolean Preference 2', + id: '12345678-1234-1234-1234-123456789024', }, displayDescription: { - defaultMessage: "Disable this preference", - id: "12345678-1234-1234-1234-123456789025", + defaultMessage: 'Disable this preference', + id: '12345678-1234-1234-1234-123456789025', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, { - id: "34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "MultiSelectPreference", + id: '34b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', type: PreferenceTopicType.MultiSelect, - defaultConfiguration: "", + defaultConfiguration: '', title: { - defaultMessage: "Multi Select Preference", - id: "12345678-1234-1234-1234-123456789028", + defaultMessage: 'Multi Select Preference', + id: '12345678-1234-1234-1234-123456789028', }, displayDescription: { - defaultMessage: "Choose multiple options", - id: "12345678-1234-1234-1234-123456789029", + defaultMessage: 'Choose multiple options', + id: '12345678-1234-1234-1234-123456789029', }, showInPrivacyCenter: true, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Option 1", - id: "12345678-1234-1234-1234-123456789026", + defaultMessage: 'Option 1', + id: '12345678-1234-1234-1234-123456789026', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Option 2", - id: "12345678-1234-1234-1234-123456789027", + defaultMessage: 'Option 2', + id: '12345678-1234-1234-1234-123456789027', }, }, ], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, { - id: "44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "SingleSelectPreference", + id: '44b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', type: PreferenceTopicType.Select, - defaultConfiguration: "", + defaultConfiguration: '', title: { - defaultMessage: "Single Select Preference", - id: "12345678-1234-1234-1234-123456789030", + defaultMessage: 'Single Select Preference', + id: '12345678-1234-1234-1234-123456789030', }, displayDescription: { - defaultMessage: "Choose one option", - id: "12345678-1234-1234-1234-123456789031", + defaultMessage: 'Choose one option', + id: '12345678-1234-1234-1234-123456789031', }, showInPrivacyCenter: true, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Option 1", - id: "12345678-1234-1234-1234-123456789030", + defaultMessage: 'Option 1', + id: '12345678-1234-1234-1234-123456789030', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Option 2", - id: "12345678-1234-1234-1234-123456789031", + defaultMessage: 'Option 2', + id: '12345678-1234-1234-1234-123456789031', }, }, ], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ], columnToPurposeName: { my_purpose: { - purpose: "Marketing", + purpose: 'Marketing', preference: null, valueMapping: { true: true, @@ -400,65 +400,65 @@ describe("getPreferenceUpdatesFromRow", () => { }, }, has_topic_1: { - purpose: "Marketing", - preference: "BooleanPreference1", + purpose: 'Marketing', + preference: 'BooleanPreference1', valueMapping: { true: true, false: false, }, }, has_topic_2: { - purpose: "Marketing", - preference: "BooleanPreference2", + purpose: 'Marketing', + preference: 'BooleanPreference2', valueMapping: { true: true, false: false, }, }, has_topic_3: { - purpose: "Marketing", - preference: "SingleSelectPreference", + purpose: 'Marketing', + preference: 'SingleSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, has_topic_4: { - purpose: "Marketing", - preference: "MultiSelectPreference", + purpose: 'Marketing', + preference: 'MultiSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, }, - }) + }), ).to.deep.equal({ Marketing: { enabled: true, preferences: [ { - topic: "BooleanPreference1", + topic: 'BooleanPreference1', choice: { booleanValue: true, }, }, { - topic: "BooleanPreference2", + topic: 'BooleanPreference2', choice: { booleanValue: false, }, }, { - topic: "SingleSelectPreference", + topic: 'SingleSelectPreference', choice: { - selectValue: "Value1", + selectValue: 'Value1', }, }, { - topic: "MultiSelectPreference", + topic: 'MultiSelectPreference', choice: { - selectValues: ["Value1", "Value2"], + selectValues: ['Value1', 'Value2'], }, }, ], @@ -466,57 +466,57 @@ describe("getPreferenceUpdatesFromRow", () => { }); }); - it("should error if missing purpose", () => { + it('should error if missing purpose', () => { try { getPreferenceUpdatesFromRow({ row: { - has_topic_1: "true", + has_topic_1: 'true', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference1", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, displayDescription: { - defaultMessage: "Enable this preference", - id: "12345678-1234-1234-1234-123456789032", + defaultMessage: 'Enable this preference', + id: '12345678-1234-1234-1234-123456789032', }, title: { - defaultMessage: "Boolean Preference 1", - id: "12345678-1234-1234-1234-123456789033", + defaultMessage: 'Boolean Preference 1', + id: '12345678-1234-1234-1234-123456789033', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', }, { - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference2", + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, displayDescription: { - defaultMessage: "Disable this preference", - id: "12345678-1234-1234-1234-123456789034", + defaultMessage: 'Disable this preference', + id: '12345678-1234-1234-1234-123456789034', }, title: { - defaultMessage: "Boolean Preference 2", - id: "12345678-1234-1234-1234-123456789035", + defaultMessage: 'Boolean Preference 2', + id: '12345678-1234-1234-1234-123456789035', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', }, ], columnToPurposeName: { has_topic_1: { - purpose: "Marketing", - preference: "BooleanPreference1", + purpose: 'Marketing', + preference: 'BooleanPreference1', valueMapping: { true: true, false: false, @@ -524,63 +524,63 @@ describe("getPreferenceUpdatesFromRow", () => { }, }, }); - expect.fail("Should have thrown"); + expect.fail('Should have thrown'); } catch (error) { - expect(error.message).to.include("No mapping provided"); + expect(error.message).to.include('No mapping provided'); } }); - it("should error if purpose name is not valid", () => { + it('should error if purpose name is not valid', () => { try { getPreferenceUpdatesFromRow({ row: { - has_topic_1: "true", + has_topic_1: 'true', }, - purposeSlugs: ["Marketing", "Advertising"], + purposeSlugs: ['Marketing', 'Advertising'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference1", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference1', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, displayDescription: { - defaultMessage: "Enable this preference", - id: "12345678-1234-1234-1234-123456789036", + defaultMessage: 'Enable this preference', + id: '12345678-1234-1234-1234-123456789036', }, title: { - defaultMessage: "Boolean Preference 1", - id: "12345678-1234-1234-1234-123456789037", + defaultMessage: 'Boolean Preference 1', + id: '12345678-1234-1234-1234-123456789037', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', }, { - id: "24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "BooleanPreference2", + id: '24b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'BooleanPreference2', type: PreferenceTopicType.Boolean, preferenceOptionValues: [], purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, displayDescription: { - defaultMessage: "Disable this preference", - id: "12345678-1234-1234-1234-123456789038", + defaultMessage: 'Disable this preference', + id: '12345678-1234-1234-1234-123456789038', }, title: { - defaultMessage: "Boolean Preference 2", - id: "12345678-1234-1234-1234-123456789039", + defaultMessage: 'Boolean Preference 2', + id: '12345678-1234-1234-1234-123456789039', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', }, ], columnToPurposeName: { has_topic_1: { - purpose: "InvalidPurpose", - preference: "BooleanPreference1", + purpose: 'InvalidPurpose', + preference: 'BooleanPreference1', valueMapping: { true: true, false: false, @@ -588,134 +588,134 @@ describe("getPreferenceUpdatesFromRow", () => { }, }, }); - expect.fail("Should have thrown"); + expect.fail('Should have thrown'); } catch (error) { expect(error.message).to.equal( - "Invalid purpose slug: InvalidPurpose, expected: Marketing, Advertising" + 'Invalid purpose slug: InvalidPurpose, expected: Marketing, Advertising', ); } }); - it("should error if single select option is invalid", () => { + it('should error if single select option is invalid', () => { try { getPreferenceUpdatesFromRow({ row: { - has_topic_1: "true", + has_topic_1: 'true', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "SingleSelectPreference", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'SingleSelectPreference', type: PreferenceTopicType.Select, preferenceOptionValues: [ { - slug: "Value1", + slug: 'Value1', title: { - defaultMessage: "Option 1", - id: "12345678-1234-1234-1234-123456789040", + defaultMessage: 'Option 1', + id: '12345678-1234-1234-1234-123456789040', }, }, { - slug: "Value2", + slug: 'Value2', title: { - defaultMessage: "Option 2", - id: "12345678-1234-1234-1234-123456789041", + defaultMessage: 'Option 2', + id: '12345678-1234-1234-1234-123456789041', }, }, ], title: { - defaultMessage: "Single Select Preference", - id: "12345678-1234-1234-1234-123456789042", + defaultMessage: 'Single Select Preference', + id: '12345678-1234-1234-1234-123456789042', }, displayDescription: { - defaultMessage: "Choose one option", - id: "12345678-1234-1234-1234-123456789043", + defaultMessage: 'Choose one option', + id: '12345678-1234-1234-1234-123456789043', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ], columnToPurposeName: { has_topic_1: { - purpose: "Marketing", - preference: "SingleSelectPreference", + purpose: 'Marketing', + preference: 'SingleSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, }, }); - expect.fail("Should have thrown"); + expect.fail('Should have thrown'); } catch (error) { expect(error.message).to.equal( - "Invalid value for select preference: SingleSelectPreference, expected string or null, got: true" + 'Invalid value for select preference: SingleSelectPreference, expected string or null, got: true', ); } }); - it("should error if multi select value is invalid", () => { + it('should error if multi select value is invalid', () => { try { getPreferenceUpdatesFromRow({ row: { - has_topic_1: "true", + has_topic_1: 'true', }, - purposeSlugs: ["Marketing"], + purposeSlugs: ['Marketing'], preferenceTopics: [ { - id: "14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b", - slug: "MultiSelectPreference", + id: '14b3b3b3-4b3b-4b3b-4b3b-4b3b3b3b3b3b', + slug: 'MultiSelectPreference', type: PreferenceTopicType.MultiSelect, preferenceOptionValues: [ { title: { - defaultMessage: "Option 1", - id: "12345678-1234-1234-1234-123456789044", + defaultMessage: 'Option 1', + id: '12345678-1234-1234-1234-123456789044', }, - slug: "Value1", + slug: 'Value1', }, { title: { - defaultMessage: "Option 2", - id: "12345678-1234-1234-1234-123456789045", + defaultMessage: 'Option 2', + id: '12345678-1234-1234-1234-123456789045', }, - slug: "Value2", + slug: 'Value2', }, ], title: { - defaultMessage: "Multi Select Preference", - id: "12345678-1234-1234-1234-123456789046", + defaultMessage: 'Multi Select Preference', + id: '12345678-1234-1234-1234-123456789046', }, displayDescription: { - defaultMessage: "Choose multiple options", - id: "12345678-1234-1234-1234-123456789047", + defaultMessage: 'Choose multiple options', + id: '12345678-1234-1234-1234-123456789047', }, showInPrivacyCenter: true, - defaultConfiguration: "", + defaultConfiguration: '', purpose: { - trackingType: "Marketing", + trackingType: 'Marketing', }, }, ], columnToPurposeName: { has_topic_1: { - purpose: "Marketing", - preference: "MultiSelectPreference", + purpose: 'Marketing', + preference: 'MultiSelectPreference', valueMapping: { - "Option 1": "Value1", - "Option 2": "Value2", + 'Option 1': 'Value1', + 'Option 2': 'Value2', }, }, }, }); - expect.fail("Should have thrown"); + expect.fail('Should have thrown'); } catch (error) { expect(error.message).to.equal( - "Invalid value for multi select preference: MultiSelectPreference, expected one of: Value1, Value2, got: true" + 'Invalid value for multi select preference: MultiSelectPreference, expected one of: Value1, Value2, got: true', ); } }); diff --git a/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts b/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts index 06da6e1f..0f592e5e 100644 --- a/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts +++ b/src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts @@ -1,12 +1,12 @@ -import { PersistedState } from "@transcend-io/persisted-state"; -import { PreferenceUpdateItem } from "@transcend-io/privacy-types"; -import { apply } from "@transcend-io/type-utils"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { chunk } from "lodash-es"; -import { DEFAULT_TRANSCEND_CONSENT_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { PersistedState } from '@transcend-io/persisted-state'; +import { PreferenceUpdateItem } from '@transcend-io/privacy-types'; +import { apply } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { chunk } from 'lodash-es'; +import { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, @@ -14,12 +14,12 @@ import { fetchAllPurposes, PreferenceTopic, Purpose, -} from "../graphql"; -import { parseAttributesFromString } from "../requests"; -import { PreferenceState } from "./codecs"; -import { getPreferenceUpdatesFromRow } from "./getPreferenceUpdatesFromRow"; -import { parsePreferenceManagementCsvWithCache } from "./parsePreferenceManagementCsv"; -import { NONE_PREFERENCE_MAP } from "./parsePreferenceTimestampsFromCsv"; +} from '../graphql'; +import { parseAttributesFromString } from '../requests'; +import { PreferenceState } from './codecs'; +import { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow'; +import { parsePreferenceManagementCsvWithCache } from './parsePreferenceManagementCsv'; +import { NONE_PREFERENCE_MAP } from './parsePreferenceTimestampsFromCsv'; /** * Upload a set of consent preferences @@ -80,13 +80,13 @@ export async function uploadPreferenceManagementPreferencesInteractive({ failingUpdates: {}, pendingUpdates: {}, }); - const failingRequests = preferenceState.getValue("failingUpdates"); - const pendingRequests = preferenceState.getValue("pendingUpdates"); - let fileMetadata = preferenceState.getValue("fileMetadata"); + const failingRequests = preferenceState.getValue('failingUpdates'); + const pendingRequests = preferenceState.getValue('pendingUpdates'); + let fileMetadata = preferenceState.getValue('fileMetadata'); logger.info( colors.magenta( - "Restored cache, there are: \n" + + 'Restored cache, there are: \n' + `${ Object.values(failingRequests).length } failing requests to be retried\n` + @@ -94,12 +94,12 @@ export async function uploadPreferenceManagementPreferencesInteractive({ Object.values(pendingRequests).length } pending requests to be processed\n` + `The following files are stored in cache and will be used:\n${Object.keys( - fileMetadata + fileMetadata, ) .map((x) => x) - .join("\n")}\n` + - `The following file will be processed: ${file}\n` - ) + .join('\n')}\n` + + `The following file will be processed: ${file}\n`, + ), ); // Create GraphQL client to connect to Transcend backend @@ -128,34 +128,34 @@ export async function uploadPreferenceManagementPreferencesInteractive({ skipExistingRecordCheck, forceTriggerWorkflows, }, - preferenceState + preferenceState, ); // Construct the pending updates const pendingUpdates: Record = {}; - fileMetadata = preferenceState.getValue("fileMetadata"); + fileMetadata = preferenceState.getValue('fileMetadata'); const metadata = fileMetadata[file]; logger.info( colors.magenta( `Found ${ Object.entries(metadata.pendingSafeUpdates).length - } safe updates in ${file}` - ) + } safe updates in ${file}`, + ), ); logger.info( colors.magenta( `Found ${ Object.entries(metadata.pendingConflictUpdates).length - } conflict updates in ${file}` - ) + } conflict updates in ${file}`, + ), ); logger.info( colors.magenta( `Found ${ Object.entries(metadata.skippedUpdates).length - } skipped updates in ${file}` - ) + } skipped updates in ${file}`, + ), ); // Update either safe updates only or safe + conflict @@ -193,8 +193,8 @@ export async function uploadPreferenceManagementPreferencesInteractive({ })), }; } - await preferenceState.setValue(pendingUpdates, "pendingUpdates"); - await preferenceState.setValue({}, "failingUpdates"); + await preferenceState.setValue(pendingUpdates, 'pendingUpdates'); + await preferenceState.setValue({}, 'failingUpdates'); // Exist early if dry run if (dryRun) { @@ -202,8 +202,8 @@ export async function uploadPreferenceManagementPreferencesInteractive({ colors.green( `Dry run complete, exiting. ${ Object.values(pendingUpdates).length - } pending updates. Check file: ${receiptFilepath}` - ) + } pending updates. Check file: ${receiptFilepath}`, + ), ); return; } @@ -212,8 +212,8 @@ export async function uploadPreferenceManagementPreferencesInteractive({ colors.magenta( `Uploading ${ Object.values(pendingUpdates).length - } preferences to partition: ${partition}` - ) + } preferences to partition: ${partition}`, + ), ); // Time duration @@ -222,7 +222,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Build a GraphQL client @@ -236,7 +236,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ // Make the request try { await sombra - .put("v1/preferences", { + .put('v1/preferences', { json: { records: currentChunk.map(([, update]) => update), skipWorkflowTriggers, @@ -246,7 +246,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ .json(); } catch (error) { try { - const parsed = JSON.parse(error?.response?.body || "{}"); + const parsed = JSON.parse(error?.response?.body || '{}'); if (parsed.error) { logger.error(colors.red(`Error: ${parsed.error}`)); } @@ -259,18 +259,18 @@ export async function uploadPreferenceManagementPreferencesInteractive({ currentChunk.length } user preferences to partition ${partition}: ${ error?.response?.body || error?.message - }` - ) + }`, + ), ); - const failingUpdates = preferenceState.getValue("failingUpdates"); + const failingUpdates = preferenceState.getValue('failingUpdates'); for (const [userId, update] of currentChunk) { failingUpdates[userId] = { uploadedAt: new Date().toISOString(), update, - error: error?.response?.body || error?.message || "Unknown error", + error: error?.response?.body || error?.message || 'Unknown error', }; } - await preferenceState.setValue(failingUpdates, "failingUpdates"); + await preferenceState.setValue(failingUpdates, 'failingUpdates'); } total += currentChunk.length; @@ -278,7 +278,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ }, { concurrency: 40, - } + }, ); progressBar.stop(); @@ -290,7 +290,7 @@ export async function uploadPreferenceManagementPreferencesInteractive({ updatesToRun.length } user preferences to partition ${partition} in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); } diff --git a/src/lib/readTranscendYaml.ts b/src/lib/readTranscendYaml.ts index d31ed407..ea8e645e 100644 --- a/src/lib/readTranscendYaml.ts +++ b/src/lib/readTranscendYaml.ts @@ -1,10 +1,10 @@ -import { readFileSync, writeFileSync } from "node:fs"; -import { decodeCodec, ObjByString } from "@transcend-io/type-utils"; -import yaml from "js-yaml"; -import { TranscendInput } from "../codecs"; +import { readFileSync, writeFileSync } from 'node:fs'; +import { decodeCodec, ObjByString } from '@transcend-io/type-utils'; +import yaml from 'js-yaml'; +import { TranscendInput } from '../codecs'; export const VARIABLE_PARAMETERS_REGEXP = /<>/; -export const VARIABLE_PARAMETERS_NAME = "parameters"; +export const VARIABLE_PARAMETERS_NAME = 'parameters'; /** * Function that replaces variables in a text file. @@ -18,7 +18,7 @@ export const VARIABLE_PARAMETERS_NAME = "parameters"; export function replaceVariablesInYaml( input: string, variables: ObjByString, - extraErrorMessage = "" + extraErrorMessage = '', ): string { let contents = input; // Replace variables @@ -34,7 +34,7 @@ export function replaceVariablesInYaml( throw new Error( `Found variable that was not set: ${name}. Make sure you are passing all parameters through the --${VARIABLE_PARAMETERS_NAME}=${name}:value-for-param flag. -${extraErrorMessage}` +${extraErrorMessage}`, ); } @@ -51,16 +51,16 @@ ${extraErrorMessage}` */ export function readTranscendYaml( filePath: string, - variables: ObjByString = {} + variables: ObjByString = {}, ): TranscendInput { // Read in contents - const fileContents = readFileSync(filePath, "utf-8"); + const fileContents = readFileSync(filePath, 'utf-8'); // Replace variables const replacedVariables = replaceVariablesInYaml( fileContents, variables, - `Also check that there are no extra variables defined in your yaml: ${filePath}` + `Also check that there are no extra variables defined in your yaml: ${filePath}`, ); // Validate shape @@ -75,7 +75,7 @@ export function readTranscendYaml( */ export function writeTranscendYaml( filePath: string, - input: TranscendInput + input: TranscendInput, ): void { writeFileSync(filePath, yaml.dump(decodeCodec(TranscendInput, input))); } diff --git a/src/lib/requests/approvePrivacyRequests.ts b/src/lib/requests/approvePrivacyRequests.ts index bfe2eebe..dd331f2e 100644 --- a/src/lib/requests/approvePrivacyRequests.ts +++ b/src/lib/requests/approvePrivacyRequests.ts @@ -2,19 +2,19 @@ import { RequestAction, RequestOrigin, RequestStatus, -} from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +} from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { APPROVE_PRIVACY_REQUEST, buildTranscendGraphQLClient, fetchAllRequests, makeGraphQLRequest, UPDATE_PRIVACY_REQUEST, -} from "../graphql"; +} from '../graphql'; /** * Approve a set of privacy requests @@ -57,7 +57,7 @@ export async function approvePrivacyRequests({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Pull in the requests @@ -98,7 +98,7 @@ export async function approvePrivacyRequests({ input: { requestId: requestToApprove.id }, }); } catch (error) { - if (error.message.includes("Request must be in an approving state,")) { + if (error.message.includes('Request must be in an approving state,')) { skipped += 1; } } @@ -106,7 +106,7 @@ export async function approvePrivacyRequests({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -119,8 +119,8 @@ export async function approvePrivacyRequests({ colors.green( `Successfully approved ${total} requests in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return allRequests.length; } diff --git a/src/lib/requests/bulkRestartRequests.ts b/src/lib/requests/bulkRestartRequests.ts index b022421b..07a04e91 100644 --- a/src/lib/requests/bulkRestartRequests.ts +++ b/src/lib/requests/bulkRestartRequests.ts @@ -1,22 +1,22 @@ -import { join } from "node:path"; -import { PersistedState } from "@transcend-io/persisted-state"; -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import * as t from "io-ts"; -import { difference } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { join } from 'node:path'; +import { PersistedState } from '@transcend-io/persisted-state'; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import * as t from 'io-ts'; +import { difference } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequestIdentifiers, fetchAllRequests, -} from "../graphql"; -import { SuccessfulRequest } from "./constants"; -import { extractClientError } from "./extractClientError"; -import { restartPrivacyRequest } from "./restartPrivacyRequest"; +} from '../graphql'; +import { SuccessfulRequest } from './constants'; +import { extractClientError } from './extractClientError'; +import { restartPrivacyRequest } from './restartPrivacyRequest'; /** Minimal state we need to keep a list of requests */ const ErrorRequest = t.intersection([ @@ -96,13 +96,13 @@ export async function bulkRestartRequests({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Create a new state file to store the requests from this run const cacheFile = join( requestReceiptFolder, - `tr-request-restart-${new Date().toISOString()}` + `tr-request-restart-${new Date().toISOString()}`, ); const state = new PersistedState(cacheFile, CachedRequestState, { restartedRequests: [], @@ -122,33 +122,33 @@ export async function bulkRestartRequests({ createdAtAfter, }); const requests = allRequests.filter( - (request) => new Date(request.createdAt) < createdAt + (request) => new Date(request.createdAt) < createdAt, ); logger.info(`Found ${requests.length} requests to process`); if (copyIdentifiers) { - logger.info("copyIdentifiers detected - All Identifiers will be copied."); + logger.info('copyIdentifiers detected - All Identifiers will be copied.'); } if (sendEmailReceipt) { - logger.info("sendEmailReceipt detected - Email receipts will be sent."); + logger.info('sendEmailReceipt detected - Email receipts will be sent.'); } if (skipWaitingPeriod) { - logger.info("skipWaitingPeriod detected - Waiting period will be skipped."); + logger.info('skipWaitingPeriod detected - Waiting period will be skipped.'); } // Validate request IDs if (requestIds.length > 0 && requestIds.length !== requests.length) { const missingRequests = difference( requestIds, - requests.map(({ id }) => id) + requests.map(({ id }) => id), ); if (missingRequests.length > 0) { logger.error( colors.red( `Failed to find the following requests by ID: ${missingRequests.join( - "," - )}.` - ) + ',', + )}.`, + ), ); process.exit(1); } @@ -185,11 +185,11 @@ export async function bulkRestartRequests({ skipWaitingPeriod, sendEmailReceipt, emailIsVerified, - } + }, ); // Cache successful upload - const restartedRequests = state.getValue("restartedRequests"); + const restartedRequests = state.getValue('restartedRequests'); restartedRequests.push({ id: requestResponse.id, link: requestResponse.link, @@ -197,16 +197,16 @@ export async function bulkRestartRequests({ coreIdentifier: requestResponse.coreIdentifier, attemptedAt: new Date().toISOString(), }); - await state.setValue(restartedRequests, "restartedRequests"); + await state.setValue(restartedRequests, 'restartedRequests'); } catch (error) { const message = `${error.message} - ${JSON.stringify( error.response?.body, null, - 2 + 2, )}`; const clientError = extractClientError(message); - const failingRequests = state.getValue("failingRequests"); + const failingRequests = state.getValue('failingRequests'); failingRequests.push({ id: request.id, link: request.link, @@ -215,12 +215,12 @@ export async function bulkRestartRequests({ attemptedAt: new Date().toISOString(), error: clientError || message, }); - await state.setValue(failingRequests, "failingRequests"); + await state.setValue(failingRequests, 'failingRequests'); } total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -230,17 +230,17 @@ export async function bulkRestartRequests({ // Log completion time logger.info( colors.green( - `Completed restarting of requests in "${totalTime / 1000}" seconds.` - ) + `Completed restarting of requests in "${totalTime / 1000}" seconds.`, + ), ); // Log errors - if (state.getValue("failingRequests").length > 0) { + if (state.getValue('failingRequests').length > 0) { logger.error( colors.red( - `Encountered "${state.getValue("failingRequests").length}" errors. ` + - `See "${cacheFile}" to review the error messages and inputs.` - ) + `Encountered "${state.getValue('failingRequests').length}" errors. ` + + `See "${cacheFile}" to review the error messages and inputs.`, + ), ); process.exit(1); } diff --git a/src/lib/requests/bulkRetryEnrichers.ts b/src/lib/requests/bulkRetryEnrichers.ts index 7e83e71f..d854fb35 100644 --- a/src/lib/requests/bulkRetryEnrichers.ts +++ b/src/lib/requests/bulkRetryEnrichers.ts @@ -2,19 +2,19 @@ import { RequestAction, RequestEnricherStatus, RequestStatus, -} from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { difference } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +} from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { difference } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequestEnrichers, fetchAllRequests, retryRequestEnricher, -} from "../graphql"; +} from '../graphql'; /** * Restart a bunch of request enrichers @@ -56,13 +56,13 @@ export async function bulkRetryEnrichers({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Find all requests made before createdAt that are in a removing data state const client = buildTranscendGraphQLClient(transcendUrl, auth); - logger.info(colors.magenta("Fetching requests to restart...")); + logger.info(colors.magenta('Fetching requests to restart...')); const requests = await fetchAllRequests(client, { actions: requestActions, @@ -78,15 +78,15 @@ export async function bulkRetryEnrichers({ if (requestIds.length > 0 && requestIds.length !== requests.length) { const missingRequests = difference( requestIds, - requests.map(({ id }) => id) + requests.map(({ id }) => id), ); if (missingRequests.length > 0) { logger.error( colors.red( `Failed to find the following requests by ID: ${missingRequests.join( - "," - )}.` - ) + ',', + )}.`, + ), ); process.exit(1); } @@ -105,7 +105,7 @@ export async function bulkRetryEnrichers({ const requestEnrichersToRestart = requestEnrichers.filter( (requestEnricher) => requestEnricher.enricher.id === enricherId && - requestEnricherStatuses.includes(requestEnricher.status) + requestEnricherStatuses.includes(requestEnricher.status), ); await map(requestEnrichersToRestart, async (requestEnricher) => { await retryRequestEnricher(client, requestEnricher.id); @@ -116,7 +116,7 @@ export async function bulkRetryEnrichers({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -130,7 +130,7 @@ export async function bulkRetryEnrichers({ requests.length } requests and ${totalRestarted} enrichers in "${ totalTime / 1000 - }" seconds.` - ) + }" seconds.`, + ), ); } diff --git a/src/lib/requests/cancelPrivacyRequests.ts b/src/lib/requests/cancelPrivacyRequests.ts index e3b43a6b..815ff5d5 100644 --- a/src/lib/requests/cancelPrivacyRequests.ts +++ b/src/lib/requests/cancelPrivacyRequests.ts @@ -1,9 +1,9 @@ -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, CANCEL_PRIVACY_REQUEST, @@ -12,7 +12,7 @@ import { makeGraphQLRequest, Template, UPDATE_PRIVACY_REQUEST, -} from "../graphql"; +} from '../graphql'; /** * Cancel a set of privacy requests @@ -70,7 +70,7 @@ export async function cancelPrivacyRequests({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Grab the template with that title @@ -78,14 +78,14 @@ export async function cancelPrivacyRequests({ if (cancellationTitle) { const matchingTemplates = await fetchAllTemplates( client, - cancellationTitle + cancellationTitle, ); const exactTitleMatch = matchingTemplates.find( - (template) => template.title === cancellationTitle + (template) => template.title === cancellationTitle, ); if (!exactTitleMatch) { throw new Error( - `Failed to find a template with title: "${cancellationTitle}"` + `Failed to find a template with title: "${cancellationTitle}"`, ); } cancelationTemplate = exactTitleMatch; @@ -106,9 +106,9 @@ export async function cancelPrivacyRequests({ `Canceling "${allRequests.length}" requests${ cancelationTemplate ? ` Using template: ${cancelationTemplate.title}` - : "" - }.` - ) + : '' + }.`, + ), ); let total = 0; @@ -146,7 +146,7 @@ export async function cancelPrivacyRequests({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -157,8 +157,8 @@ export async function cancelPrivacyRequests({ colors.green( `Successfully canceled ${total} requests in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return allRequests.length; } diff --git a/src/lib/requests/constants.ts b/src/lib/requests/constants.ts index 3e785dd8..d917b352 100644 --- a/src/lib/requests/constants.ts +++ b/src/lib/requests/constants.ts @@ -1,44 +1,44 @@ -import { LanguageKey } from "@transcend-io/internationalization"; +import { LanguageKey } from '@transcend-io/internationalization'; import { CompletedRequestStatus, IsoCountryCode, IsoCountrySubdivisionCode, RequestAction, -} from "@transcend-io/privacy-types"; -import { applyEnum, valuesOf } from "@transcend-io/type-utils"; -import * as t from "io-ts"; +} from '@transcend-io/privacy-types'; +import { applyEnum, valuesOf } from '@transcend-io/type-utils'; +import * as t from 'io-ts'; -export const NONE = "[NONE]" as const; -export const BULK_APPLY = "[APPLY VALUE TO ALL ROWS]" as const; -export const BLANK = "" as const; +export const NONE = '[NONE]' as const; +export const BULK_APPLY = '[APPLY VALUE TO ALL ROWS]' as const; +export const BLANK = '' as const; /** These are uploaded at the top level of the request */ -export const IDENTIFIER_BLOCK_LIST = ["email", "coreIdentifier"]; +export const IDENTIFIER_BLOCK_LIST = ['email', 'coreIdentifier']; /** * Column names to map */ export enum ColumnName { /** The title of the email column */ - Email = "email", + Email = 'email', /** The title of the core identifier column */ - CoreIdentifier = "coreIdentifier", + CoreIdentifier = 'coreIdentifier', /** The title of the requestType column */ - RequestType = "requestType", + RequestType = 'requestType', /** The title of the subjectType column */ - SubjectType = "subjectType", + SubjectType = 'subjectType', /** The title of the locale column */ - Locale = "locale", + Locale = 'locale', /** The country */ - Country = "country", + Country = 'country', /** The country sub division */ - CountrySubDivision = "countrySubDivision", + CountrySubDivision = 'countrySubDivision', /** The title of the requestStatus column */ - RequestStatus = "requestStatus", + RequestStatus = 'requestStatus', /** The title of the createdAt column */ - CreatedAt = "createdAt", + CreatedAt = 'createdAt', /** The title of the dataSiloIds column */ - DataSiloIds = "dataSiloIds", + DataSiloIds = 'dataSiloIds', } /** These parameters are required in the Transcend DSR API */ @@ -78,17 +78,17 @@ export const CachedFileState = t.type({ /** Mapping between region and country code */ regionToCountry: t.record( t.string, - valuesOf({ ...IsoCountryCode, [NONE]: NONE }) + valuesOf({ ...IsoCountryCode, [NONE]: NONE }), ), /** Mapping between region and country sub division code */ regionToCountrySubDivision: t.record( t.string, - valuesOf({ ...IsoCountrySubdivisionCode, [NONE]: NONE }) + valuesOf({ ...IsoCountrySubdivisionCode, [NONE]: NONE }), ), /** Mapping between request status in import to Transcend request status */ statusToRequestStatus: t.record( t.string, - valuesOf({ ...CompletedRequestStatus, [NONE]: NONE }) + valuesOf({ ...CompletedRequestStatus, [NONE]: NONE }), ), }); @@ -121,7 +121,7 @@ export const CachedRequestState = t.type({ rowIndex: t.number, coreIdentifier: t.string, attemptedAt: t.string, - }) + }), ), }); diff --git a/src/lib/requests/downloadPrivacyRequestFiles.ts b/src/lib/requests/downloadPrivacyRequestFiles.ts index e5771a85..83fc4ee1 100644 --- a/src/lib/requests/downloadPrivacyRequestFiles.ts +++ b/src/lib/requests/downloadPrivacyRequestFiles.ts @@ -1,20 +1,20 @@ -import { existsSync, mkdirSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { APPROVE_PRIVACY_REQUEST, buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequests, makeGraphQLRequest, -} from "../graphql"; -import { getFileMetadataForPrivacyRequests } from "./getFileMetadataForPrivacyRequests"; -import { streamPrivacyRequestFiles } from "./streamPrivacyRequestFiles"; +} from '../graphql'; +import { getFileMetadataForPrivacyRequests } from './getFileMetadataForPrivacyRequests'; +import { streamPrivacyRequestFiles } from './streamPrivacyRequestFiles'; /** * Download a set of privacy requests to disk @@ -81,14 +81,14 @@ export async function downloadPrivacyRequestFiles({ { sombra, concurrency, - } + }, ); // Start timer for download process const t0 = Date.now(); const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); let total = 0; let totalApproved = 0; @@ -134,7 +134,7 @@ export async function downloadPrivacyRequestFiles({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -145,12 +145,12 @@ export async function downloadPrivacyRequestFiles({ colors.green( `Successfully downloaded ${total} requests in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); if (totalApproved > 0) { logger.info( - colors.green(`Approved ${totalApproved} requests in Transcend.`) + colors.green(`Approved ${totalApproved} requests in Transcend.`), ); } return allRequests.length; diff --git a/src/lib/requests/filterRows.ts b/src/lib/requests/filterRows.ts index 9cb62f30..ac289466 100644 --- a/src/lib/requests/filterRows.ts +++ b/src/lib/requests/filterRows.ts @@ -1,10 +1,10 @@ -import { ObjByString } from "@transcend-io/type-utils"; -import colors from "colors"; -import inquirer from "inquirer"; -import { uniq } from "lodash-es"; -import { logger } from "../../logger"; -import { NONE } from "./constants"; -import { getUniqueValuesForColumn } from "./getUniqueValuesForColumn"; +import { ObjByString } from '@transcend-io/type-utils'; +import colors from 'colors'; +import inquirer from 'inquirer'; +import { uniq } from 'lodash-es'; +import { logger } from '../../logger'; +import { NONE } from './constants'; +import { getUniqueValuesForColumn } from './getUniqueValuesForColumn'; /** * Filter a list of CSV rows by column values @@ -30,10 +30,10 @@ export async function filterRows(rows: ObjByString[]): Promise { filterColumnName: string; }>([ { - name: "filterColumnName", + name: 'filterColumnName', message: `If you need to filter the list of requests to import, choose the column to filter on. Currently ${filteredRows.length} rows.`, - type: "list", + type: 'list', default: columnNames, choices: [NONE, ...columnNames], }, @@ -49,16 +49,16 @@ export async function filterRows(rows: ObjByString[]): Promise { valuesToKeep: string[]; }>([ { - name: "valuesToKeep", - message: "Keep rows matching this value", - type: "checkbox", + name: 'valuesToKeep', + message: 'Keep rows matching this value', + type: 'checkbox', default: columnNames, choices: options, }, ]); filteredRows = filteredRows.filter((request) => - valuesToKeep.includes(request[filterColumnName]) + valuesToKeep.includes(request[filterColumnName]), ); } } diff --git a/src/lib/requests/getFileMetadataForPrivacyRequests.ts b/src/lib/requests/getFileMetadataForPrivacyRequests.ts index c74a98d2..0c856642 100644 --- a/src/lib/requests/getFileMetadataForPrivacyRequests.ts +++ b/src/lib/requests/getFileMetadataForPrivacyRequests.ts @@ -1,12 +1,12 @@ -import { TableEncryptionType } from "@transcend-io/privacy-types"; -import { decodeCodec, valuesOf } from "@transcend-io/type-utils"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import type { Got } from "got"; -import * as t from "io-ts"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { PrivacyRequest } from "../graphql"; +import { TableEncryptionType } from '@transcend-io/privacy-types'; +import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import type { Got } from 'got'; +import * as t from 'io-ts'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { PrivacyRequest } from '../graphql'; export const IntlMessage = t.type({ /** The message key */ @@ -93,7 +93,7 @@ export type RequestFileMetadataResponse = t.TypeOf< * @returns The number of requests canceled */ export async function getFileMetadataForPrivacyRequests( - requests: Pick[], + requests: Pick[], { sombra, concurrency = 5, @@ -105,10 +105,10 @@ export async function getFileMetadataForPrivacyRequests( limit?: number; /** Concurrency limit for approving */ concurrency?: number; - } -): Promise<[Pick, RequestFileMetadata[]][]> { + }, +): Promise<[Pick, RequestFileMetadata[]][]> { logger.info( - colors.magenta(`Pulling file metadata for ${requests.length} requests`) + colors.magenta(`Pulling file metadata for ${requests.length} requests`), ); // Time duration @@ -116,7 +116,7 @@ export async function getFileMetadataForPrivacyRequests( // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Start timer @@ -127,9 +127,9 @@ export async function getFileMetadataForPrivacyRequests( const results = await map( requests, async ( - requestToDownload + requestToDownload, ): Promise< - [Pick, RequestFileMetadata[]] + [Pick, RequestFileMetadata[]] > => { const localResults: RequestFileMetadata[] = []; @@ -149,7 +149,7 @@ export async function getFileMetadataForPrivacyRequests( limit, offset, }, - } + }, ) .json(); response = decodeCodec(RequestFileMetadataResponse, rawResponse); @@ -163,7 +163,7 @@ export async function getFileMetadataForPrivacyRequests( throw new Error( `Received an error from server: ${ error?.response?.body || error?.message - }` + }`, ); } } @@ -172,7 +172,7 @@ export async function getFileMetadataForPrivacyRequests( progressBar.update(total); return [requestToDownload, localResults]; }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -183,8 +183,8 @@ export async function getFileMetadataForPrivacyRequests( colors.green( `Successfully downloaded file metadata ${requests.length} requests in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return results; diff --git a/src/lib/requests/getUniqueValuesForColumn.ts b/src/lib/requests/getUniqueValuesForColumn.ts index fb4f56ee..99527b67 100644 --- a/src/lib/requests/getUniqueValuesForColumn.ts +++ b/src/lib/requests/getUniqueValuesForColumn.ts @@ -1,5 +1,5 @@ -import { ObjByString } from "@transcend-io/type-utils"; -import { uniq } from "lodash-es"; +import { ObjByString } from '@transcend-io/type-utils'; +import { uniq } from 'lodash-es'; /** * Return the unique set of values for a column in a CSV @@ -10,7 +10,7 @@ import { uniq } from "lodash-es"; */ export function getUniqueValuesForColumn( rows: ObjByString[], - columnName: string + columnName: string, ): string[] { - return uniq(rows.flatMap((row) => row[columnName] || "")); + return uniq(rows.flatMap((row) => row[columnName] || '')); } diff --git a/src/lib/requests/mapColumnsToAttributes.ts b/src/lib/requests/mapColumnsToAttributes.ts index 3697d0da..fa86032b 100644 --- a/src/lib/requests/mapColumnsToAttributes.ts +++ b/src/lib/requests/mapColumnsToAttributes.ts @@ -1,9 +1,9 @@ -import type { PersistedState } from "@transcend-io/persisted-state"; -import type { GraphQLClient } from "graphql-request"; -import inquirer from "inquirer"; -import { AttributeKey } from "../graphql"; -import { CachedFileState } from "./constants"; -import { fuzzyMatchColumns } from "./fuzzyMatchColumns"; +import type { PersistedState } from '@transcend-io/persisted-state'; +import type { GraphQLClient } from 'graphql-request'; +import inquirer from 'inquirer'; +import { AttributeKey } from '../graphql'; +import { CachedFileState } from './constants'; +import { fuzzyMatchColumns } from './fuzzyMatchColumns'; /** * Mapping from attribute name to request input parameter @@ -25,11 +25,11 @@ export async function mapColumnsToAttributes( client: GraphQLClient, columnNames: string[], state: PersistedState, - requestAttributeKeys: AttributeKey[] + requestAttributeKeys: AttributeKey[], ): Promise { // Determine the columns that should be mapped const columnQuestions = requestAttributeKeys.filter( - ({ name }) => !state.getValue("attributeNames", name) + ({ name }) => !state.getValue('attributeNames', name), ); // Skip mapping when everything is mapped @@ -43,20 +43,20 @@ export async function mapColumnsToAttributes( return { name, message: `Choose the column that will be used to map in the attribute: ${name}`, - type: "list", + type: 'list', default: matches[0], choices: matches, }; - }) + }), ); await Promise.all( Object.entries(attributeNameMap).map(([k, v]) => - state.setValue(v, "attributeNames", k) - ) + state.setValue(v, 'attributeNames', k), + ), ); return { - ...state.getValue("attributeNames"), + ...state.getValue('attributeNames'), ...attributeNameMap, }; } diff --git a/src/lib/requests/mapColumnsToIdentifiers.ts b/src/lib/requests/mapColumnsToIdentifiers.ts index 214cc888..d88deaea 100644 --- a/src/lib/requests/mapColumnsToIdentifiers.ts +++ b/src/lib/requests/mapColumnsToIdentifiers.ts @@ -1,9 +1,9 @@ -import type { PersistedState } from "@transcend-io/persisted-state"; -import type { GraphQLClient } from "graphql-request"; -import inquirer from "inquirer"; -import { INITIALIZER, Initializer, makeGraphQLRequest } from "../graphql"; -import { CachedFileState, IDENTIFIER_BLOCK_LIST } from "./constants"; -import { fuzzyMatchColumns } from "./fuzzyMatchColumns"; +import type { PersistedState } from '@transcend-io/persisted-state'; +import type { GraphQLClient } from 'graphql-request'; +import inquirer from 'inquirer'; +import { INITIALIZER, Initializer, makeGraphQLRequest } from '../graphql'; +import { CachedFileState, IDENTIFIER_BLOCK_LIST } from './constants'; +import { fuzzyMatchColumns } from './fuzzyMatchColumns'; /** * Mapping from identifier name to request input parameter @@ -23,7 +23,7 @@ export type IdentifierNameMap = Record; export async function mapColumnsToIdentifiers( client: GraphQLClient, columnNames: string[], - state: PersistedState + state: PersistedState, ): Promise { // Grab the initializer const { initializer } = await makeGraphQLRequest<{ @@ -34,8 +34,8 @@ export async function mapColumnsToIdentifiers( // Determine the columns that should be mapped const columnQuestions = initializer.identifiers.filter( ({ name }) => - !state.getValue("identifierNames", name) && - !IDENTIFIER_BLOCK_LIST.includes(name) + !state.getValue('identifierNames', name) && + !IDENTIFIER_BLOCK_LIST.includes(name), ); // Skip mapping when everything is mapped @@ -49,20 +49,20 @@ export async function mapColumnsToIdentifiers( return { name, message: `Choose the column that will be used to map in the identifier: ${name}`, - type: "list", + type: 'list', default: matches[0], choices: matches, }; - }) + }), ); await Promise.all( Object.entries(identifierNameMap).map(([k, v]) => - state.setValue(v, "identifierNames", k) - ) + state.setValue(v, 'identifierNames', k), + ), ); return { - ...state.getValue("identifierNames"), + ...state.getValue('identifierNames'), ...identifierNameMap, }; } diff --git a/src/lib/requests/mapCsvColumnsToApi.ts b/src/lib/requests/mapCsvColumnsToApi.ts index 76c1b0d1..11547c04 100644 --- a/src/lib/requests/mapCsvColumnsToApi.ts +++ b/src/lib/requests/mapCsvColumnsToApi.ts @@ -1,14 +1,14 @@ -import type { PersistedState } from "@transcend-io/persisted-state"; -import { getEntries, getValues } from "@transcend-io/type-utils"; -import inquirer from "inquirer"; -import { startCase } from "lodash-es"; +import type { PersistedState } from '@transcend-io/persisted-state'; +import { getEntries, getValues } from '@transcend-io/type-utils'; +import inquirer from 'inquirer'; +import { startCase } from 'lodash-es'; import { CachedFileState, CAN_APPLY_IN_BULK, ColumnName, IS_REQUIRED, -} from "./constants"; -import { fuzzyMatchColumns } from "./fuzzyMatchColumns"; +} from './constants'; +import { fuzzyMatchColumns } from './fuzzyMatchColumns'; /** * Mapping from column name to request input parameter @@ -24,11 +24,11 @@ export type ColumnNameMap = Partial>; */ export async function mapCsvColumnsToApi( columnNames: string[], - state: PersistedState + state: PersistedState, ): Promise { // Determine the columns that should be mapped const columnQuestions = getValues(ColumnName).filter( - (name) => !state.getValue("columnNames", name) + (name) => !state.getValue('columnNames', name), ); // Skip mapping when everything is mapped @@ -38,27 +38,27 @@ export async function mapCsvColumnsToApi( : // prompt questions to map columns await inquirer.prompt>>( columnQuestions.map((name) => { - const field = startCase(name.replace("ColumnName", "")); + const field = startCase(name.replace('ColumnName', '')); const matches = fuzzyMatchColumns( columnNames, field, IS_REQUIRED[name], - !!CAN_APPLY_IN_BULK[name] + !!CAN_APPLY_IN_BULK[name], ); return { name, message: `Choose the column that will be used to map in the field: ${field}`, - type: "list", + type: 'list', default: matches[0], choices: matches, }; - }) + }), ); await Promise.all( getEntries(columnNameMap).map(([k, v]) => - state.setValue(v, "columnNames", k) - ) + state.setValue(v, 'columnNames', k), + ), ); return columnNameMap; } diff --git a/src/lib/requests/mapCsvRowsToRequestInputs.ts b/src/lib/requests/mapCsvRowsToRequestInputs.ts index 125ad974..da8ebefa 100644 --- a/src/lib/requests/mapCsvRowsToRequestInputs.ts +++ b/src/lib/requests/mapCsvRowsToRequestInputs.ts @@ -1,5 +1,5 @@ -import { LanguageKey } from "@transcend-io/internationalization"; -import type { PersistedState } from "@transcend-io/persisted-state"; +import { LanguageKey } from '@transcend-io/internationalization'; +import type { PersistedState } from '@transcend-io/persisted-state'; import { CompletedRequestStatus, IdentifierType, @@ -7,23 +7,23 @@ import { IsoCountrySubdivisionCode, NORMALIZE_PHONE_NUMBER, RequestAction, -} from "@transcend-io/privacy-types"; -import { ObjByString, valuesOf } from "@transcend-io/type-utils"; -import * as t from "io-ts"; -import { DateFromISOString } from "io-ts-types"; -import { AttributeKey } from "../graphql"; +} from '@transcend-io/privacy-types'; +import { ObjByString, valuesOf } from '@transcend-io/type-utils'; +import * as t from 'io-ts'; +import { DateFromISOString } from 'io-ts-types'; +import { AttributeKey } from '../graphql'; import { BLANK, BULK_APPLY, CachedFileState, ColumnName, NONE, -} from "./constants"; -import { AttributeNameMap } from "./mapColumnsToAttributes"; -import { IdentifierNameMap } from "./mapColumnsToIdentifiers"; -import { ColumnNameMap } from "./mapCsvColumnsToApi"; -import { ParsedAttributeInput } from "./parseAttributesFromString"; -import { splitCsvToList } from "./splitCsvToList"; +} from './constants'; +import { AttributeNameMap } from './mapColumnsToAttributes'; +import { IdentifierNameMap } from './mapColumnsToIdentifiers'; +import { ColumnNameMap } from './mapCsvColumnsToApi'; +import { ParsedAttributeInput } from './parseAttributesFromString'; +import { splitCsvToList } from './splitCsvToList'; /** * Shape of additional identifiers @@ -42,8 +42,8 @@ export const AttestedExtraIdentifiers = t.record( /** Name of identifier - option for non-custom identifier types */ name: t.string, }), - ]) - ) + ]), + ), ); /** Type override */ @@ -96,7 +96,7 @@ export type PrivacyRequestInput = t.TypeOf; export function normalizeIdentifierValue( identifierValue: string, identifierType: IdentifierType, - defaultPhoneCountryCode: string + defaultPhoneCountryCode: string, ): string { // Lowercase email if (identifierType === IdentifierType.Email) { @@ -106,17 +106,17 @@ export function normalizeIdentifierValue( // Normalize phone number if (identifierType === IdentifierType.Phone) { const normalized = identifierValue - .replace(NORMALIZE_PHONE_NUMBER, "") - .replaceAll(/[()]/g, "") - .replaceAll(/[–]/g, "") - .replaceAll(/[:]/g, "") - .replaceAll(/[‭‬]/g, "") - .replaceAll(/[A-Za-z]/g, ""); + .replace(NORMALIZE_PHONE_NUMBER, '') + .replaceAll(/[()]/g, '') + .replaceAll(/[–]/g, '') + .replaceAll(/[:]/g, '') + .replaceAll(/[‭‬]/g, '') + .replaceAll(/[A-Za-z]/g, ''); return normalized - ? normalized.startsWith("+") + ? normalized.startsWith('+') ? normalized : `+${defaultPhoneCountryCode}${normalized}` - : ""; + : ''; } return identifierValue; } @@ -139,7 +139,7 @@ export function mapCsvRowsToRequestInputs( identifierNameMap, attributeNameMap, requestAttributeKeys, - defaultPhoneCountryCode = "1", // US + defaultPhoneCountryCode = '1', // US }: { /** Default country code */ defaultPhoneCountryCode?: string; @@ -151,23 +151,23 @@ export function mapCsvRowsToRequestInputs( attributeNameMap: AttributeNameMap; /** Request attribute keys */ requestAttributeKeys: AttributeKey[]; - } + }, ): [Record, PrivacyRequestInput][] { // map the CSV to request input const getMappedName = (attribute: ColumnName): string => - state.getValue("columnNames", attribute) || columnNameMap[attribute]!; + state.getValue('columnNames', attribute) || columnNameMap[attribute]!; return requestInputs.map( (input): [Record, PrivacyRequestInput] => { // The extra identifiers to upload for this request const attestedExtraIdentifiers: AttestedExtraIdentifiers = {}; for (const [identifierName, columnName] of Object.entries( - identifierNameMap + identifierNameMap, ) // filter out skipped identifiers .filter(([, columnName]) => columnName !== NONE)) { // Determine the identifier type being specified const identifierType = Object.values(IdentifierType).includes( - identifierName as any // eslint-disable-line @typescript-eslint/no-explicit-any + identifierName as any, // eslint-disable-line @typescript-eslint/no-explicit-any ) ? (identifierName as IdentifierType) : IdentifierType.Custom; @@ -178,7 +178,7 @@ export function mapCsvRowsToRequestInputs( const normalized = normalizeIdentifierValue( identifierValue, identifierType, - defaultPhoneCountryCode + defaultPhoneCountryCode, ); if (normalized) { // Initialize @@ -206,8 +206,8 @@ export function mapCsvRowsToRequestInputs( // Add the attribute const isMulti = requestAttributeKeys.find( - (attribute) => attribute.name === attributeName - )?.type === "MULTI_SELECT"; + (attribute) => attribute.name === attributeName, + )?.type === 'MULTI_SELECT'; attributes.push({ values: isMulti ? splitCsvToList(attributeValueString) @@ -228,24 +228,24 @@ export function mapCsvRowsToRequestInputs( coreIdentifier: input[getMappedName(ColumnName.CoreIdentifier)], requestType: requestTypeColumn === BULK_APPLY - ? state.getValue("requestTypeToRequestAction", BLANK) + ? state.getValue('requestTypeToRequestAction', BLANK) : state.getValue( - "requestTypeToRequestAction", - input[requestTypeColumn] + 'requestTypeToRequestAction', + input[requestTypeColumn], ), subjectType: dataSubjectTypeColumn === BULK_APPLY - ? state.getValue("subjectTypeToSubjectName", BLANK) + ? state.getValue('subjectTypeToSubjectName', BLANK) : state.getValue( - "subjectTypeToSubjectName", - input[dataSubjectTypeColumn] + 'subjectTypeToSubjectName', + input[dataSubjectTypeColumn], ), ...(getMappedName(ColumnName.Locale) !== NONE && input[getMappedName(ColumnName.Locale)] ? { locale: state.getValue( - "languageToLocale", - input[getMappedName(ColumnName.Locale)] + 'languageToLocale', + input[getMappedName(ColumnName.Locale)], ), } : {}), @@ -253,8 +253,8 @@ export function mapCsvRowsToRequestInputs( input[getMappedName(ColumnName.Country)] ? { country: state.getValue( - "regionToCountry", - input[getMappedName(ColumnName.Country)] + 'regionToCountry', + input[getMappedName(ColumnName.Country)], ) as IsoCountryCode, } : {}), @@ -262,21 +262,21 @@ export function mapCsvRowsToRequestInputs( input[getMappedName(ColumnName.CountrySubDivision)] ? { countrySubDivision: state.getValue( - "regionToCountrySubDivision", - input[getMappedName(ColumnName.CountrySubDivision)] + 'regionToCountrySubDivision', + input[getMappedName(ColumnName.CountrySubDivision)], ) as IsoCountrySubdivisionCode, } : {}), ...(getMappedName(ColumnName.RequestStatus) !== NONE && state.getValue( - "statusToRequestStatus", - input[getMappedName(ColumnName.RequestStatus)] + 'statusToRequestStatus', + input[getMappedName(ColumnName.RequestStatus)], ) !== NONE && input[getMappedName(ColumnName.RequestStatus)] ? { status: state.getValue( - "statusToRequestStatus", - input[getMappedName(ColumnName.RequestStatus)] + 'statusToRequestStatus', + input[getMappedName(ColumnName.RequestStatus)], ) as CompletedRequestStatus, } : {}), @@ -290,12 +290,12 @@ export function mapCsvRowsToRequestInputs( input[getMappedName(ColumnName.DataSiloIds)] ? { dataSiloIds: splitCsvToList( - input[getMappedName(ColumnName.DataSiloIds)] + input[getMappedName(ColumnName.DataSiloIds)], ), } : {}), }, ]; - } + }, ); } diff --git a/src/lib/requests/mapEnumValues.ts b/src/lib/requests/mapEnumValues.ts index 4535a8c1..d53b4388 100644 --- a/src/lib/requests/mapEnumValues.ts +++ b/src/lib/requests/mapEnumValues.ts @@ -1,7 +1,7 @@ -import { apply, ObjByString } from "@transcend-io/type-utils"; -import inquirer from "inquirer"; -import autoCompletePrompt from "inquirer-autocomplete-prompt"; -import { fuzzySearch } from "./fuzzyMatchColumns"; +import { apply, ObjByString } from '@transcend-io/type-utils'; +import inquirer from 'inquirer'; +import autoCompletePrompt from 'inquirer-autocomplete-prompt'; +import { fuzzySearch } from './fuzzyMatchColumns'; /** * Map a set of inputs to a set of outputs @@ -14,12 +14,12 @@ import { fuzzySearch } from "./fuzzyMatchColumns"; export async function mapEnumValues( csvInputs: string[], expectedOutputs: TValue[], - cache: Record + cache: Record, ): Promise> { - inquirer.registerPrompt("autocomplete", autoCompletePrompt); + inquirer.registerPrompt('autocomplete', autoCompletePrompt); const inputs = csvInputs - .map((item) => item || "") + .map((item) => item || '') .filter((value) => !cache[value]); if (inputs.length === 0) { return cache; @@ -28,20 +28,20 @@ export async function mapEnumValues( inputs.map((value) => ({ name: value, message: `Map value of: ${value}`, - type: "autocomplete", + type: 'autocomplete', default: expectedOutputs.find((x) => fuzzySearch(value, x)), source: (answersSoFar: ObjByString, input: string) => input ? expectedOutputs.filter( - (x) => typeof x === "string" && fuzzySearch(input, x) + (x) => typeof x === 'string' && fuzzySearch(input, x), ) : expectedOutputs, - })) + })), ); return { ...cache, ...apply(result, (r) => - typeof r === "string" ? r : (Object.values(r)[0] as TValue) + typeof r === 'string' ? r : (Object.values(r)[0] as TValue), ), }; } diff --git a/src/lib/requests/mapRequestEnumValues.ts b/src/lib/requests/mapRequestEnumValues.ts index db5e93f9..baf2ee61 100644 --- a/src/lib/requests/mapRequestEnumValues.ts +++ b/src/lib/requests/mapRequestEnumValues.ts @@ -1,20 +1,20 @@ -import { LanguageKey } from "@transcend-io/internationalization"; -import type { PersistedState } from "@transcend-io/persisted-state"; +import { LanguageKey } from '@transcend-io/internationalization'; +import type { PersistedState } from '@transcend-io/persisted-state'; import { CompletedRequestStatus, IsoCountryCode, IsoCountrySubdivisionCode, RequestAction, -} from "@transcend-io/privacy-types"; -import { ObjByString } from "@transcend-io/type-utils"; -import colors from "colors"; -import { GraphQLClient } from "graphql-request"; -import { logger } from "../../logger"; -import { DATA_SUBJECTS, DataSubject, makeGraphQLRequest } from "../graphql"; -import { CachedFileState, ColumnName, NONE } from "./constants"; -import { getUniqueValuesForColumn } from "./getUniqueValuesForColumn"; -import { ColumnNameMap } from "./mapCsvColumnsToApi"; -import { mapEnumValues } from "./mapEnumValues"; +} from '@transcend-io/privacy-types'; +import { ObjByString } from '@transcend-io/type-utils'; +import colors from 'colors'; +import { GraphQLClient } from 'graphql-request'; +import { logger } from '../../logger'; +import { DATA_SUBJECTS, DataSubject, makeGraphQLRequest } from '../graphql'; +import { CachedFileState, ColumnName, NONE } from './constants'; +import { getUniqueValuesForColumn } from './getUniqueValuesForColumn'; +import { ColumnNameMap } from './mapCsvColumnsToApi'; +import { mapEnumValues } from './mapEnumValues'; /** * Map the values in a CSV to the enum values in Transcend @@ -34,11 +34,11 @@ export async function mapRequestEnumValues( state: PersistedState; /** Mapping of column names */ columnNameMap: ColumnNameMap; - } + }, ): Promise { // Get mapped value const getMappedName = (attribute: ColumnName): string => - state.getValue("columnNames", attribute) || columnNameMap[attribute]!; + state.getValue('columnNames', attribute) || columnNameMap[attribute]!; // Fetch all data subjects in the organization const { internalSubjects } = await makeGraphQLRequest<{ @@ -48,43 +48,43 @@ export async function mapRequestEnumValues( // Map RequestAction logger.info( - colors.magenta("Determining mapping of columns for request action") + colors.magenta('Determining mapping of columns for request action'), ); const requestTypeToRequestAction: Record = await mapEnumValues( getUniqueValuesForColumn(requests, getMappedName(ColumnName.RequestType)), Object.values(RequestAction), - state.getValue("requestTypeToRequestAction") + state.getValue('requestTypeToRequestAction'), ); await state.setValue( requestTypeToRequestAction, - "requestTypeToRequestAction" + 'requestTypeToRequestAction', ); // Map data subject type - logger.info(colors.magenta("Determining mapping of columns for subject")); + logger.info(colors.magenta('Determining mapping of columns for subject')); const subjectTypeToSubjectName: Record = await mapEnumValues( getUniqueValuesForColumn(requests, getMappedName(ColumnName.SubjectType)), internalSubjects.map(({ type }) => type), - state.getValue("subjectTypeToSubjectName") + state.getValue('subjectTypeToSubjectName'), ); - await state.setValue(subjectTypeToSubjectName, "subjectTypeToSubjectName"); + await state.setValue(subjectTypeToSubjectName, 'subjectTypeToSubjectName'); // Map locale - logger.info(colors.magenta("Determining mapping of columns for locale")); + logger.info(colors.magenta('Determining mapping of columns for locale')); const languageToLocale: Record = await mapEnumValues( getUniqueValuesForColumn(requests, getMappedName(ColumnName.Locale)), Object.values(LanguageKey), - state.getValue("languageToLocale") + state.getValue('languageToLocale'), ); - await state.setValue(languageToLocale, "languageToLocale"); + await state.setValue(languageToLocale, 'languageToLocale'); logger.info( - colors.magenta("Determining mapping of columns for request status") + colors.magenta('Determining mapping of columns for request status'), ); // Map request status logger.info( - colors.magenta("Determining mapping of columns for request status") + colors.magenta('Determining mapping of columns for request status'), ); const requestStatusColumn = getMappedName(ColumnName.RequestStatus); const statusToRequestStatus: Record< @@ -96,12 +96,12 @@ export async function mapRequestEnumValues( : await mapEnumValues( getUniqueValuesForColumn(requests, requestStatusColumn), [...Object.values(CompletedRequestStatus), NONE], - state.getValue("statusToRequestStatus") + state.getValue('statusToRequestStatus'), ); - await state.setValue(statusToRequestStatus, "statusToRequestStatus"); + await state.setValue(statusToRequestStatus, 'statusToRequestStatus'); // Map country - logger.info(colors.magenta("Determining mapping of columns for country")); + logger.info(colors.magenta('Determining mapping of columns for country')); const countryColumn = getMappedName(ColumnName.Country); const regionToCountry: Record = countryColumn === NONE @@ -109,13 +109,13 @@ export async function mapRequestEnumValues( : await mapEnumValues( getUniqueValuesForColumn(requests, countryColumn), [...Object.values(IsoCountryCode), NONE], - state.getValue("regionToCountry") + state.getValue('regionToCountry'), ); - await state.setValue(regionToCountry, "regionToCountry"); + await state.setValue(regionToCountry, 'regionToCountry'); // Map country sub division logger.info( - colors.magenta("Determining mapping of columns for country sub division") + colors.magenta('Determining mapping of columns for country sub division'), ); const countrySubDivisionColumn = getMappedName(ColumnName.CountrySubDivision); const regionToCountrySubDivision: Record< @@ -127,10 +127,10 @@ export async function mapRequestEnumValues( : await mapEnumValues( getUniqueValuesForColumn(requests, countrySubDivisionColumn), [...Object.values(IsoCountrySubdivisionCode), NONE], - state.getValue("regionToCountrySubDivision") + state.getValue('regionToCountrySubDivision'), ); await state.setValue( regionToCountrySubDivision, - "regionToCountrySubDivision" + 'regionToCountrySubDivision', ); } diff --git a/src/lib/requests/markSilentPrivacyRequests.ts b/src/lib/requests/markSilentPrivacyRequests.ts index 4a215125..eb6dc7e7 100644 --- a/src/lib/requests/markSilentPrivacyRequests.ts +++ b/src/lib/requests/markSilentPrivacyRequests.ts @@ -1,15 +1,15 @@ -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequests, makeGraphQLRequest, UPDATE_PRIVACY_REQUEST, -} from "../graphql"; +} from '../graphql'; /** * Mark a set of privacy requests to be in silent mode @@ -61,7 +61,7 @@ export async function markSilentPrivacyRequests({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Pull in the requests @@ -76,7 +76,7 @@ export async function markSilentPrivacyRequests({ // Notify Transcend logger.info( - colors.magenta(`Marking "${allRequests.length}" as silent mode.`) + colors.magenta(`Marking "${allRequests.length}" as silent mode.`), ); let total = 0; @@ -94,7 +94,7 @@ export async function markSilentPrivacyRequests({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -105,8 +105,8 @@ export async function markSilentPrivacyRequests({ colors.green( `Successfully marked ${total} requests as silent mode in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return allRequests.length; } diff --git a/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts b/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts index 2fc43ef3..9da01de9 100644 --- a/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts +++ b/src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts @@ -1,16 +1,16 @@ -import { RequestAction } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestAction } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequests, fetchAllTemplates, makeGraphQLRequest, NOTIFY_ADDITIONAL_TIME, -} from "../graphql"; +} from '../graphql'; /** * Mark a set of privacy requests to be in silent mode. @@ -27,7 +27,7 @@ export async function notifyPrivacyRequestsAdditionalTime({ days = 45, daysLeft = 10, createdAtAfter, - emailTemplate = "Additional Time Needed", + emailTemplate = 'Additional Time Needed', concurrency = 100, transcendUrl = DEFAULT_TRANSCEND_API, }: { @@ -63,13 +63,13 @@ export async function notifyPrivacyRequestsAdditionalTime({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Grab the template with that title const matchingTemplates = await fetchAllTemplates(client, emailTemplate); const exactTemplateMatch = matchingTemplates.find( - (template) => template.title === emailTemplate + (template) => template.title === emailTemplate, ); if (!exactTemplateMatch) { throw new Error(`Failed to find a template with title: "${emailTemplate}"`); @@ -88,15 +88,15 @@ export async function notifyPrivacyRequestsAdditionalTime({ // Filter requests by daysLeft allRequests = allRequests.filter( (request) => - typeof request.daysRemaining === "number" && - request.daysRemaining < daysLeft + typeof request.daysRemaining === 'number' && + request.daysRemaining < daysLeft, ); // Notify Transcend logger.info( colors.magenta( - `Notifying "${allRequests.length}" that more time is needed.` - ) + `Notifying "${allRequests.length}" that more time is needed.`, + ), ); let total = 0; @@ -116,7 +116,7 @@ export async function notifyPrivacyRequestsAdditionalTime({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -127,8 +127,8 @@ export async function notifyPrivacyRequestsAdditionalTime({ colors.green( `Successfully marked ${total} requests as silent mode in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return allRequests.length; } diff --git a/src/lib/requests/pullPrivacyRequests.ts b/src/lib/requests/pullPrivacyRequests.ts index 9cf43a17..5ba70bbb 100644 --- a/src/lib/requests/pullPrivacyRequests.ts +++ b/src/lib/requests/pullPrivacyRequests.ts @@ -1,9 +1,9 @@ -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import colors from "colors"; -import { groupBy } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import colors from 'colors'; +import { groupBy } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, @@ -11,7 +11,7 @@ import { fetchAllRequests, PrivacyRequest, RequestIdentifier, -} from "../graphql"; +} from '../graphql'; export interface ExportedPrivacyRequest extends PrivacyRequest { /** Request identifiers */ @@ -67,13 +67,13 @@ export async function pullPrivacyRequests({ const sombra = await createSombraGotInstance(transcendUrl, auth, sombraAuth); // Log date range - let dateRange = ""; + let dateRange = ''; if (createdAtBefore) { dateRange += ` before ${createdAtBefore.toISOString()}`; } if (createdAtAfter) { dateRange += `${ - dateRange ? ", and" : "" + dateRange ? ', and' : '' } after ${createdAtAfter.toISOString()}`; } @@ -83,9 +83,9 @@ export async function pullPrivacyRequests({ `${ actions.length > 0 ? `Pulling requests of type "${actions.join('" , "')}"` - : "Pulling all requests" - }${dateRange}` - ) + : 'Pulling all requests' + }${dateRange}`, + ), ); // fetch the requests @@ -107,7 +107,7 @@ export async function pullPrivacyRequests({ sombra, { requestId: request.id, - } + }, ); return { ...request, @@ -116,11 +116,11 @@ export async function pullPrivacyRequests({ }, { concurrency: pageLimit, - } + }, ); logger.info( - colors.magenta(`Pulled ${requestsWithRequestIdentifiers.length} requests`) + colors.magenta(`Pulled ${requestsWithRequestIdentifiers.length} requests`), ); // Write out to CSV @@ -144,32 +144,35 @@ export async function pullPrivacyRequests({ coreIdentifier, ...request }) => ({ - "Request ID": id, - "Created At": createdAt, + 'Request ID': id, + 'Created At': createdAt, Email: email, - "Core Identifier": coreIdentifier, - "Request Type": type, - "Data Subject Type": subjectType, + 'Core Identifier': coreIdentifier, + 'Request Type': type, + 'Data Subject Type': subjectType, Status: status, Country: country, - "Country Sub Division": countrySubDivision, + 'Country Sub Division': countrySubDivision, Details: details, Origin: origin, - "Silent Mode": isSilent, - "Is Test Request": isTest, + 'Silent Mode': isSilent, + 'Is Test Request': isTest, Language: locale, ...request, ...Object.fromEntries( - Object.entries(groupBy(attributeValues, "attributeKey.name")).map( - ([name, values]) => [name, values.map(({ name }) => name).join(",")] - ) + Object.entries(groupBy(attributeValues, 'attributeKey.name')).map( + ([name, values]) => [name, values.map(({ name }) => name).join(',')], + ), ), ...Object.fromEntries( - Object.entries(groupBy(requestIdentifiers, "name")).map( - ([name, values]) => [name, values.map(({ value }) => value).join(",")] - ) + Object.entries(groupBy(requestIdentifiers, 'name')).map( + ([name, values]) => [ + name, + values.map(({ value }) => value).join(','), + ], + ), ), - }) + }), ); return { requestsWithRequestIdentifiers, requestsFormattedForCsv: data }; diff --git a/src/lib/requests/readCsv.ts b/src/lib/requests/readCsv.ts index fed0f1cd..e949fe64 100644 --- a/src/lib/requests/readCsv.ts +++ b/src/lib/requests/readCsv.ts @@ -1,8 +1,8 @@ -import { readFileSync } from "node:fs"; -import { decodeCodec } from "@transcend-io/type-utils"; -import type { Options } from "csv-parse"; -import { parse } from "csv-parse/sync"; -import * as t from "io-ts"; +import { readFileSync } from 'node:fs'; +import { decodeCodec } from '@transcend-io/type-utils'; +import type { Options } from 'csv-parse'; +import { parse } from 'csv-parse/sync'; +import * as t from 'io-ts'; /** * Read in a CSV and validate its shape @@ -15,10 +15,10 @@ import * as t from "io-ts"; export function readCsv( pathToFile: string, codec: T, - options: Options = { columns: true } + options: Options = { columns: true }, ): t.TypeOf[] { // read file contents and parse - const fileContent = parse(readFileSync(pathToFile, "utf-8"), options); + const fileContent = parse(readFileSync(pathToFile, 'utf-8'), options); // validate codec const data = decodeCodec(t.array(codec), fileContent); @@ -28,10 +28,10 @@ export function readCsv( Object.entries(datum).reduce( (accumulator, [key, value]) => Object.assign(accumulator, { - [key.replaceAll(/[^a-z_.+\-A-Z -~]/g, "")]: value, + [key.replaceAll(/[^a-z_.+\-A-Z -~]/g, '')]: value, }), - {} as T - ) + {} as T, + ), ); return parsed; } diff --git a/src/lib/requests/removeUnverifiedRequestIdentifiers.ts b/src/lib/requests/removeUnverifiedRequestIdentifiers.ts index 2b8b02e3..0d4effc4 100644 --- a/src/lib/requests/removeUnverifiedRequestIdentifiers.ts +++ b/src/lib/requests/removeUnverifiedRequestIdentifiers.ts @@ -1,16 +1,16 @@ -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequestIdentifierMetadata, fetchAllRequests, makeGraphQLRequest, REMOVE_REQUEST_IDENTIFIERS, -} from "../graphql"; +} from '../graphql'; /** * Remove a set of unverified request identifier @@ -44,7 +44,7 @@ export async function removeUnverifiedRequestIdentifiers({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Pull in the requests @@ -54,7 +54,7 @@ export async function removeUnverifiedRequestIdentifiers({ }); // Notify Transcend - logger.info(colors.magenta("Fetched requests in preflight/enriching state.")); + logger.info(colors.magenta('Fetched requests in preflight/enriching state.')); let total = 0; let processed = 0; @@ -64,12 +64,12 @@ export async function removeUnverifiedRequestIdentifiers({ async (requestToRestart) => { const requestIdentifiers = await fetchAllRequestIdentifierMetadata( client, - { requestId: requestToRestart.id } + { requestId: requestToRestart.id }, ); const clearOut = requestIdentifiers .filter( ({ isVerifiedAtLeastOnce, name }) => - !isVerifiedAtLeastOnce && identifierNames.includes(name) + !isVerifiedAtLeastOnce && identifierNames.includes(name), ) .map(({ id }) => id); @@ -89,7 +89,7 @@ export async function removeUnverifiedRequestIdentifiers({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -100,8 +100,8 @@ export async function removeUnverifiedRequestIdentifiers({ colors.green( `Successfully cleared out unverified identifiers "${ totalTime / 1000 - }" seconds for ${total} requests, ${processed} identifiers were cleared out!` - ) + }" seconds for ${total} requests, ${processed} identifiers were cleared out!`, + ), ); return allRequests.length; } diff --git a/src/lib/requests/retryRequestDataSilos.ts b/src/lib/requests/retryRequestDataSilos.ts index 478841b5..bdcb9525 100644 --- a/src/lib/requests/retryRequestDataSilos.ts +++ b/src/lib/requests/retryRequestDataSilos.ts @@ -1,16 +1,16 @@ -import { RequestAction, RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestAction, RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequests, fetchRequestDataSilo, makeGraphQLRequest, RETRY_REQUEST_DATA_SILO, -} from "../graphql"; +} from '../graphql'; /** * Retry a set of RequestDataSilos @@ -44,7 +44,7 @@ export async function retryRequestDataSilos({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Pull in the requests @@ -56,8 +56,8 @@ export async function retryRequestDataSilos({ // Notify Transcend logger.info( colors.magenta( - `Retrying requests for Data Silo: "${dataSiloId}", restarting "${allRequests.length}" requests.` - ) + `Retrying requests for Data Silo: "${dataSiloId}", restarting "${allRequests.length}" requests.`, + ), ); let total = 0; @@ -80,7 +80,7 @@ export async function retryRequestDataSilos({ }); } catch (error) { // some requests may not have this data silo connected - if (!error.message.includes("Failed to find RequestDataSilo")) { + if (!error.message.includes('Failed to find RequestDataSilo')) { throw error; } skipped += 1; @@ -89,7 +89,7 @@ export async function retryRequestDataSilos({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -100,8 +100,8 @@ export async function retryRequestDataSilos({ colors.green( `Successfully notified Transcend in "${ totalTime / 1000 - }" seconds for ${total} requests, ${skipped} requests were skipped because data silo was not attached to the request!` - ) + }" seconds for ${total} requests, ${skipped} requests were skipped because data silo was not attached to the request!`, + ), ); return allRequests.length; } diff --git a/src/lib/requests/skipPreflightJobs.ts b/src/lib/requests/skipPreflightJobs.ts index 48922c98..893f6259 100644 --- a/src/lib/requests/skipPreflightJobs.ts +++ b/src/lib/requests/skipPreflightJobs.ts @@ -1,19 +1,19 @@ import { RequestEnricherStatus, RequestStatus, -} from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map, mapSeries } from "../bluebird-replace"; +} from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map, mapSeries } from '../bluebird-replace'; import { buildTranscendGraphQLClient, fetchAllRequestEnrichers, fetchAllRequests, makeGraphQLRequest, SKIP_REQUEST_ENRICHER, -} from "../graphql"; +} from '../graphql'; /** * Given an enricher ID, mark all open request enrichers as skipped @@ -52,16 +52,16 @@ export async function skipPreflightJobs({ // Notify Transcend logger.info( colors.magenta( - `Processing enricher: "${enricherIds.join(",")}" fetched "${ + `Processing enricher: "${enricherIds.join(',')}" fetched "${ requests.length - }" in enriching status.` - ) + }" in enriching status.`, + ), ); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); let total = 0; @@ -81,7 +81,7 @@ export async function skipPreflightJobs({ RequestEnricherStatus.Resolved, RequestEnricherStatus.Skipped, // eslint-disable-next-line @typescript-eslint/no-explicit-any - ].includes(enricher.status as any) + ].includes(enricher.status as any), ); // FIXME @@ -98,7 +98,7 @@ export async function skipPreflightJobs({ } catch (error) { if ( !error.message.includes( - "Client error: Cannot skip Request enricher because it has already completed" + 'Client error: Cannot skip Request enricher because it has already completed', ) ) { throw error; @@ -109,7 +109,7 @@ export async function skipPreflightJobs({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -120,8 +120,8 @@ export async function skipPreflightJobs({ colors.green( `Successfully skipped "${totalSkipped}" for "${ requests.length - }" requests in "${totalTime / 1000}" seconds!` - ) + }" requests in "${totalTime / 1000}" seconds!`, + ), ); return requests.length; } diff --git a/src/lib/requests/skipRequestDataSilos.ts b/src/lib/requests/skipRequestDataSilos.ts index 89d0154c..23000495 100644 --- a/src/lib/requests/skipRequestDataSilos.ts +++ b/src/lib/requests/skipRequestDataSilos.ts @@ -1,15 +1,15 @@ -import { RequestStatus } from "@transcend-io/privacy-types"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { RequestStatus } from '@transcend-io/privacy-types'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, CHANGE_REQUEST_DATA_SILO_STATUS, fetchRequestDataSilos, makeGraphQLRequest, -} from "../graphql"; +} from '../graphql'; /** * Given a data silo ID, mark all open request data silos as skipped @@ -21,7 +21,7 @@ export async function skipRequestDataSilos({ dataSiloId, auth, concurrency = 100, - status = "SKIPPED", + status = 'SKIPPED', transcendUrl = DEFAULT_TRANSCEND_API, requestStatuses = [RequestStatus.Compiling, RequestStatus.Secondary], }: { @@ -30,7 +30,7 @@ export async function skipRequestDataSilos({ /** Data Silo ID to pull down jobs for */ dataSiloId: string; /** Status to set */ - status?: "SKIPPED" | "RESOLVED"; + status?: 'SKIPPED' | 'RESOLVED'; /** Upload concurrency */ concurrency?: number; /** API URL for Transcend backend */ @@ -53,14 +53,14 @@ export async function skipRequestDataSilos({ // Notify Transcend logger.info( colors.magenta( - `Processing data silo: "${dataSiloId}" marking "${requestDataSilos.length}" requests as skipped.` - ) + `Processing data silo: "${dataSiloId}" marking "${requestDataSilos.length}" requests as skipped.`, + ), ); // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); let total = 0; @@ -77,7 +77,7 @@ export async function skipRequestDataSilos({ status, }); } catch (error) { - if (!error.message.includes("Client error: Request must be active:")) { + if (!error.message.includes('Client error: Request must be active:')) { throw error; } } @@ -85,7 +85,7 @@ export async function skipRequestDataSilos({ total += 1; progressBar.update(total); }, - { concurrency } + { concurrency }, ); progressBar.stop(); @@ -96,8 +96,8 @@ export async function skipRequestDataSilos({ colors.green( `Successfully skipped "${requestDataSilos.length}" requests in "${ totalTime / 1000 - }" seconds!` - ) + }" seconds!`, + ), ); return requestDataSilos.length; } diff --git a/src/lib/requests/splitCsvToList.ts b/src/lib/requests/splitCsvToList.ts index 7daa7a9c..b937aa77 100644 --- a/src/lib/requests/splitCsvToList.ts +++ b/src/lib/requests/splitCsvToList.ts @@ -10,7 +10,7 @@ */ export function splitCsvToList(value: string): string[] { return value - .split(",") + .split(',') .map((x) => x.trim()) .filter(Boolean); } diff --git a/src/lib/requests/streamPrivacyRequestFiles.ts b/src/lib/requests/streamPrivacyRequestFiles.ts index 76bb964c..35d20a85 100644 --- a/src/lib/requests/streamPrivacyRequestFiles.ts +++ b/src/lib/requests/streamPrivacyRequestFiles.ts @@ -1,8 +1,8 @@ -import colors from "colors"; -import type { Got } from "got"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; -import { RequestFileMetadata } from "./getFileMetadataForPrivacyRequests"; +import colors from 'colors'; +import type { Got } from 'got'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; +import { RequestFileMetadata } from './getFileMetadataForPrivacyRequests'; /** * This function will take in a set of file metadata for privacy requests @@ -28,7 +28,7 @@ export async function streamPrivacyRequestFiles( onFileDownloaded: (metadata: RequestFileMetadata, stream: Buffer) => void; /** Concurrent downloads at once */ concurrency?: number; - } + }, ): Promise { // Loop over each file await map( @@ -37,7 +37,7 @@ export async function streamPrivacyRequestFiles( try { // Construct the stream await sombra - .get("v1/files", { + .get('v1/files', { searchParams: { downloadKey: metadata.downloadKey, }, @@ -47,26 +47,26 @@ export async function streamPrivacyRequestFiles( onFileDownloaded(metadata, fileResponse); }); } catch (error) { - if (error?.response?.body?.includes("fileMetadata#verify")) { + if (error?.response?.body?.includes('fileMetadata#verify')) { logger.error( colors.red( `Failed to pull file for: ${metadata.fileName} (request:${requestId}) - JWT expired. ` + - "This likely means that the file is no longer available. " + - "Try restarting the request from scratch in Transcend Admin Dashboard. " + - "Skipping the download of this file." - ) + 'This likely means that the file is no longer available. ' + + 'Try restarting the request from scratch in Transcend Admin Dashboard. ' + + 'Skipping the download of this file.', + ), ); return; } throw new Error( `Received an error from server: ${ error?.response?.body || error?.message - }` + }`, ); } }, { concurrency, - } + }, ); } diff --git a/src/lib/requests/submitPrivacyRequest.ts b/src/lib/requests/submitPrivacyRequest.ts index f6d875ae..ffb7cba7 100644 --- a/src/lib/requests/submitPrivacyRequest.ts +++ b/src/lib/requests/submitPrivacyRequest.ts @@ -3,13 +3,13 @@ import { IsoCountrySubdivisionCode, RequestAction, RequestStatus, -} from "@transcend-io/privacy-types"; -import { decodeCodec, valuesOf } from "@transcend-io/type-utils"; -import type { Got } from "got"; -import * as t from "io-ts"; -import { uniq } from "lodash-es"; -import { PrivacyRequestInput } from "./mapCsvRowsToRequestInputs"; -import { ParsedAttributeInput } from "./parseAttributesFromString"; +} from '@transcend-io/privacy-types'; +import { decodeCodec, valuesOf } from '@transcend-io/type-utils'; +import type { Got } from 'got'; +import * as t from 'io-ts'; +import { uniq } from 'lodash-es'; +import { PrivacyRequestInput } from './mapCsvRowsToRequestInputs'; +import { ParsedAttributeInput } from './parseAttributesFromString'; export const PrivacyRequestResponse = t.type({ id: t.string, @@ -27,7 +27,7 @@ export const PrivacyRequestResponse = t.type({ t.type({ attributeKey: t.type({ name: t.string }), name: t.string, - }) + }), ), }); @@ -46,7 +46,7 @@ export async function submitPrivacyRequest( sombra: Got, input: PrivacyRequestInput, { - details = "", + details = '', isTest = false, emailIsVerified = true, skipSendingReceipt = false, @@ -65,14 +65,14 @@ export async function submitPrivacyRequest( details?: string; /** Additional attributes to tag the requests with */ additionalAttributes?: ParsedAttributeInput[]; - } = {} + } = {}, ): Promise { // Merge the per-request attributes with the // global attributes const mergedAttributes = [...additionalAttributes]; for (const attribute of input.attributes || []) { const existing = mergedAttributes.find( - (attribute_) => attribute_.key === attribute.key + (attribute_) => attribute_.key === attribute.key, ); if (existing) { existing.values.push(...attribute.values); @@ -86,7 +86,7 @@ export async function submitPrivacyRequest( let response: unknown; try { response = await sombra - .post("v1/data-subject-request", { + .post('v1/data-subject-request', { json: { type: input.requestType, subject: { @@ -110,8 +110,8 @@ export async function submitPrivacyRequest( country: input.country, } : input.countrySubDivision - ? { country: input.countrySubDivision.split("-")[0] } - : {}), + ? { country: input.countrySubDivision.split('-')[0] } + : {}), ...(input.countrySubDivision ? { countrySubDivision: input.countrySubDivision } : {}), @@ -128,7 +128,7 @@ export async function submitPrivacyRequest( throw new Error( `Received an error from server: ${ error?.response?.body || error?.message - }` + }`, ); } @@ -136,7 +136,7 @@ export async function submitPrivacyRequest( t.type({ request: PrivacyRequestResponse, }), - response + response, ); return requestResponse; } diff --git a/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts b/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts index 4f4f4163..1c1523f9 100644 --- a/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts +++ b/src/lib/requests/tests/mapCsvRowsToRequestInputs.test.ts @@ -1,4 +1,4 @@ -import { describe } from "vitest"; +import { describe } from 'vitest'; // TODO: https://transcend.height.app/T-10772 - mapCsvRowsToRequestInputs test -describe.skip("mapCsvRowsToRequestInputs", () => {}); +describe.skip('mapCsvRowsToRequestInputs', () => {}); diff --git a/src/lib/requests/tests/readCsv.test.ts b/src/lib/requests/tests/readCsv.test.ts index 6a8feef5..b3c3ae84 100644 --- a/src/lib/requests/tests/readCsv.test.ts +++ b/src/lib/requests/tests/readCsv.test.ts @@ -1,54 +1,54 @@ -import { join } from "node:path"; -import * as t from "io-ts"; -import { describe, expect, it } from "vitest"; -import { readCsv } from "../index"; +import { join } from 'node:path'; +import * as t from 'io-ts'; +import { describe, expect, it } from 'vitest'; +import { readCsv } from '../index'; -describe("readCsv", () => { - it("should successfully parse a csv", () => { +describe('readCsv', () => { + it('should successfully parse a csv', () => { expect( - readCsv(join(__dirname, "sample.csv"), t.record(t.string, t.string)) + readCsv(join(__dirname, 'sample.csv'), t.record(t.string, t.string)), ).to.deep.equal([ { - CASL_STATUS: "Undefined", - CONTACT_ID: "949858", - "Data Subject": "Customer", - FIRST_NAME: "Mike", - GDPR_STATUS: "LBI Notice Not Sent", - LAST_NAME: "Farrell", - LINKEDIN_HANDLE: "michaelfarrell", - MOBILE_PHONE: "(860) 906-6012", - OTHER_URL: "", - PERSONAL_EMAIL: "mike@transcend.io", - PHONE: "", - PREFERRED_EMAIL: "mike@transcend.io", - "Primary Company": "Transcend", - "Request Type": "Opt In", - SEARCH_ID: "10749", - SEARCH_NAME: "Transcend - Chief Technology Officer", - SKYPE_HANDLE: "", - STAGE_START_DATE: "2022-11-22", - TITLE: "Chief Technology Officer", - WORK_EMAIL: "", - WORK_PHONE: "", + CASL_STATUS: 'Undefined', + CONTACT_ID: '949858', + 'Data Subject': 'Customer', + FIRST_NAME: 'Mike', + GDPR_STATUS: 'LBI Notice Not Sent', + LAST_NAME: 'Farrell', + LINKEDIN_HANDLE: 'michaelfarrell', + MOBILE_PHONE: '(860) 906-6012', + OTHER_URL: '', + PERSONAL_EMAIL: 'mike@transcend.io', + PHONE: '', + PREFERRED_EMAIL: 'mike@transcend.io', + 'Primary Company': 'Transcend', + 'Request Type': 'Opt In', + SEARCH_ID: '10749', + SEARCH_NAME: 'Transcend - Chief Technology Officer', + SKYPE_HANDLE: '', + STAGE_START_DATE: '2022-11-22', + TITLE: 'Chief Technology Officer', + WORK_EMAIL: '', + WORK_PHONE: '', }, ]); }); - it("throw an error for invalid file", () => { + it('throw an error for invalid file', () => { expect(() => - readCsv(join(__dirname, "sample.csv"), t.type({ notValid: t.string })) - ).to.throw("Failed to decode codec"); + readCsv(join(__dirname, 'sample.csv'), t.type({ notValid: t.string })), + ).to.throw('Failed to decode codec'); }); - it("throw an error for invalid codec", () => { + it('throw an error for invalid codec', () => { expect(() => - readCsv(join(__dirname, "sample.csvs"), t.record(t.string, t.string)) - ).to.throw("ENOENT: no such file or directory, open"); + readCsv(join(__dirname, 'sample.csvs'), t.record(t.string, t.string)), + ).to.throw('ENOENT: no such file or directory, open'); }); - it("throw an error for invalid format", () => { + it('throw an error for invalid format', () => { expect(() => - readCsv(join(__dirname, "readCsv.test.ts"), t.record(t.string, t.string)) + readCsv(join(__dirname, 'readCsv.test.ts'), t.record(t.string, t.string)), ).to.throw(); }); }); diff --git a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts index b897697d..4fc243c5 100644 --- a/src/lib/requests/uploadPrivacyRequestsFromCsv.ts +++ b/src/lib/requests/uploadPrivacyRequestsFromCsv.ts @@ -1,28 +1,28 @@ -import { join } from "node:path"; -import { PersistedState } from "@transcend-io/persisted-state"; -import cliProgress from "cli-progress"; -import colors from "colors"; -import * as t from "io-ts"; -import { uniq } from "lodash-es"; -import { DEFAULT_TRANSCEND_API } from "../../constants"; -import { logger } from "../../logger"; -import { map } from "../bluebird-replace"; +import { join } from 'node:path'; +import { PersistedState } from '@transcend-io/persisted-state'; +import cliProgress from 'cli-progress'; +import colors from 'colors'; +import * as t from 'io-ts'; +import { uniq } from 'lodash-es'; +import { DEFAULT_TRANSCEND_API } from '../../constants'; +import { logger } from '../../logger'; +import { map } from '../bluebird-replace'; import { buildTranscendGraphQLClient, createSombraGotInstance, fetchAllRequestAttributeKeys, -} from "../graphql"; -import { CachedFileState, CachedRequestState } from "./constants"; -import { extractClientError } from "./extractClientError"; -import { filterRows } from "./filterRows"; -import { mapColumnsToAttributes } from "./mapColumnsToAttributes"; -import { mapColumnsToIdentifiers } from "./mapColumnsToIdentifiers"; -import { mapCsvColumnsToApi } from "./mapCsvColumnsToApi"; -import { mapCsvRowsToRequestInputs } from "./mapCsvRowsToRequestInputs"; -import { mapRequestEnumValues } from "./mapRequestEnumValues"; -import { parseAttributesFromString } from "./parseAttributesFromString"; -import { readCsv } from "./readCsv"; -import { submitPrivacyRequest } from "./submitPrivacyRequest"; +} from '../graphql'; +import { CachedFileState, CachedRequestState } from './constants'; +import { extractClientError } from './extractClientError'; +import { filterRows } from './filterRows'; +import { mapColumnsToAttributes } from './mapColumnsToAttributes'; +import { mapColumnsToIdentifiers } from './mapColumnsToIdentifiers'; +import { mapCsvColumnsToApi } from './mapCsvColumnsToApi'; +import { mapCsvRowsToRequestInputs } from './mapCsvRowsToRequestInputs'; +import { mapRequestEnumValues } from './mapRequestEnumValues'; +import { parseAttributesFromString } from './parseAttributesFromString'; +import { readCsv } from './readCsv'; +import { submitPrivacyRequest } from './submitPrivacyRequest'; /** * Upload a set of privacy requests from CSV @@ -36,7 +36,7 @@ export async function uploadPrivacyRequestsFromCsv({ auth, sombraAuth, concurrency = 100, - defaultPhoneCountryCode = "1", // USA + defaultPhoneCountryCode = '1', // USA transcendUrl = DEFAULT_TRANSCEND_API, attributes = [], emailIsVerified = true, @@ -85,7 +85,7 @@ export async function uploadPrivacyRequestsFromCsv({ // create a new progress bar instance and use shades_classic theme const progressBar = new cliProgress.SingleBar( {}, - cliProgress.Presets.shades_classic + cliProgress.Presets.shades_classic, ); // Parse out the extra attributes to apply to all requests uploaded @@ -109,8 +109,8 @@ export async function uploadPrivacyRequestsFromCsv({ const requestCacheFile = join( requestReceiptFolder, `tr-request-upload-${new Date().toISOString()}-${file - .split("/") - .pop()}`.replace(".csv", ".json") + .split('/') + .pop()}`.replace('.csv', '.json'), ); const requestState = new PersistedState( requestCacheFile, @@ -119,7 +119,7 @@ export async function uploadPrivacyRequestsFromCsv({ successfulRequests: [], duplicateRequests: [], failingRequests: [], - } + }, ); // Create sombra instance to communicate with @@ -132,13 +132,13 @@ export async function uploadPrivacyRequestsFromCsv({ // Log out an example request if (requestsList.length === 0) { throw new Error( - "No Requests found in list! Ensure the first row of the CSV is a header and the rest are requests." + 'No Requests found in list! Ensure the first row of the CSV is a header and the rest are requests.', ); } if (debug) { const firstRequest = requestsList[0]; logger.info( - colors.magenta(`First request: ${JSON.stringify(firstRequest, null, 2)}`) + colors.magenta(`First request: ${JSON.stringify(firstRequest, null, 2)}`), ); } // Determine what rows in the CSV should be imported @@ -156,13 +156,13 @@ export async function uploadPrivacyRequestsFromCsv({ const identifierNameMap = await mapColumnsToIdentifiers( client, columnNames, - state + state, ); const attributeNameMap = await mapColumnsToAttributes( client, columnNames, state, - requestAttributeKeys + requestAttributeKeys, ); await mapRequestEnumValues(client, filteredRequestList, { state, @@ -198,16 +198,16 @@ export async function uploadPrivacyRequestsFromCsv({ `[${ind + 1}/${requestInputs.length}] Importing: ${JSON.stringify( requestInput, null, - 2 - )}` - ) + 2, + )}`, + ), ); } // Skip on dry run if (dryRun) { logger.info( - colors.magenta("Bailing out on dry run because dryRun is set") + colors.magenta('Bailing out on dry run because dryRun is set'), ); return; } @@ -221,14 +221,14 @@ export async function uploadPrivacyRequestsFromCsv({ details: `Uploaded by Transcend Cli: "tr-request-upload" : ${JSON.stringify( rawRow, null, - 2 + 2, )}`, isTest, emailIsVerified, skipSendingReceipt, isSilent, additionalAttributes: parsedAttributes, - } + }, ); // Log success @@ -237,20 +237,20 @@ export async function uploadPrivacyRequestsFromCsv({ colors.green( `[${ind + 1}/${ requestInputs.length - }] Successfully submitted the test data subject request: "${requestLogId}"` - ) + }] Successfully submitted the test data subject request: "${requestLogId}"`, + ), ); logger.info( colors.green( `[${ind + 1}/${requestInputs.length}] View it at: "${ requestResponse.link - }"` - ) + }"`, + ), ); } // Cache successful upload - const successfulRequests = requestState.getValue("successfulRequests"); + const successfulRequests = requestState.getValue('successfulRequests'); successfulRequests.push({ id: requestResponse.id, link: requestResponse.link, @@ -258,51 +258,51 @@ export async function uploadPrivacyRequestsFromCsv({ coreIdentifier: requestResponse.coreIdentifier, attemptedAt: new Date().toISOString(), }); - await requestState.setValue(successfulRequests, "successfulRequests"); + await requestState.setValue(successfulRequests, 'successfulRequests'); } catch (error) { const message = `${error.message} - ${JSON.stringify( error.response?.body, null, - 2 + 2, )}`; const clientError = extractClientError(message); if ( - clientError === "Client error: You have already made this request." + clientError === 'Client error: You have already made this request.' ) { if (debug) { logger.info( colors.yellow( `[${ind + 1}/${ requestInputs.length - }] Skipping request as it is a duplicate` - ) + }] Skipping request as it is a duplicate`, + ), ); } - const duplicateRequests = requestState.getValue("duplicateRequests"); + const duplicateRequests = requestState.getValue('duplicateRequests'); duplicateRequests.push({ coreIdentifier: requestInput.coreIdentifier, rowIndex: ind, attemptedAt: new Date().toISOString(), }); - await requestState.setValue(duplicateRequests, "duplicateRequests"); + await requestState.setValue(duplicateRequests, 'duplicateRequests'); } else { - const failingRequests = requestState.getValue("failingRequests"); + const failingRequests = requestState.getValue('failingRequests'); failingRequests.push({ ...requestInput, rowIndex: ind, error: clientError || message, attemptedAt: new Date().toISOString(), }); - await requestState.setValue(failingRequests, "failingRequests"); + await requestState.setValue(failingRequests, 'failingRequests'); if (debug) { logger.error(colors.red(clientError || message)); logger.error( colors.red( `[${ind + 1}/${ requestInputs.length - }] Failed to submit request for: "${requestLogId}"` - ) + }] Failed to submit request for: "${requestLogId}"`, + ), ); } } @@ -315,7 +315,7 @@ export async function uploadPrivacyRequestsFromCsv({ }, { concurrency, - } + }, ); progressBar.stop(); @@ -324,30 +324,30 @@ export async function uploadPrivacyRequestsFromCsv({ // Log completion time logger.info( - colors.green(`Completed upload in "${totalTime / 1000}" seconds.`) + colors.green(`Completed upload in "${totalTime / 1000}" seconds.`), ); // Log duplicates - if (requestState.getValue("duplicateRequests").length > 0) { + if (requestState.getValue('duplicateRequests').length > 0) { logger.info( colors.yellow( `Encountered "${ - requestState.getValue("duplicateRequests").length + requestState.getValue('duplicateRequests').length }" duplicate requests. ` + - `See "${requestCacheFile}" to review the core identifiers for these requests.` - ) + `See "${requestCacheFile}" to review the core identifiers for these requests.`, + ), ); } // Log errors - if (requestState.getValue("failingRequests").length > 0) { + if (requestState.getValue('failingRequests').length > 0) { logger.error( colors.red( `Encountered "${ - requestState.getValue("failingRequests").length + requestState.getValue('failingRequests').length }" errors. ` + - `See "${requestCacheFile}" to review the error messages and inputs.` - ) + `See "${requestCacheFile}" to review the error messages and inputs.`, + ), ); process.exit(1); } diff --git a/src/lib/tests/findCodePackagesInFolder.test.ts b/src/lib/tests/findCodePackagesInFolder.test.ts index 0cb4d54f..078f92e3 100644 --- a/src/lib/tests/findCodePackagesInFolder.test.ts +++ b/src/lib/tests/findCodePackagesInFolder.test.ts @@ -1,699 +1,699 @@ -import { join } from "node:path"; -import { describe, expect, it } from "vitest"; -import type { CodePackageInput } from "../../codecs"; -import { findCodePackagesInFolder } from "../code-scanning/findCodePackagesInFolder"; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import type { CodePackageInput } from '../../codecs'; +import { findCodePackagesInFolder } from '../code-scanning/findCodePackagesInFolder'; const expected: CodePackageInput[] = [ { - name: "YourAppTargetName", - type: "COCOA_PODS", + name: 'YourAppTargetName', + type: 'COCOA_PODS', softwareDevelopmentKits: [ { - name: "Braze-iOS-SDK", + name: 'Braze-iOS-SDK', version: undefined, }, { - name: "Branch", + name: 'Branch', version: undefined, }, { - name: "Firebase/Analytics", + name: 'Firebase/Analytics', version: undefined, }, { - name: "Mixpanel", + name: 'Mixpanel', version: undefined, }, { - name: "Amplitude-iOS", - version: "8.0", + name: 'Amplitude-iOS', + version: '8.0', }, { - name: "Google-Mobile-Ads-SDK", + name: 'Google-Mobile-Ads-SDK', version: undefined, }, { - name: "FacebookAdsSDK", + name: 'FacebookAdsSDK', version: undefined, }, { - name: "MoPub-SDK", + name: 'MoPub-SDK', version: undefined, }, { - name: "Alamofire", - version: "5.2", + name: 'Alamofire', + version: '5.2', }, { - name: "SDWebImage", + name: 'SDWebImage', version: undefined, }, { - name: "AppsFlyerFramework", + name: 'AppsFlyerFramework', version: undefined, }, { - name: "Adjust", + name: 'Adjust', version: undefined, }, { - name: "Flurry-iOS-SDK/FlurrySDK", + name: 'Flurry-iOS-SDK/FlurrySDK', version: undefined, }, ], - relativePath: "test-cocoa-pods/Podfile", - repositoryName: "transcend-io/cli", + relativePath: 'test-cocoa-pods/Podfile', + repositoryName: 'transcend-io/cli', }, { - name: "ExampleBootstrap", - type: "COCOA_PODS", + name: 'ExampleBootstrap', + type: 'COCOA_PODS', softwareDevelopmentKits: [ { - name: "ExampleLib", + name: 'ExampleLib', version: undefined, }, { - name: "AppsFlyerFramework", + name: 'AppsFlyerFramework', version: undefined, }, { - name: "Adjust", + name: 'Adjust', version: undefined, }, { - name: "Flurry-iOS-SDK/FlurrySDK", + name: 'Flurry-iOS-SDK/FlurrySDK', version: undefined, }, ], - relativePath: "test-requirements-txt/nested-cocoapods/Podfile", - repositoryName: "transcend-io/cli", + relativePath: 'test-requirements-txt/nested-cocoapods/Podfile', + repositoryName: 'transcend-io/cli', }, { - name: "ExampleBootstrapTests", - type: "COCOA_PODS", + name: 'ExampleBootstrapTests', + type: 'COCOA_PODS', softwareDevelopmentKits: [ { - name: "ExampleLib", + name: 'ExampleLib', version: undefined, }, { - name: "Braze-iOS-SDK", + name: 'Braze-iOS-SDK', version: undefined, }, { - name: "Branch", + name: 'Branch', version: undefined, }, { - name: "Firebase/Analytics", + name: 'Firebase/Analytics', version: undefined, }, { - name: "Mixpanel", + name: 'Mixpanel', version: undefined, }, { - name: "Amplitude-iOS", - version: "8.0", + name: 'Amplitude-iOS', + version: '8.0', }, ], - relativePath: "test-requirements-txt/nested-cocoapods/Podfile", - repositoryName: "transcend-io/cli", + relativePath: 'test-requirements-txt/nested-cocoapods/Podfile', + repositoryName: 'transcend-io/cli', }, { - name: "Acme", - relativePath: "test-requirements-txt/nested-cocoapods-2/Podfile", - repositoryName: "transcend-io/cli", + name: 'Acme', + relativePath: 'test-requirements-txt/nested-cocoapods-2/Podfile', + repositoryName: 'transcend-io/cli', softwareDevelopmentKits: [ { - name: "RZVinyl", + name: 'RZVinyl', version: undefined, }, { - name: "RZTransitions", + name: 'RZTransitions', version: undefined, }, { - name: "SDWebImage", + name: 'SDWebImage', version: undefined, }, { - name: "SwiftLint", + name: 'SwiftLint', version: undefined, }, ], - type: "COCOA_PODS", + type: 'COCOA_PODS', }, { - name: "AcmeTests", - type: "COCOA_PODS", + name: 'AcmeTests', + type: 'COCOA_PODS', softwareDevelopmentKits: [ { - name: "RZVinyl", + name: 'RZVinyl', version: undefined, }, { - name: "iOSSnapshotTestCase", + name: 'iOSSnapshotTestCase', version: undefined, }, { - name: "SnapshotTesting", - version: "1.8.1", + name: 'SnapshotTesting', + version: '1.8.1', }, ], - relativePath: "test-requirements-txt/nested-cocoapods-2/Podfile", - repositoryName: "transcend-io/cli", + relativePath: 'test-requirements-txt/nested-cocoapods-2/Podfile', + repositoryName: 'transcend-io/cli', }, { - name: "NotificationServiceExtension", - relativePath: "test-requirements-txt/nested-cocoapods-2/Podfile", - repositoryName: "transcend-io/cli", + name: 'NotificationServiceExtension', + relativePath: 'test-requirements-txt/nested-cocoapods-2/Podfile', + repositoryName: 'transcend-io/cli', softwareDevelopmentKits: [], - type: "COCOA_PODS", + type: 'COCOA_PODS', }, { - name: "com.yourcompany.yourapp", + name: 'com.yourcompany.yourapp', softwareDevelopmentKits: [ { - name: "androidx.appcompat", - version: "1.2.0", + name: 'androidx.appcompat', + version: '1.2.0', }, { - name: "androidx.constraintlayout", - version: "2.0.4", + name: 'androidx.constraintlayout', + version: '2.0.4', }, { - name: "com.appboy", - version: "14.0.0", + name: 'com.appboy', + version: '14.0.0', }, { - name: "io.branch.sdk.android", - version: "5.0.1", + name: 'io.branch.sdk.android', + version: '5.0.1', }, { - name: "com.google.firebase", - version: "18.0.0", + name: 'com.google.firebase', + version: '18.0.0', }, { - name: "com.google.android.gms", - version: "19.7.0", + name: 'com.google.android.gms', + version: '19.7.0', }, { - name: "com.facebook.android", - version: "6.2.0", + name: 'com.facebook.android', + version: '6.2.0', }, { - name: "com.mixpanel.android", - version: "5.8.7", + name: 'com.mixpanel.android', + version: '5.8.7', }, { - name: "com.amplitude", - version: "2.30.0", + name: 'com.amplitude', + version: '2.30.0', }, { - name: "com.squareup.retrofit2", - version: "2.9.0", + name: 'com.squareup.retrofit2', + version: '2.9.0', }, { - name: "com.squareup.okhttp3", - version: "4.9.0", + name: 'com.squareup.okhttp3', + version: '4.9.0', }, { - name: "com.squareup.picasso", - version: "2.71828", + name: 'com.squareup.picasso', + version: '2.71828', }, { - name: "org.eclipse.jdt.core", - version: "3.28.0", + name: 'org.eclipse.jdt.core', + version: '3.28.0', }, { - name: "com.android.application", + name: 'com.android.application', version: undefined, }, { - name: "com.google.gms.google-services", + name: 'com.google.gms.google-services', version: undefined, }, ], - relativePath: "test-gradle/build.gradle", - type: "GRADLE", - repositoryName: "transcend-io/cli", + relativePath: 'test-gradle/build.gradle', + type: 'GRADLE', + repositoryName: 'transcend-io/cli', }, { - name: "@test-example/test", - description: "Example npm package.", + name: '@test-example/test', + description: 'Example npm package.', softwareDevelopmentKits: [ { - name: "dd-trace", - version: "2.45.1", + name: 'dd-trace', + version: '2.45.1', }, { - name: "fast-csv", - version: "^4.3.6", + name: 'fast-csv', + version: '^4.3.6', }, { - name: "sequelize", - version: "^6.37.3", + name: 'sequelize', + version: '^6.37.3', }, { - name: "sequelize-mock", - version: "^0.10.2", + name: 'sequelize-mock', + version: '^0.10.2', }, { isDevDependency: true, - name: "@types/sequelize", - version: "^4.28.20", + name: '@types/sequelize', + version: '^4.28.20', }, { - name: "typescript", - version: "^5.0.4", + name: 'typescript', + version: '^5.0.4', isDevDependency: true, }, ], - relativePath: "test-package-json/package.json", - type: "PACKAGE_JSON", - repositoryName: "transcend-io/cli", + relativePath: 'test-package-json/package.json', + type: 'PACKAGE_JSON', + repositoryName: 'transcend-io/cli', }, { - name: "@test-example/nested-test", - description: "Example npm nested package.", + name: '@test-example/nested-test', + description: 'Example npm nested package.', softwareDevelopmentKits: [ { - name: "dd-trace", - version: "2.45.1", + name: 'dd-trace', + version: '2.45.1', }, { - name: "fast-csv", - version: "^4.3.6", + name: 'fast-csv', + version: '^4.3.6', }, { - name: "typescript", - version: "^5.0.4", + name: 'typescript', + version: '^5.0.4', isDevDependency: true, }, ], - relativePath: "test-gradle/test-nested-package-json/package.json", - type: "PACKAGE_JSON", - repositoryName: "transcend-io/cli", + relativePath: 'test-gradle/test-nested-package-json/package.json', + type: 'PACKAGE_JSON', + repositoryName: 'transcend-io/cli', }, { - name: "test_requirements_txt", - type: "REQUIREMENTS_TXT", - description: "A sample Python package", + name: 'test_requirements_txt', + type: 'REQUIREMENTS_TXT', + description: 'A sample Python package', softwareDevelopmentKits: [ { - name: "pyarrow", - version: "14.0.1", + name: 'pyarrow', + version: '14.0.1', }, { - name: "cryptography", - version: "41.0.6", + name: 'cryptography', + version: '41.0.6', }, { - name: "Flask", - version: "2.2.5", + name: 'Flask', + version: '2.2.5', }, { - name: "cachetools", - version: "5.3.0", + name: 'cachetools', + version: '5.3.0', }, ], - relativePath: "test-requirements-txt/requirements.txt", - repositoryName: "transcend-io/cli", + relativePath: 'test-requirements-txt/requirements.txt', + repositoryName: 'transcend-io/cli', }, { - name: "test-nested-requirements-txt", - type: "REQUIREMENTS_TXT", + name: 'test-nested-requirements-txt', + type: 'REQUIREMENTS_TXT', description: undefined, softwareDevelopmentKits: [ { - name: "pyarrow", - version: "14.0.1", + name: 'pyarrow', + version: '14.0.1', }, { - name: "pandas", - version: "2.0.3", + name: 'pandas', + version: '2.0.3', }, ], relativePath: - "test-package-json/test-nested-requirements-txt/requirements.txt", - repositoryName: "transcend-io/cli", + 'test-package-json/test-nested-requirements-txt/requirements.txt', + repositoryName: 'transcend-io/cli', }, { - name: "test-gemfile", - type: "GEMFILE", + name: 'test-gemfile', + type: 'GEMFILE', description: undefined, softwareDevelopmentKits: [ { - name: "rails", - version: "~> 6.1.4", + name: 'rails', + version: '~> 6.1.4', }, { - name: "ahoy_matey", + name: 'ahoy_matey', version: undefined, }, { - name: "rack-tracker", + name: 'rack-tracker', version: undefined, }, { - name: "adroll", + name: 'adroll', version: undefined, }, { - name: "google-ads-googleads", + name: 'google-ads-googleads', version: undefined, }, { - name: "facebookads", + name: 'facebookads', version: undefined, }, { - name: "devise", + name: 'devise', version: undefined, }, { - name: "impressionist", + name: 'impressionist', version: undefined, }, { - name: "sidekiq", + name: 'sidekiq', version: undefined, }, { - name: "sidekiq-cron", - version: "~> 1.2", + name: 'sidekiq-cron', + version: '~> 1.2', }, { - name: "byebug", + name: 'byebug', version: undefined, }, { - name: "listen", - version: "~> 3.3", + name: 'listen', + version: '~> 3.3', }, { - name: "capybara", - version: ">= 2.15", + name: 'capybara', + version: '>= 2.15', }, { - name: "selenium-webdriver", + name: 'selenium-webdriver', version: undefined, }, { - name: "webdrivers", + name: 'webdrivers', version: undefined, }, { - name: "bundler-audit", + name: 'bundler-audit', version: undefined, }, ], - relativePath: "test-gemfile/Gemfile", - repositoryName: "transcend-io/cli", + relativePath: 'test-gemfile/Gemfile', + repositoryName: 'transcend-io/cli', }, { - name: "test-nested-gemfile", - type: "GEMFILE", + name: 'test-nested-gemfile', + type: 'GEMFILE', softwareDevelopmentKits: [ { - name: "rails", - version: "~> 6.1.4", + name: 'rails', + version: '~> 6.1.4', }, { - name: "ahoy_matey", + name: 'ahoy_matey', version: undefined, }, { - name: "rack-tracker", + name: 'rack-tracker', version: undefined, }, { - name: "adroll", + name: 'adroll', version: undefined, }, { - name: "google-ads-googleads", + name: 'google-ads-googleads', version: undefined, }, { - name: "facebookads", + name: 'facebookads', version: undefined, }, { - name: "byebug", + name: 'byebug', version: undefined, }, { - name: "listen", - version: "~> 3.3", + name: 'listen', + version: '~> 3.3', }, { - name: "capybara", - version: ">= 2.15", + name: 'capybara', + version: '>= 2.15', }, { - name: "selenium-webdriver", + name: 'selenium-webdriver', version: undefined, }, { - name: "webdrivers", + name: 'webdrivers', version: undefined, }, { - name: "bundler-audit", + name: 'bundler-audit', version: undefined, }, ], description: undefined, - relativePath: "test-gradle/test-nested-gemfile/Gemfile", - repositoryName: "transcend-io/cli", + relativePath: 'test-gradle/test-nested-gemfile/Gemfile', + repositoryName: 'transcend-io/cli', }, { - name: "example", - description: "test example app", - type: "PUBSPEC", + name: 'example', + description: 'test example app', + type: 'PUBSPEC', softwareDevelopmentKits: [ { - name: "flutter", - version: "flutter", + name: 'flutter', + version: 'flutter', }, { - name: "flutter_localizations", - version: "flutter", + name: 'flutter_localizations', + version: 'flutter', }, { - name: "firebase_core", - version: "2.16.0", + name: 'firebase_core', + version: '2.16.0', }, { - name: "firebase_analytics", - version: "10.5.0", + name: 'firebase_analytics', + version: '10.5.0', }, { - name: "firebase_crashlytics", - version: "3.3.6", + name: 'firebase_crashlytics', + version: '3.3.6', }, { - name: "video_player", - version: "2.6.1", + name: 'video_player', + version: '2.6.1', }, { - name: "appsflyer_sdk", - version: "6.12.2", + name: 'appsflyer_sdk', + version: '6.12.2', }, { - name: "isolate", - version: "2.1.1", + name: 'isolate', + version: '2.1.1', }, { - name: "custom_platform_device_id", - version: "1.0.8", + name: 'custom_platform_device_id', + version: '1.0.8', }, { - name: "image_editor", - version: "1.3.0", + name: 'image_editor', + version: '1.3.0', }, { - name: "firebase_remote_config", - version: "4.2.6", + name: 'firebase_remote_config', + version: '4.2.6', }, { - name: "intercom_flutter", - version: "7.8.4", + name: 'intercom_flutter', + version: '7.8.4', }, { - name: "dismissible_page", - version: "1.0.2", + name: 'dismissible_page', + version: '1.0.2', }, { - name: "extended_text", - version: "11.1.0", + name: 'extended_text', + version: '11.1.0', }, { - name: "recaptcha_enterprise_flutter", - version: "18.3.0", + name: 'recaptcha_enterprise_flutter', + version: '18.3.0', }, { - name: "flutter_test", - version: "flutter", + name: 'flutter_test', + version: 'flutter', isDevDependency: true, }, { - name: "test", - version: "1.24.3", + name: 'test', + version: '1.24.3', isDevDependency: true, }, { - name: "lints", - version: "3.0.0", + name: 'lints', + version: '3.0.0', isDevDependency: true, }, { - name: "mocktail", - version: "1.0.1", + name: 'mocktail', + version: '1.0.1', isDevDependency: true, }, ], - relativePath: "test-pubspec/pubspec.yml", - repositoryName: "transcend-io/cli", + relativePath: 'test-pubspec/pubspec.yml', + repositoryName: 'transcend-io/cli', }, { - name: "composer/example", - description: "Example app", + name: 'composer/example', + description: 'Example app', softwareDevelopmentKits: [ { - name: "php", - version: "^7.2.5 || ^8.0", + name: 'php', + version: '^7.2.5 || ^8.0', }, { - name: "composer/ca-bundle", - version: "^1.5", + name: 'composer/ca-bundle', + version: '^1.5', }, { - name: "composer/class-map-generator", - version: "^1.3.3", + name: 'composer/class-map-generator', + version: '^1.3.3', }, { - name: "composer/metadata-minifier", - version: "^1.0", + name: 'composer/metadata-minifier', + version: '^1.0', }, { - name: "composer/semver", - version: "^3.3", + name: 'composer/semver', + version: '^3.3', }, { - name: "composer/spdx-licenses", - version: "^1.5.7", + name: 'composer/spdx-licenses', + version: '^1.5.7', }, { - name: "composer/xdebug-handler", - version: "^2.0.2 || ^3.0.3", + name: 'composer/xdebug-handler', + version: '^2.0.2 || ^3.0.3', }, { - name: "justinrainbow/json-schema", - version: "^5.3", + name: 'justinrainbow/json-schema', + version: '^5.3', }, { - name: "psr/log", - version: "^1.0 || ^2.0 || ^3.0", + name: 'psr/log', + version: '^1.0 || ^2.0 || ^3.0', }, { - name: "seld/jsonlint", - version: "^1.4", + name: 'seld/jsonlint', + version: '^1.4', }, { - name: "seld/phar-utils", - version: "^1.2", + name: 'seld/phar-utils', + version: '^1.2', }, { - name: "symfony/console", - version: "^5.4.35 || ^6.3.12 || ^7.0.3", + name: 'symfony/console', + version: '^5.4.35 || ^6.3.12 || ^7.0.3', }, { - name: "symfony/filesystem", - version: "^5.4.35 || ^6.3.12 || ^7.0.3", + name: 'symfony/filesystem', + version: '^5.4.35 || ^6.3.12 || ^7.0.3', }, { - name: "symfony/finder", - version: "^5.4.35 || ^6.3.12 || ^7.0.3", + name: 'symfony/finder', + version: '^5.4.35 || ^6.3.12 || ^7.0.3', }, { - name: "symfony/process", - version: "^5.4.35 || ^6.3.12 || ^7.0.3", + name: 'symfony/process', + version: '^5.4.35 || ^6.3.12 || ^7.0.3', }, { - name: "react/promise", - version: "^3.2", + name: 'react/promise', + version: '^3.2', }, { - name: "composer/pcre", - version: "^2.2 || ^3.2", + name: 'composer/pcre', + version: '^2.2 || ^3.2', }, { - name: "symfony/polyfill-php73", - version: "^1.24", + name: 'symfony/polyfill-php73', + version: '^1.24', }, { - name: "symfony/polyfill-php80", - version: "^1.24", + name: 'symfony/polyfill-php80', + version: '^1.24', }, { - name: "symfony/polyfill-php81", - version: "^1.24", + name: 'symfony/polyfill-php81', + version: '^1.24', }, { - name: "seld/signal-handler", - version: "^2.0", + name: 'seld/signal-handler', + version: '^2.0', }, { - name: "symfony/phpunit-bridge", - version: "^6.4.3 || ^7.0.1", + name: 'symfony/phpunit-bridge', + version: '^6.4.3 || ^7.0.1', isDevDependency: true, }, { - name: "phpstan/phpstan", - version: "^1.11.8", + name: 'phpstan/phpstan', + version: '^1.11.8', isDevDependency: true, }, { - name: "phpstan/phpstan-phpunit", - version: "^1.4.0", + name: 'phpstan/phpstan-phpunit', + version: '^1.4.0', isDevDependency: true, }, { - name: "phpstan/phpstan-deprecation-rules", - version: "^1.2.0", + name: 'phpstan/phpstan-deprecation-rules', + version: '^1.2.0', isDevDependency: true, }, { - name: "phpstan/phpstan-strict-rules", - version: "^1.6.0", + name: 'phpstan/phpstan-strict-rules', + version: '^1.6.0', isDevDependency: true, }, { - name: "phpstan/phpstan-symfony", - version: "^1.4.0", + name: 'phpstan/phpstan-symfony', + version: '^1.4.0', isDevDependency: true, }, ], - relativePath: "test-php/composer.json", - type: "COMPOSER_JSON", - repositoryName: "transcend-io/cli", + relativePath: 'test-php/composer.json', + type: 'COMPOSER_JSON', + repositoryName: 'transcend-io/cli', }, { - name: "test-swift", - type: "SWIFT", - relativePath: "test-swift/Package.resolved", - repositoryName: "transcend-io/cli", + name: 'test-swift', + type: 'SWIFT', + relativePath: 'test-swift/Package.resolved', + repositoryName: 'transcend-io/cli', softwareDevelopmentKits: [ { - name: "alamofire", - version: "5.8.1", + name: 'alamofire', + version: '5.8.1', }, { - name: "swift-numerics", - version: "1.0.2", + name: 'swift-numerics', + version: '1.0.2', }, ], }, @@ -706,24 +706,24 @@ const expected: CodePackageInput[] = [ * @returns Sorted code packages */ function sortCodePackages( - codePackages: CodePackageInput[] + codePackages: CodePackageInput[], ): CodePackageInput[] { return codePackages .sort((a, b) => a.name.localeCompare(b.name)) .map((c) => ({ ...c, softwareDevelopmentKits: c.softwareDevelopmentKits?.sort((a, b) => - a.name.localeCompare(b.name) + a.name.localeCompare(b.name), ), })); } // not easy to test this but can uncomment to run against current commit -describe("findCodePackagesInFolder", () => { - it("should remove links", async () => { +describe('findCodePackagesInFolder', () => { + it('should remove links', async () => { const result = await findCodePackagesInFolder({ - repositoryName: "transcend-io/cli", - scanPath: join(__dirname, "../../../examples/code-scanning"), + repositoryName: 'transcend-io/cli', + scanPath: join(__dirname, '../../../examples/code-scanning'), }); expect(sortCodePackages(result)).to.deep.equal(sortCodePackages(expected)); }); diff --git a/src/lib/tests/getGitFilesThatChanged.test.ts b/src/lib/tests/getGitFilesThatChanged.test.ts index a90c7116..837126de 100644 --- a/src/lib/tests/getGitFilesThatChanged.test.ts +++ b/src/lib/tests/getGitFilesThatChanged.test.ts @@ -1,22 +1,22 @@ -import { join } from "node:path"; -import { describe, expect, it } from "vitest"; -import { getGitFilesThatChanged } from "../ai/getGitFilesThatChanged"; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { getGitFilesThatChanged } from '../ai/getGitFilesThatChanged'; // not easy to test this but can uncomment to run against current commit -describe.skip("getGitFilesThatChanged", () => { - it("should remove links", () => { +describe.skip('getGitFilesThatChanged', () => { + it('should remove links', () => { expect( getGitFilesThatChanged({ - baseBranch: "main", - githubRepo: "https://github.com/transcend-io/cli.git", - rootDirectory: join(__dirname, "../../"), - }) + baseBranch: 'main', + githubRepo: 'https://github.com/transcend-io/cli.git', + rootDirectory: join(__dirname, '../../'), + }), ).to.deep.equal({ changedFiles: [ - "package.json", - "src/ai/TranscendAiPrompt.ts", - "src/cli-analyze-pull-request.ts", - "src/tests/TranscendAiPrompt.test.ts", + 'package.json', + 'src/ai/TranscendAiPrompt.ts', + 'src/cli-analyze-pull-request.ts', + 'src/tests/TranscendAiPrompt.test.ts', ], fileDiffs: {}, }); diff --git a/src/lib/tests/readTranscendYaml.test.ts b/src/lib/tests/readTranscendYaml.test.ts index 2c632f87..050ae708 100644 --- a/src/lib/tests/readTranscendYaml.test.ts +++ b/src/lib/tests/readTranscendYaml.test.ts @@ -1,18 +1,18 @@ -import { join } from "node:path"; -import { describe, expect, it } from "vitest"; -import { readTranscendYaml } from "../../index"; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { readTranscendYaml } from '../../index'; -const EXAMPLE_DIR = join(__dirname, "..", "..", "..", "examples"); +const EXAMPLE_DIR = join(__dirname, '..', '..', '..', 'examples'); -describe("readTranscendYaml", () => { - it("simple.yml should pass the codec validation for TranscendInput", () => { +describe('readTranscendYaml', () => { + it('simple.yml should pass the codec validation for TranscendInput', () => { expect(() => - readTranscendYaml(join(EXAMPLE_DIR, "simple.yml")) + readTranscendYaml(join(EXAMPLE_DIR, 'simple.yml')), ).to.not.throw(); }); - it("invalid.yml should fail the codec validation for TranscendInput", () => { - expect(() => readTranscendYaml(join(EXAMPLE_DIR, "invalid.yml"))).to + it('invalid.yml should fail the codec validation for TranscendInput', () => { + expect(() => readTranscendYaml(join(EXAMPLE_DIR, 'invalid.yml'))).to .throw(` ".enrichers.0.0.title expected type 'string'", ".enrichers.1.0.output-identifiers expected type 'Array'", ".data-silos.0.0.title expected type 'string'", @@ -22,26 +22,26 @@ describe("readTranscendYaml", () => { ".data-silos.1.1.disabled expected type 'boolean'"`); }); - it("multi-instance.yml should fail when no variables are provided", () => { + it('multi-instance.yml should fail when no variables are provided', () => { expect(() => - readTranscendYaml(join(EXAMPLE_DIR, "multi-instance.yml")) - ).to.throw("Found variable that was not set: domain"); + readTranscendYaml(join(EXAMPLE_DIR, 'multi-instance.yml')), + ).to.throw('Found variable that was not set: domain'); }); - it("multi-instance.yml should be successful when variables are provided", () => { - const result = readTranscendYaml(join(EXAMPLE_DIR, "multi-instance.yml"), { - domain: "acme.com", - stage: "Staging", + it('multi-instance.yml should be successful when variables are provided', () => { + const result = readTranscendYaml(join(EXAMPLE_DIR, 'multi-instance.yml'), { + domain: 'acme.com', + stage: 'Staging', }); expect(result.enrichers![0].url).to.equal( - "https://example.acme.com/transcend-enrichment-webhook" + 'https://example.acme.com/transcend-enrichment-webhook', ); expect(result.enrichers![1].url).to.equal( - "https://example.acme.com/transcend-fraud-check" + 'https://example.acme.com/transcend-fraud-check', ); - expect(result["data-silos"]![0].description).to.equal( - "The mega-warehouse that contains a copy over all SQL backed databases - Staging" + expect(result['data-silos']![0].description).to.equal( + 'The mega-warehouse that contains a copy over all SQL backed databases - Staging', ); }); });