diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7f3e0b5..9653f97a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,6 +24,7 @@ repos: - id: check-json exclude: (.vscode/|tsconfig.json|.devcontainer/) - id: detect-private-key + exclude: (examples/chats/) - id: detect-aws-credentials args: [--allow-missing-credentials] - id: check-yaml diff --git a/.vscode/settings.json b/.vscode/settings.json index e48e3e8a..4bc44644 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -61,6 +61,7 @@ "pipeable", "preact", "pubspec", + "redactions", "Requirize", "respawned", "respawning", diff --git a/README.md b/README.md index 23c08748..e69de29b 100644 --- a/README.md +++ b/README.md @@ -1,3381 +0,0 @@ -# Transcend CLI - - - - -## Table of Contents - -- [Changelog](#changelog) -- [Overview](#overview) -- [Installation](#installation) -- [transcend.yml](#transcendyml) -- [Usage](#usage) - - [`transcend request approve`](#transcend-request-approve) - - [Examples](#examples) - - [`transcend request upload`](#transcend-request-upload) - - [Examples](#examples-1) - - [`transcend request download-files`](#transcend-request-download-files) - - [Examples](#examples-2) - - [`transcend request cancel`](#transcend-request-cancel) - - [Examples](#examples-3) - - [`transcend request restart`](#transcend-request-restart) - - [Examples](#examples-4) - - [`transcend request notify-additional-time`](#transcend-request-notify-additional-time) - - [Examples](#examples-5) - - [`transcend request mark-silent`](#transcend-request-mark-silent) - - [Examples](#examples-6) - - [`transcend request enricher-restart`](#transcend-request-enricher-restart) - - [Examples](#examples-7) - - [`transcend request reject-unverified-identifiers`](#transcend-request-reject-unverified-identifiers) - - [Examples](#examples-8) - - [`transcend request export`](#transcend-request-export) - - [Examples](#examples-9) - - [`transcend request skip-preflight-jobs`](#transcend-request-skip-preflight-jobs) - - [Examples](#examples-10) - - [`transcend request system mark-request-data-silos-completed`](#transcend-request-system-mark-request-data-silos-completed) - - [Examples](#examples-11) - - [`transcend request system retry-request-data-silos`](#transcend-request-system-retry-request-data-silos) - - [Examples](#examples-12) - - [`transcend request system skip-request-data-silos`](#transcend-request-system-skip-request-data-silos) - - [Examples](#examples-13) - - [`transcend request preflight pull-identifiers`](#transcend-request-preflight-pull-identifiers) - - [Examples](#examples-14) - - [`transcend request preflight push-identifiers`](#transcend-request-preflight-push-identifiers) - - [Examples](#examples-15) - - [`transcend request cron pull-identifiers`](#transcend-request-cron-pull-identifiers) - - [Examples](#examples-16) - - [`transcend request cron mark-identifiers-completed`](#transcend-request-cron-mark-identifiers-completed) - - [Examples](#examples-17) - - [`transcend consent build-xdi-sync-endpoint`](#transcend-consent-build-xdi-sync-endpoint) - - [Examples](#examples-18) - - [`transcend consent pull-consent-metrics`](#transcend-consent-pull-consent-metrics) - - [Examples](#examples-19) - - [`transcend consent pull-consent-preferences`](#transcend-consent-pull-consent-preferences) - - [Examples](#examples-20) - - [`transcend consent update-consent-manager`](#transcend-consent-update-consent-manager) - - [Examples](#examples-21) - - [`transcend consent upload-consent-preferences`](#transcend-consent-upload-consent-preferences) - - [Examples](#examples-22) - - [`transcend consent upload-cookies-from-csv`](#transcend-consent-upload-cookies-from-csv) - - [Examples](#examples-23) - - [`transcend consent upload-data-flows-from-csv`](#transcend-consent-upload-data-flows-from-csv) - - [Examples](#examples-24) - - [`transcend consent upload-preferences`](#transcend-consent-upload-preferences) - - [Examples](#examples-25) - - [`transcend inventory pull`](#transcend-inventory-pull) - - [Scopes](#scopes) - - [Examples](#examples-26) - - [`transcend inventory push`](#transcend-inventory-push) - - [Scopes](#scopes-1) - - [Examples](#examples-27) - - [CI Integration](#ci-integration) - - [Dynamic Variables](#dynamic-variables) - - [`transcend inventory scan-packages`](#transcend-inventory-scan-packages) - - [Examples](#examples-28) - - [`transcend inventory discover-silos`](#transcend-inventory-discover-silos) - - [Examples](#examples-29) - - [`transcend inventory pull-datapoints`](#transcend-inventory-pull-datapoints) - - [Examples](#examples-30) - - [`transcend inventory pull-unstructured-discovery-files`](#transcend-inventory-pull-unstructured-discovery-files) - - [Examples](#examples-31) - - [`transcend inventory derive-data-silos-from-data-flows`](#transcend-inventory-derive-data-silos-from-data-flows) - - [Examples](#examples-32) - - [`transcend inventory derive-data-silos-from-data-flows-cross-instance`](#transcend-inventory-derive-data-silos-from-data-flows-cross-instance) - - [Examples](#examples-33) - - [`transcend inventory consent-manager-service-json-to-yml`](#transcend-inventory-consent-manager-service-json-to-yml) - - [Examples](#examples-34) - - [`transcend inventory consent-managers-to-business-entities`](#transcend-inventory-consent-managers-to-business-entities) - - [Examples](#examples-35) - - [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys) - - [Examples](#examples-36) - - [`transcend admin chunk-csv`](#transcend-admin-chunk-csv) - - [Examples](#examples-37) - - [`transcend migration sync-ot`](#transcend-migration-sync-ot) - - [Authentication](#authentication) - - [Examples](#examples-38) -- [Prompt Manager](#prompt-manager) -- [Proxy usage](#proxy-usage) - - - -## Changelog - -To stay up to date on breaking changes to the CLI between major version updates, please refer to [CHANGELOG.md](CHANGELOG.md). - -## Overview - -A command line interface that allows you to programatically interact with the Transcend. - -## Installation - -This package is distributed through npm, and assumes an installation of [npm and Node](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm). - -```sh -npm install --global @transcend-io/cli -transcend --help -``` - -You can also run the CLI using npx: - -```sh -npx -p @transcend-io/cli -- transcend --help -``` - -Note - -_The CLI commands which interact with Transcend's API will default to using Transcend's EU backend. To use these commands with the US backend, you will need to add the flag --transcendUrl=https://api.us.transcend.io. You can also set the environment variable `TRANSCEND_API_URL=https://api.us.transcend.io`_ - -## transcend.yml - -Within your git repositories, you can define a file `transcend.yml`. This file allows you define part of your Data Map in code. Using the CLI, you can sync that configuration back to the Transcend Admin Dashboard (https://app.transcend.io/privacy-requests/connected-services). - -You can find various examples for your `transcend.yml` file in the [examples/](./examples/) folder. If you are looking for a starting point to copy and paste, [simple.yml](./examples/simple.yml) is a good place to start. This file is annotated with links and documentations that new members of your team can use if they come across the file. - -The API for this YAML file can be found in [./src/codecs.ts](./src/codecs.ts) under the variable named "TranscendInput". The shape of the YAML file will be type-checked every time a command is run. - -By default, your editor or IDE should recognize `transcend.yml` and validate it against our latest published [JSON schema](./transcend-yml-schema-latest.json). This is dependent on whether your editor uses [yaml-language-server](https://github.com/redhat-developer/yaml-language-server), such as through the [VS Code YAML extension](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml). - -Your editor will use the latest version's schema. To pin the `transcend.yml` schema to a previous major version, include this at the top of your file (and change `v4` to your target major version): - -```yml -# yaml-language-server: $schema=https://raw.githubusercontent.com/transcend-io/cli/main/transcend-yml-schema-v4.json -``` - -The structure of `transcend.yml` looks something like the following: - -```yaml -# Manage at: https://app.transcend.io/infrastructure/api-keys -# See https://docs.transcend.io/docs/authentication -# Define API keys that may be shared across data silos -# in the data map. When creating new data silos through the YAML -# CLI, it is possible to specify which API key should be associated -# with the newly created data silo. -api-keys: - - title: Webhook Key - - title: Analytics Key - -# Manage at: https://app.transcend.io/privacy-requests/identifiers -# See https://docs.transcend.io/docs/identity-enrichment -# Define enricher or pre-flight check webhooks that will be executed -# prior to privacy request workflows. Some examples may include: -# - identity enrichment: look up additional identifiers for that user. -# i.e. map an email address to a user ID -# - fraud check: auto-cancel requests if the user is flagged for fraudulent behavior -# - customer check: auto-cancel request for some custom business criteria -enrichers: - - title: Basic Identity Enrichment - description: Enrich an email address to the userId and phone number - url: https://example.acme.com/transcend-enrichment-webhook - input-identifier: email - output-identifiers: - - userId - - phone - - myUniqueIdentifier - - title: Fraud Check - description: Ensure the email address is not marked as fraudulent - url: https://example.acme.com/transcend-fraud-check - input-identifier: email - output-identifiers: - - email - privacy-actions: - - ERASURE - -# Manage at: https://app.transcend.io/privacy-requests/connected-services -# See https://docs.transcend.io/docs/the-data-map#data-silos -# Define the data silos in your data map. A data silo can be a database, -# or a web service that may use a collection of different data stores under the hood. -data-silos: - # Note: title is the only required top-level field for a data silo - - title: Redshift Data Warehouse - description: The mega-warehouse that contains a copy over all SQL backed databases - integrationName: server - url: https://example.acme.com/transcend-webhook - api-key-title: Webhook Key - data-subjects: - - customer - - employee - - newsletter-subscriber - - b2b-contact - identity-keys: - - email - - userId - deletion-dependencies: - - Identity Service - owners: - - alice@transcend.io - datapoints: - - title: Webhook Notification - key: _global - privacy-actions: - - ACCESS - - ERASURE - - SALE_OPT_OUT - - title: User Model - description: The centralized user model user - key: users - privacy-actions: - - ACCESS - fields: - - key: firstName - title: First Name - description: The first name of the user, inputted during onboarding - - key: email - title: Email - description: The email address of the user -``` - -## Usage - - - -### `transcend request approve` - -```txt -USAGE - transcend request approve (--auth value) (--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD) [--origins PRIVACY_CENTER|ADMIN_DASHBOARD|API|SHOPIFY] [--silentModeBefore value] [--createdAtBefore value] [--createdAtAfter value] [--transcendUrl value] [--concurrency value] - transcend request approve --help - -Bulk approve a set of privacy requests from the DSR Automation -> Incoming Requests tab. - -FLAGS - --auth The Transcend API key. Requires scopes: "Request Approval and Communication", "View Incoming Requests", "Manage Request Compilation" - --actions The request actions to approve [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--origins] The request origins to approve [PRIVACY_CENTER|ADMIN_DASHBOARD|API|SHOPIFY, separator = ,] - [--silentModeBefore] Any requests made before this date should be marked as silent mode - [--createdAtBefore] Approve requests that were submitted before this time - [--createdAtAfter] Approve requests that were submitted after this time - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] - -h --help Print help information and exit -``` - -#### Examples - -**Bulk approve all SALE_OPT_OUT and ERASURE requests** - -```sh -transcend request approve --auth="$TRANSCEND_API_KEY" --actions=SALE_OPT_OUT,ERASURE -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request approve --auth="$TRANSCEND_API_KEY" --actions=ERASURE --transcendUrl=https://api.us.transcend.io -``` - -**Approve all Erasure requests that came through the API** - -```sh -transcend request approve --auth="$TRANSCEND_API_KEY" --actions=ERASURE --origins=API -``` - -**Approve all requests, but mark any request made before 05/03/2023 as silent mode to prevent emailing those requests** - -```sh -transcend request approve \ - --auth="$TRANSCEND_API_KEY" \ - --actions=SALE_OPT_OUT \ - --silentModeBefore=2024-05-03T00:00:00.000Z -``` - -**Increase the concurrency (defaults to 50)** - -```sh -transcend request approve --auth="$TRANSCEND_API_KEY" --actions=ERASURE --concurrency=100 -``` - -**Approve ERASURE requests created within a specific time frame** - -```sh -transcend request approve \ - --auth="$TRANSCEND_API_KEY" \ - --actions=SALE_OPT_OUT \ - --createdAtBefore=2024-05-03T00:00:00.000Z \ - --createdAtAfter=2024-04-03T00:00:00.000Z -``` - -### `transcend request upload` - -```txt -USAGE - transcend request upload (--auth value) [--file value] [--transcendUrl value] [--cacheFilepath value] [--requestReceiptFolder value] [--sombraAuth value] [--concurrency value] [--attributes value] [--isTest] [--isSilent] [--skipSendingReceipt] [--emailIsVerified] [--skipFilterStep] [--dryRun] [--debug] [--defaultPhoneCountryCode value] - transcend request upload --help - -Upload a set of requests from a CSV. - -This command prompts you to map the shape of the CSV to the shape of the Transcend API. There is no requirement for the shape of the incoming CSV, as the script will handle the mapping process. - -The script will also produce a JSON cache file that allows for the mappings to be preserved between runs. - -FLAGS - --auth The Transcend API key. Requires scopes: "Submit New Data Subject Request", "View Identity Verification Settings", "View Global Attributes" - [--file] Path to the CSV file of requests to upload [default = ./requests.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--cacheFilepath] The path to the JSON file encoding the metadata used to map the CSV shape to Transcend API [default = ./transcend-privacy-requests-cache.json] - [--requestReceiptFolder] The path to the folder where receipts of each upload are stored [default = ./privacy-request-upload-receipts] - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] - [--attributes] Tag all of the requests with the following attributes. Format: key1:value1;value2,key2:value3;value4 [default = Tags:transcend-cli] - [--isTest] Flag whether the requests being uploaded are test requests or regular requests [default = false] - [--isSilent/--noIsSilent] Flag whether the requests being uploaded should be submitted in silent mode [default = true] - [--skipSendingReceipt] Flag whether to skip sending of the receipt email [default = false] - [--emailIsVerified/--noEmailIsVerified] Indicate whether the email address being uploaded is pre-verified. Set to false to send a verification email [default = true] - [--skipFilterStep] When true, skip the interactive step to filter down the CSV [default = false] - [--dryRun] When true, perform a dry run of the upload instead of calling the API to submit the requests [default = false] - [--debug] Debug logging [default = false] - [--defaultPhoneCountryCode] When uploading phone numbers, if the phone number is missing a country code, assume this country code [default = 1] - -h --help Print help information and exit -``` - -See a demo of the interactive mapping processbelow (_note: the command is slightly different from the one shown in the video, but the arguments are the same._) - -https://user-images.githubusercontent.com/10264973/205477183-d4762087-668c-43f1-a84c-0fce0ec3e132.mov - -#### Examples - -**Upload requests from a CSV file** - -```sh -transcend request upload --auth="$TRANSCEND_API_KEY" --file=/Users/transcend/Desktop/test.csv -``` - -**For self-hosted sombras that use an internal key** - -```sh -transcend request upload \ - --auth="$TRANSCEND_API_KEY" \ - --sombraAuth="$SOMBRA_INTERNAL_KEY" \ - --file=/Users/transcend/Desktop/test.csv -``` - -**Run without being prompted to filter requests** - -```sh -transcend request upload --auth="$TRANSCEND_API_KEY" --file=/Users/transcend/Desktop/test.csv --skipFilterStep -``` - -**Perform a dry run to see what will be uploaded, without calling the Transcend API** - -```sh -transcend request upload --auth="$TRANSCEND_API_KEY" --file=/Users/transcend/Desktop/test.csv --dryRun -``` - -**Mark the uploaded requests as test requests** - -```sh -transcend request upload --auth="$TRANSCEND_API_KEY" --file=/Users/transcend/Desktop/test.csv --isTest -``` - -**Send email communications to the users throughout the request lifecycle** - -```sh -transcend request upload --auth="$TRANSCEND_API_KEY" --file=/Users/transcend/Desktop/test.csv --isSilent=false -``` - -**Upload requests without sending initial email receipt, but still send later emails** - -```sh -transcend request upload --auth="$TRANSCEND_API_KEY" --file=/Users/transcend/Desktop/test.csv --skipSendingReceipt -``` - -**Increase the concurrency (defaults to 50)** - -```sh -transcend request upload --auth="$TRANSCEND_API_KEY" --file=/Users/transcend/Desktop/test.csv --concurrency=100 -``` - -**Specify default country code for phone numbers** - -```sh -transcend request upload \ - --auth="$TRANSCEND_API_KEY" \ - --file=/Users/transcend/Desktop/test.csv \ - --defaultPhoneCountryCode=44 -``` - -**Include debug logs - warning, this logs out personal data** - -```sh -transcend request upload --auth="$TRANSCEND_API_KEY" --file=/Users/transcend/Desktop/test.csv --debug -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request upload \ - --auth="$TRANSCEND_API_KEY" \ - --sombraAuth="$SOMBRA_INTERNAL_KEY" \ - --file=/Users/transcend/Desktop/test.csv \ - --transcendUrl=https://api.us.transcend.io -``` - -**Send email verification to user before request continues** - -```sh -transcend request upload \ - --auth="$TRANSCEND_API_KEY" \ - --file=/Users/transcend/Desktop/test.csv \ - --isSilent=false \ - --emailIsVerified=false -``` - -**Tag all uploaded requests with custom fields (formerly known as "attributes")** - -```sh -transcend request upload \ - --auth="$TRANSCEND_API_KEY" \ - --file=/Users/transcend/Desktop/test.csv \ - --attributes=Tags:transcend-cli;my-customer-tag,Customer:acme-corp -``` - -### `transcend request download-files` - -```txt -USAGE - transcend request download-files (--auth value) [--sombraAuth value] [--concurrency value] [--requestIds value]... [--statuses REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED] [--folderPath value] [--createdAtBefore value] [--createdAtAfter value] [--approveAfterDownload] [--transcendUrl value] - transcend request download-files --help - -Download the files associated with a Data Subject Access Request (DSAR) from DSR Automation -> Incoming Requests tab. - -FLAGS - --auth The Transcend API key. Requires scopes: "View the Request Compilation", "View Incoming Requests", "Request Approval and Communication" - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--concurrency] The concurrency to use when downloading requests in parallel [default = 10] - [--requestIds]... Specify the specific request IDs to download [separator = ,] - [--statuses] The request statuses to download. Comma-separated list. Defaults to APPROVING,DOWNLOADABLE. [REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED, separator = ,] - [--folderPath] The folder to download files to [default = ./dsr-files] - [--createdAtBefore] Download requests that were submitted before this time - [--createdAtAfter] Download requests that were submitted after this time - [--approveAfterDownload] If the request is in status=APPROVING, approve the request after its downloaded [default = false] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -Download the files associated with a Data Subject Access Request (DSAR) from [DSR Automation -> Incoming Requests](https://app.transcend.io/privacy-requests/incoming-requests) tab. - -Screenshot 2025-06-03 at 3 32 00 PM - -#### Examples - -**Download all requests in status=APPROVING or status=DOWNLOADABLE** - -```sh -transcend request download-files --auth="$TRANSCEND_API_KEY" -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request download-files --auth="$TRANSCEND_API_KEY" --transcendUrl=https://api.us.transcend.io -``` - -**Write files to a specific folder on disk** - -```sh -transcend request download-files --auth="$TRANSCEND_API_KEY" --folderPath=./my-folder -``` - -**Auto approve after download** - -```sh -transcend request download-files --auth="$TRANSCEND_API_KEY" --approveAfterDownload -``` - -**Download requests in APPROVING state only** - -```sh -transcend request download-files --auth="$TRANSCEND_API_KEY" --statuses=APPROVING -``` - -**Increase the concurrency (defaults to 10)** - -```sh -transcend request download-files --auth="$TRANSCEND_API_KEY" --concurrency=100 -``` - -**Download requests in a timeframe** - -```sh -transcend request download-files \ - --auth="$TRANSCEND_API_KEY" \ - --createdAtBefore=2024-05-03T00:00:00.000Z \ - --createdAtAfter=2024-04-03T00:00:00.000Z -``` - -**Download specific requests** - -```sh -transcend request download-files \ - --auth="$TRANSCEND_API_KEY" \ - --requestIds=b8c2ce13-9e40-4104-af79-23c68f2a87ba,d5eedc52-0f85-4034-bc01-14951acad5aa -``` - -### `transcend request cancel` - -```txt -USAGE - transcend request cancel (--auth value) (--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD) [--statuses REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED] [--requestIds value]... [--silentModeBefore value] [--createdAtBefore value] [--createdAtAfter value] [--cancellationTitle value] [--transcendUrl value] [--concurrency value] - transcend request cancel --help - -Bulk cancel a set of privacy requests from the DSR Automation -> Incoming Requests tab. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Incoming Requests", "Request Approval and Communication" - --actions The request actions to cancel [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--statuses] The request statuses to cancel. Comma-separated list. [REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED, separator = ,] - [--requestIds]... Specify the specific request IDs to cancel [separator = ,] - [--silentModeBefore] Any requests made before this date should be marked as silent mode for canceling to skip email sending - [--createdAtBefore] Cancel requests that were submitted before this time - [--createdAtAfter] Cancel requests that were submitted after this time - [--cancellationTitle] The title of the email template that should be sent to the requests upon cancelation [default = Request Canceled] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] - -h --help Print help information and exit -``` - -#### Examples - -**Bulk cancel all open SALE_OPT_OUT and ERASURE requests** - -```sh -transcend request cancel --auth="$TRANSCEND_API_KEY" --actions=SALE_OPT_OUT,ERASURE -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request cancel --auth="$TRANSCEND_API_KEY" --actions=ERASURE --transcendUrl=https://api.us.transcend.io -``` - -**Bulk cancel all Erasure (request.type=ERASURE) requests that are in an enriching state (request.status=ENRICHING)** - -```sh -transcend request cancel --auth="$TRANSCEND_API_KEY" --actions=ERASURE --statuses=ENRICHING -``` - -**Send a specific email template to the request that are being canceled** - -```sh -transcend request cancel --auth="$TRANSCEND_API_KEY" --actions=ERASURE --cancellationTitle="Custom Email Template" -``` - -**Cancel all open SALE_OPT_OUT, but mark any request made before 05/03/2023 as silent mode to prevent emailing those requests** - -```sh -transcend request cancel \ - --auth="$TRANSCEND_API_KEY" \ - --actions=SALE_OPT_OUT \ - --silentModeBefore=2024-05-03T00:00:00.000Z -``` - -**Cancel all open SALE_OPT_OUT, within a specific time frame** - -```sh -transcend request cancel \ - --auth="$TRANSCEND_API_KEY" \ - --actions=SALE_OPT_OUT \ - --createdAtBefore=2024-05-03T00:00:00.000Z \ - --createdAtAfter=2024-04-03T00:00:00.000Z -``` - -**Increase the concurrency (defaults to 50)** - -```sh -transcend request cancel --auth="$TRANSCEND_API_KEY" --actions=ERASURE --concurrency=500 -``` - -**Bulk cancel requests by ID** - -```sh -transcend request cancel \ - --auth="$TRANSCEND_API_KEY" \ - --actions=ACCESS,ERASURE,SALE_OPT_OUT,CONTACT_OPT_OUT \ - --statuses=ENRICHING,COMPILING,APPROVING,WAITING,REQUEST_MADE,ON_HOLD,DELAYED,SECONDARY \ - --requestIds=c3ae78c9-2768-4666-991a-d2f729503337,342e4bd1-64ea-4af0-a4ad-704b5a07cfe4 -``` - -### `transcend request restart` - -```txt -USAGE - transcend request restart (--auth value) (--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD) (--statuses REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED) [--transcendUrl value] [--requestReceiptFolder value] [--sombraAuth value] [--concurrency value] [--requestIds value]... [--emailIsVerified] [--createdAt value] [--silentModeBefore value] [--createdAtBefore value] [--createdAtAfter value] [--sendEmailReceipt] [--copyIdentifiers] [--skipWaitingPeriod] - transcend request restart --help - -Bulk update a set of privacy requests based on a set of request filters. - -FLAGS - --auth The Transcend API key. Requires scopes: "Submit New Data Subject Request", "View the Request Compilation" - --actions The request actions to restart [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - --statuses The request statuses to restart [REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED, separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--requestReceiptFolder] The path to the folder where receipts of each upload are stored [default = ./privacy-request-upload-receipts] - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--concurrency] The concurrency to use when uploading requests in parallel [default = 15] - [--requestIds]... Specify the specific request IDs to restart [separator = ,] - [--emailIsVerified/--noEmailIsVerified] Indicate whether the primary email address is verified. Set to false to send a verification email [default = true] - [--createdAt] Restart requests that were submitted before a specific date - [--silentModeBefore] Requests older than this date should be marked as silent mode - [--createdAtBefore] Restart requests that were submitted before this time - [--createdAtAfter] Restart requests that were submitted after this time - [--sendEmailReceipt] Send email receipts to the restarted requests [default = false] - [--copyIdentifiers] Copy over all enriched identifiers from the initial request [default = false] - [--skipWaitingPeriod] Skip queued state of request and go straight to compiling [default = false] - -h --help Print help information and exit -``` - -#### Examples - -**Restart requests with specific statuses and actions** - -```sh -transcend request restart --auth="$TRANSCEND_API_KEY" --statuses=COMPILING,ENRICHING --actions=ACCESS,ERASURE -``` - -**For self-hosted sombras that use an internal key** - -```sh -transcend request restart \ - --auth="$TRANSCEND_API_KEY" \ - --sombraAuth="$SOMBRA_INTERNAL_KEY" \ - --statuses=COMPILING,ENRICHING \ - --actions=ACCESS,ERASURE -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request restart \ - --auth="$TRANSCEND_API_KEY" \ - --sombraAuth="$SOMBRA_INTERNAL_KEY" \ - --statuses=COMPILING,ENRICHING \ - --actions=ACCESS,ERASURE \ - --transcendUrl=https://api.us.transcend.io -``` - -**Increase the concurrency (defaults to 15)** - -```sh -transcend request restart \ - --auth="$TRANSCEND_API_KEY" \ - --statuses=COMPILING,ENRICHING \ - --actions=ACCESS,ERASURE \ - --concurrency=100 -``` - -**Re-verify emails** - -```sh -transcend request restart \ - --auth="$TRANSCEND_API_KEY" \ - --statuses=COMPILING,ENRICHING \ - --actions=ACCESS,ERASURE \ - --emailIsVerified=false -``` - -**Restart specific requests by ID** - -```sh -transcend request restart \ - --auth="$TRANSCEND_API_KEY" \ - --statuses=COMPILING,ENRICHING \ - --actions=ACCESS,ERASURE \ - --requestIds=c3ae78c9-2768-4666-991a-d2f729503337,342e4bd1-64ea-4af0-a4ad-704b5a07cfe4 -``` - -**Restart requests that were submitted before a specific date** - -```sh -transcend request restart \ - --auth="$TRANSCEND_API_KEY" \ - --statuses=COMPILING,ENRICHING \ - --actions=ACCESS,ERASURE \ - --createdAt=2024-05-11T00:00:00.000Z -``` - -**Restart requests and place everything in silent mode submitted before a certain date** - -```sh -transcend request restart \ - --auth="$TRANSCEND_API_KEY" \ - --statuses=COMPILING,ENRICHING \ - --actions=ACCESS,ERASURE \ - --silentModeBefore=2024-12-05T00:00:00.000Z -``` - -**Restart requests within a specific timeframe** - -```sh -transcend request restart \ - --auth="$TRANSCEND_API_KEY" \ - --statuses=COMPILING,ENRICHING \ - --actions=ACCESS,ERASURE \ - --createdAtBefore=2024-04-05T00:00:00.000Z \ - --createdAtAfter=2024-02-21T00:00:00.000Z -``` - -**Send email receipts to the restarted requests** - -```sh -transcend request restart \ - --auth="$TRANSCEND_API_KEY" \ - --statuses=COMPILING,ENRICHING \ - --actions=ACCESS,ERASURE \ - --sendEmailReceipt -``` - -**Copy over all enriched identifiers from the initial request** - -```sh -transcend request restart \ - --auth="$TRANSCEND_API_KEY" \ - --statuses=COMPILING,ENRICHING \ - --actions=ACCESS,ERASURE \ - --copyIdentifiers -``` - -**Skip queued state of request and go straight to compiling** - -```sh -transcend request restart \ - --auth="$TRANSCEND_API_KEY" \ - --statuses=COMPILING,ENRICHING \ - --actions=ACCESS,ERASURE \ - --skipWaitingPeriod -``` - -### `transcend request notify-additional-time` - -```txt -USAGE - transcend request notify-additional-time (--auth value) (--createdAtBefore value) [--createdAtAfter value] [--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD] [--daysLeft value] [--days value] [--requestIds value]... [--emailTemplate value] [--transcendUrl value] [--concurrency value] - transcend request notify-additional-time --help - -Bulk notify a set of privacy requests from the DSR Automation -> Incoming Requests tab that more time is needed to complete the request. Note any request in silent mode will not be emailed. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Incoming Requests", "Request Approval and Communication" - --createdAtBefore Notify requests that are open but submitted before this time - [--createdAtAfter] Notify requests that are open but submitted after this time - [--actions] The request actions to notify [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--daysLeft] Only notify requests that have less than this number of days until they are considered expired [default = 10] - [--days] The number of days to adjust the expiration of the request to [default = 45] - [--requestIds]... Specify the specific request IDs to notify [separator = ,] - [--emailTemplate] The title of the email template that should be sent to the requests [default = Additional Time Needed] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] - -h --help Print help information and exit -``` - -#### Examples - -**Notify all request types that were made before 01/01/2024** - -```sh -transcend request notify-additional-time --auth="$TRANSCEND_API_KEY" --createdAtBefore=2024-01-01T00:00:00.000Z -``` - -**Notify all request types that were made during a date range** - -```sh -transcend request notify-additional-time \ - --auth="$TRANSCEND_API_KEY" \ - --createdAtBefore=2024-01-01T00:00:00.000Z \ - --createdAtAfter=2024-12-15T00:00:00.000Z -``` - -**Notify certain request types** - -```sh -transcend request notify-additional-time \ - --auth="$TRANSCEND_API_KEY" \ - --createdAtBefore=2024-01-01T00:00:00.000Z \ - --actions=SALE_OPT_OUT,ERASURE -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request notify-additional-time \ - --auth="$TRANSCEND_API_KEY" \ - --createdAtBefore=2024-01-01T00:00:00.000Z \ - --transcendUrl=https://api.us.transcend.io -``` - -**Bulk notify requests by ID** - -```sh -transcend request notify-additional-time \ - --auth="$TRANSCEND_API_KEY" \ - --createdAtBefore=2024-01-01T00:00:00.000Z \ - --requestIds=c3ae78c9-2768-4666-991a-d2f729503337,342e4bd1-64ea-4af0-a4ad-704b5a07cfe4 -``` - -**Only notify requests that are expiring in the next 3 days or less** - -```sh -transcend request notify-additional-time \ - --auth="$TRANSCEND_API_KEY" \ - --createdAtBefore=2024-01-01T00:00:00.000Z \ - --daysLeft=3 -``` - -**Change number of days to extend request by** - -```sh -transcend request notify-additional-time \ - --auth="$TRANSCEND_API_KEY" \ - --createdAtBefore=2024-01-01T00:00:00.000Z \ - --days=30 -``` - -**Send a specific email template to the request that instead of the default** - -```sh -transcend request notify-additional-time \ - --auth="$TRANSCEND_API_KEY" \ - --createdAtBefore=2024-01-01T00:00:00.000Z \ - --emailTemplate="Custom Email Template" -``` - -**Increase the concurrency (defaults to 50)** - -```sh -transcend request notify-additional-time \ - --auth="$TRANSCEND_API_KEY" \ - --createdAtBefore=2024-01-01T00:00:00.000Z \ - --concurrency=500 -``` - -### `transcend request mark-silent` - -```txt -USAGE - transcend request mark-silent (--auth value) (--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD) [--statuses REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED] [--requestIds value]... [--createdAtBefore value] [--createdAtAfter value] [--transcendUrl value] [--concurrency value] - transcend request mark-silent --help - -Bulk update a set of privacy requests from the DSR Automation -> Incoming Requests tab to be in silent mode. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --actions The request actions to mark silent [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--statuses] The request statuses to mark silent. Comma-separated list. Defaults to REQUEST_MADE,WAITING,ENRICHING,COMPILING,DELAYED,APPROVING,SECONDARY,SECONDARY_APPROVING. [REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED, separator = ,] - [--requestIds]... Specify the specific request IDs to mark silent [separator = ,] - [--createdAtBefore] Mark silent requests that were submitted before this time - [--createdAtAfter] Mark silent requests that were submitted after this time - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 50] - -h --help Print help information and exit -``` - -#### Examples - -**Bulk mark silent all open SALE_OPT_OUT and ERASURE requests** - -```sh -transcend request mark-silent --auth="$TRANSCEND_API_KEY" --actions=SALE_OPT_OUT,ERASURE -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request mark-silent \ - --auth="$TRANSCEND_API_KEY" \ - --actions=ERASURE \ - --transcendUrl=https://api.us.transcend.io -``` - -**Bulk mark as silent all Erasure (request.type=ERASURE) requests that are in an enriching state (request.status=ENRICHING)** - -```sh -transcend request mark-silent --auth="$TRANSCEND_API_KEY" --actions=ERASURE --statuses=ENRICHING -``` - -**Bulk mark as silent requests by ID** - -```sh -transcend request mark-silent \ - --auth="$TRANSCEND_API_KEY" \ - --actions=ACCESS,ERASURE,SALE_OPT_OUT,CONTACT_OPT_OUT \ - --statuses=ENRICHING,COMPILING,APPROVING,WAITING,REQUEST_MADE,ON_HOLD,DELAYED,SECONDARY \ - --requestIds=c3ae78c9-2768-4666-991a-d2f729503337,342e4bd1-64ea-4af0-a4ad-704b5a07cfe4 -``` - -**Mark sale opt out requests as silent within a certain date range** - -```sh -transcend request mark-silent \ - --auth="$TRANSCEND_API_KEY" \ - --actions=SALE_OPT_OUT \ - --createdAtBefore=2024-05-03T00:00:00.000Z \ - --createdAtAfter=2024-04-03T00:00:00.000Z -``` - -**Increase the concurrency (defaults to 50)** - -```sh -transcend request mark-silent --auth="$TRANSCEND_API_KEY" --actions=ERASURE --concurrency=500 -``` - -### `transcend request enricher-restart` - -```txt -USAGE - transcend request enricher-restart (--auth value) (--enricherId value) [--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD] [--requestEnricherStatuses QUEUED|WAITING|SKIPPED|ERROR|RESOLVED|ACTION_REQUIRED|REMOTE_PROCESSING|WAITING_ON_DEPENDENCIES|POLLING] [--transcendUrl value] [--concurrency value] [--requestIds value]... [--createdAtBefore value] [--createdAtAfter value] - transcend request enricher-restart --help - -Bulk restart a particular enricher across a series of DSRs. - -The API key needs the following scopes: -- Manage Request Compilation - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --enricherId The ID of the enricher to restart - [--actions] The request action to restart [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--requestEnricherStatuses] The request enricher statuses to restart [QUEUED|WAITING|SKIPPED|ERROR|RESOLVED|ACTION_REQUIRED|REMOTE_PROCESSING|WAITING_ON_DEPENDENCIES|POLLING, separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 15] - [--requestIds]... Specify the specific request IDs to restart [separator = ,] - [--createdAtBefore] Restart requests that were submitted before this time - [--createdAtAfter] Restart requests that were submitted after this time - -h --help Print help information and exit -``` - -#### Examples - -**Restart a particular enricher across a series of DSRs** - -```sh -transcend request enricher-restart --auth="$TRANSCEND_API_KEY" --enricherId=3be5e898-fea9-4614-84de-88cd5265c557 -``` - -**Restart specific request types** - -```sh -transcend request enricher-restart \ - --auth="$TRANSCEND_API_KEY" \ - --enricherId=3be5e898-fea9-4614-84de-88cd5265c557 \ - --actions=ACCESS,ERASURE -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request enricher-restart \ - --auth="$TRANSCEND_API_KEY" \ - --enricherId=3be5e898-fea9-4614-84de-88cd5265c557 \ - --transcendUrl=https://api.us.transcend.io -``` - -**Increase the concurrency (defaults to 15)** - -```sh -transcend request enricher-restart \ - --auth="$TRANSCEND_API_KEY" \ - --enricherId=3be5e898-fea9-4614-84de-88cd5265c557 \ - --concurrency=100 -``` - -**Restart requests within a specific timeframe** - -```sh -transcend request enricher-restart \ - --auth="$TRANSCEND_API_KEY" \ - --enricherId=3be5e898-fea9-4614-84de-88cd5265c557 \ - --createdAtBefore=2024-04-05T00:00:00.000Z \ - --createdAtAfter=2024-02-21T00:00:00.000Z -``` - -**Restart requests that are in an error state** - -```sh -transcend request enricher-restart \ - --auth="$TRANSCEND_API_KEY" \ - --enricherId=3be5e898-fea9-4614-84de-88cd5265c557 \ - --requestEnricherStatuses=ERROR -``` - -### `transcend request reject-unverified-identifiers` - -```txt -USAGE - transcend request reject-unverified-identifiers (--auth value) (--identifierNames value)... [--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD] [--transcendUrl value] - transcend request reject-unverified-identifiers --help - -Bulk clear out any request identifiers that are unverified. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --identifierNames... The names of identifiers to clear out [separator = ,] - [--actions] The request action to restart [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Bulk clear out any request identifiers that are unverified** - -```sh -transcend request reject-unverified-identifiers --auth="$TRANSCEND_API_KEY" --identifierNames=phone -``` - -**Restart specific request types** - -```sh -transcend request reject-unverified-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --identifierNames=phone \ - --actions=ACCESS,ERASURE -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request reject-unverified-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --identifierNames=phone \ - --transcendUrl=https://api.us.transcend.io -``` - -### `transcend request export` - -```txt -USAGE - transcend request export (--auth value) [--sombraAuth value] [--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD] [--statuses REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED] [--transcendUrl value] [--file value] [--concurrency value] [--createdAtBefore value] [--createdAtAfter value] [--showTests] [--pageLimit value] - transcend request export --help - -Export privacy requests and request identifiers to a CSV file. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Incoming Requests", "View the Request Compilation" - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--actions] The request actions to export [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--statuses] The request statuses to export [REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED, separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--file] Path to the CSV file where identifiers will be written to [default = ./transcend-request-export.csv] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 100] - [--createdAtBefore] Pull requests that were submitted before this time - [--createdAtAfter] Pull requests that were submitted after this time - [--showTests/--noShowTests] Filter for test requests or production requests - when not provided, pulls both - [--pageLimit] The page limit to use when pulling in pages of requests [default = 100] - -h --help Print help information and exit -``` - -#### Examples - -**Pull all requests** - -```sh -transcend request export --auth="$TRANSCEND_API_KEY" -``` - -**Filter for specific actions and statuses** - -```sh -transcend request export --auth="$TRANSCEND_API_KEY" --statuses=COMPILING,ENRICHING --actions=ACCESS,ERASURE -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request export --auth="$TRANSCEND_API_KEY" --transcendUrl=https://api.us.transcend.io -``` - -**With Sombra authentication** - -```sh -transcend request export --auth="$TRANSCEND_API_KEY" --sombraAuth="$SOMBRA_INTERNAL_KEY" -``` - -**Increase the concurrency (defaults to 100)** - -```sh -transcend request export --auth="$TRANSCEND_API_KEY" --concurrency=500 -``` - -**Filter for production requests only** - -```sh -transcend request export --auth="$TRANSCEND_API_KEY" --showTests=false -``` - -**Filter for requests within a date range** - -```sh -transcend request export \ - --auth="$TRANSCEND_API_KEY" \ - --createdAtBefore=2024-04-05T00:00:00.000Z \ - --createdAtAfter=2024-02-21T00:00:00.000Z -``` - -**Write to a specific file location** - -```sh -transcend request export --auth="$TRANSCEND_API_KEY" --file=./path/to/file.csv -``` - -### `transcend request skip-preflight-jobs` - -```txt -USAGE - transcend request skip-preflight-jobs (--auth value) (--enricherIds value)... [--transcendUrl value] - transcend request skip-preflight-jobs --help - -This command allows for bulk skipping preflight checks. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --enricherIds... The ID of the enrichers to skip privacy request jobs for [separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Bulk skipping preflight checks** - -```sh -transcend request skip-preflight-jobs --auth="$TRANSCEND_API_KEY" --enricherIds=70810f2e-cf90-43f6-9776-901a5950599f -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request skip-preflight-jobs \ - --auth="$TRANSCEND_API_KEY" \ - --enricherIds=70810f2e-cf90-43f6-9776-901a5950599f,db1e64ba-cea6-43ff-ad27-5dc8122e5224 \ - --transcendUrl=https://api.us.transcend.io -``` - -### `transcend request system mark-request-data-silos-completed` - -```txt -USAGE - transcend request system mark-request-data-silos-completed (--auth value) (--dataSiloId value) [--file value] [--transcendUrl value] - transcend request system mark-request-data-silos-completed --help - -This command takes in a CSV of Request IDs as well as a Data Silo ID and marks all associated privacy request jobs as completed. -This command is useful with the "Bulk Response" UI. The CSV is expected to have 1 column named "Request Id". - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --dataSiloId The ID of the data silo to pull in - [--file] Path to the CSV file where identifiers will be written to. The CSV is expected to have 1 column named "Request Id". [default = ./request-identifiers.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Mark all associated privacy request jobs as completed** - -```sh -transcend request system mark-request-data-silos-completed \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f -``` - -**Pull to a specific file location** - -```sh -transcend request system mark-request-data-silos-completed \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --file=/Users/transcend/Desktop/test.csv -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request system mark-request-data-silos-completed \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --transcendUrl=https://api.us.transcend.io -``` - -### `transcend request system retry-request-data-silos` - -```txt -USAGE - transcend request system retry-request-data-silos (--auth value) (--dataSiloId value) (--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD) [--transcendUrl value] - transcend request system retry-request-data-silos --help - -This command allows for bulk restarting a set of data silos jobs for open privacy requests. This is equivalent to clicking the "Wipe and Retry" button for a particular data silo across a set of privacy requests. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --dataSiloId The ID of the data silo to pull in - --actions The request actions to restart [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Bulk restarting a set of data silos jobs for open privacy requests** - -```sh -transcend request system retry-request-data-silos \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --actions=ACCESS -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request system retry-request-data-silos \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --actions=ACCESS \ - --transcendUrl=https://api.us.transcend.io -``` - -### `transcend request system skip-request-data-silos` - -```txt -USAGE - transcend request system skip-request-data-silos (--auth value) (--dataSiloId value) [--transcendUrl value] (--statuses REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED) [--status SKIPPED|RESOLVED] - transcend request system skip-request-data-silos --help - -This command allows for bulk skipping all open privacy request jobs for a particular data silo. This command is useful if you want to disable a data silo and then clear out any active privacy requests that are still queued up for that data silo. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Compilation" - --dataSiloId The ID of the data silo to skip privacy request jobs for - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - --statuses The request statuses to skip [REQUEST_MADE|FAILED_VERIFICATION|ENRICHING|ON_HOLD|WAITING|COMPILING|APPROVING|DELAYED|COMPLETED|DOWNLOADABLE|VIEW_CATEGORIES|CANCELED|SECONDARY|SECONDARY_COMPLETED|SECONDARY_APPROVING|REVOKED, separator = ,] - [--status] The status to set the request data silo job to [SKIPPED|RESOLVED, default = SKIPPED] - -h --help Print help information and exit -``` - -#### Examples - -**Bulk skipping all open privacy request jobs for a particular data silo** - -```sh -transcend request system skip-request-data-silos \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request system skip-request-data-silos \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --transcendUrl=https://api.us.transcend.io -``` - -**Only mark as completed requests in "removing data" phase** - -```sh -transcend request system skip-request-data-silos \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --statuses=SECONDARY -``` - -**Set to status "RESOLVED" instead of status "SKIPPED"** - -```sh -transcend request system skip-request-data-silos \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --status=RESOLVED -``` - -### `transcend request preflight pull-identifiers` - -```txt -USAGE - transcend request preflight pull-identifiers (--auth value) [--sombraAuth value] [--transcendUrl value] [--file value] [--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD] [--concurrency value] - transcend request preflight pull-identifiers --help - -This command pulls down the set of privacy requests that are currently pending manual enrichment. - -This is useful for the following workflow: - -1. Pull identifiers to CSV: - - transcend request preflight pull-identifiers --file=./enrichment-requests.csv - -2. Fill out the CSV with additional identifiers - -3. Push updated back to Transcend: - - transcend request preflight push-identifiers --file=./enrichment-requests.csv - -FLAGS - --auth The Transcend API key. Requires scopes: "View Incoming Requests", "View the Request Compilation" - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--file] Path to the CSV file where requests will be written to [default = ./manual-enrichment-identifiers.csv] - [--actions] The request actions to pull for [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 100] - -h --help Print help information and exit -``` - -#### Examples - -**Pull down the set of privacy requests that are currently pending manual enrichment** - -```sh -transcend request preflight pull-identifiers --auth="$TRANSCEND_API_KEY" -``` - -**Pull to a specific file location** - -```sh -transcend request preflight pull-identifiers --auth="$TRANSCEND_API_KEY" --file=/Users/transcend/Desktop/test.csv -``` - -**For specific types of requests** - -```sh -transcend request preflight pull-identifiers --auth="$TRANSCEND_API_KEY" --actions=ACCESS,ERASURE -``` - -**For US hosted infrastructure** - -```sh -transcend request preflight pull-identifiers --auth="$TRANSCEND_API_KEY" --transcendUrl=https://api.us.transcend.io -``` - -**With Sombra authentication** - -```sh -transcend request preflight pull-identifiers --auth="$TRANSCEND_API_KEY" --sombraAuth="$SOMBRA_INTERNAL_KEY" -``` - -**With specific concurrency** - -```sh -transcend request preflight pull-identifiers --auth="$TRANSCEND_API_KEY" --concurrency=200 -``` - -### `transcend request preflight push-identifiers` - -```txt -USAGE - transcend request preflight push-identifiers (--auth value) (--enricherId value) [--sombraAuth value] [--transcendUrl value] [--file value] [--markSilent] [--concurrency value] - transcend request preflight push-identifiers --help - -This command push up a set of identifiers for a set of requests pending manual enrichment. - -This is useful for the following workflow: - -1. Pull identifiers to CSV: - - transcend request preflight pull-identifiers --file=./enrichment-requests.csv - -2. Fill out the CSV with additional identifiers - -3. Push updated back to Transcend: - - transcend request preflight push-identifiers --file=./enrichment-requests.csv - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Request Identity Verification", "Manage Request Compilation" - --enricherId The ID of the Request Enricher to upload to - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--file] Path to the CSV file where requests will be written to [default = ./manual-enrichment-identifiers.csv] - [--markSilent] When true, set requests into silent mode before enriching [default = false] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 100] - -h --help Print help information and exit -``` - -#### Examples - -**Push up a set of identifiers for a set of requests pending manual enrichment** - -```sh -transcend request preflight push-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --enricherId=27d45a0d-7d03-47fa-9b30-6d697005cfcf -``` - -**Pull to a specific file location** - -```sh -transcend request preflight push-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --enricherId=27d45a0d-7d03-47fa-9b30-6d697005cfcf \ - --file=/Users/transcend/Desktop/test.csv -``` - -**For US hosted infrastructure** - -```sh -transcend request preflight push-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --enricherId=27d45a0d-7d03-47fa-9b30-6d697005cfcf \ - --transcendUrl=https://api.us.transcend.io -``` - -**With Sombra authentication** - -```sh -transcend request preflight push-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --enricherId=27d45a0d-7d03-47fa-9b30-6d697005cfcf \ - --sombraAuth="$SOMBRA_INTERNAL_KEY" -``` - -**With specific concurrency** - -```sh -transcend request preflight push-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --enricherId=27d45a0d-7d03-47fa-9b30-6d697005cfcf \ - --concurrency=200 -``` - -**When enriching requests, mark all requests as silent mode before processing** - -```sh -transcend request preflight push-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --enricherId=27d45a0d-7d03-47fa-9b30-6d697005cfcf \ - --markSilent -``` - -### `transcend request cron pull-identifiers` - -```txt -USAGE - transcend request cron pull-identifiers (--auth value) (--dataSiloId value) (--actions AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD) [--file value] [--transcendUrl value] [--sombraAuth value] [--pageLimit value] [--skipRequestCount] [--chunkSize value] - transcend request cron pull-identifiers --help - -If you are using the cron job integration, you can run this command to pull the outstanding identifiers for the data silo to a CSV. - -For large datasets, the output will be automatically split into multiple CSV files to avoid file system size limits. Use the --chunkSize parameter to control the maximum number of rows per file. - -Read more at https://docs.transcend.io/docs/integrations/cron-job-integration. - -FLAGS - --auth The Transcend API key. This key must be associated with the data silo(s) being operated on. No scopes are required for this command. - --dataSiloId The ID of the data silo to pull in - --actions The request actions to restart [AUTOMATED_DECISION_MAKING_OPT_OUT|USE_OF_SENSITIVE_INFORMATION_OPT_OUT|CONTACT_OPT_OUT|SALE_OPT_OUT|TRACKING_OPT_OUT|CUSTOM_OPT_OUT|AUTOMATED_DECISION_MAKING_OPT_IN|USE_OF_SENSITIVE_INFORMATION_OPT_IN|SALE_OPT_IN|TRACKING_OPT_IN|CONTACT_OPT_IN|CUSTOM_OPT_IN|ACCESS|ERASURE|RECTIFICATION|RESTRICTION|BUSINESS_PURPOSE|PLACE_ON_LEGAL_HOLD|REMOVE_FROM_LEGAL_HOLD, separator = ,] - [--file] Path to the CSV file where identifiers will be written to [default = ./cron-identifiers.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--pageLimit] The page limit to use when pulling in pages of identifiers [default = 100] - [--skipRequestCount] Whether to skip the count of all outstanding requests. This is required to render the progress bar, but can take a long time to run if you have a large number of outstanding requests to process. In that case, we recommend setting skipRequestCount=true so that you can still proceed with fetching the identifiers [default = false] - [--chunkSize] Maximum number of rows per CSV file. For large datasets, the output will be automatically split into multiple files to avoid file system size limits. Each file will contain at most this many rows [default = 10000] - -h --help Print help information and exit -``` - -#### Examples - -**Pull outstanding identifiers for a data silo** - -```sh -transcend request cron pull-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --actions=ERASURE -``` - -**Pull to a specific file location** - -```sh -transcend request cron pull-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --actions=ERASURE \ - --file=/Users/transcend/Desktop/test.csv -``` - -**For self-hosted sombras that use an internal key** - -```sh -transcend request cron pull-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --actions=ERASURE \ - --sombraAuth="$SOMBRA_INTERNAL_KEY" -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request cron pull-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --actions=ERASURE \ - --transcendUrl=https://api.us.transcend.io -``` - -**Specifying the page limit, defaults to 100** - -```sh -transcend request cron pull-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --actions=ERASURE \ - --pageLimit=300 \ - --chunkSize=6000 -``` - -**Specifying the chunk size for large datasets to avoid file size limits (defaults to 100,000 rows per file)** - -```sh -transcend request cron pull-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --actions=ERASURE \ - --chunkSize=50000 -``` - -### `transcend request cron mark-identifiers-completed` - -```txt -USAGE - transcend request cron mark-identifiers-completed (--auth value) (--dataSiloId value) [--file value] [--transcendUrl value] [--sombraAuth value] - transcend request cron mark-identifiers-completed --help - -This command takes the output of "transcend request cron pull-identifiers" and notifies Transcend that all of the requests in the CSV have been processed. -This is used in the workflow like: - -1. Pull identifiers to CSV: - - transcend request cron pull-identifiers \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --actions=ERASURE \ - --file=./outstanding-requests.csv - -2. Run your process to operate on that CSV of requests. - -3. Notify Transcend of completion - - transcend request cron mark-identifiers-completed \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --file=./outstanding-requests.csv - -Read more at https://docs.transcend.io/docs/integrations/cron-job-integration. - -FLAGS - --auth The Transcend API key. This key must be associated with the data silo(s) being operated on. No scopes are required for this command. - --dataSiloId The ID of the data silo to pull in - [--file] Path to the CSV file where identifiers will be written to [default = ./cron-identifiers.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - -h --help Print help information and exit -``` - -#### Examples - -**Mark identifiers as completed** - -```sh -transcend request cron mark-identifiers-completed \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f -``` - -**Pull to a specific file location** - -```sh -transcend request cron mark-identifiers-completed \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --file=/Users/transcend/Desktop/test.csv -``` - -**For self-hosted sombras that use an internal key** - -```sh -transcend request cron mark-identifiers-completed \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --sombraAuth="$SOMBRA_INTERNAL_KEY" -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend request cron mark-identifiers-completed \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f \ - --transcendUrl=https://api.us.transcend.io -``` - -### `transcend consent build-xdi-sync-endpoint` - -```txt -USAGE - transcend consent build-xdi-sync-endpoint (--auth value) (--xdiLocation value) [--file value] [--removeIpAddresses] [--domainBlockList value] [--xdiAllowedCommands value] [--transcendUrl value] - transcend consent build-xdi-sync-endpoint --help - -This command allows for building of the XDI Sync Endpoint across a set of Transcend accounts. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Consent Manager" - --xdiLocation The location of the XDI that will be loaded by the generated sync endpoint - [--file] The HTML file path where the sync endpoint should be written [default = ./sync-endpoint.html] - [--removeIpAddresses/--noRemoveIpAddresses] When true, remove IP addresses from the domain list [default = true] - [--domainBlockList] The set of domains that should be excluded from the sync endpoint. Comma-separated list. [default = localhost] - [--xdiAllowedCommands] The allowed set of XDI commands [default = ConsentManager:Sync] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Build XDI sync endpoint** - -```sh -transcend consent build-xdi-sync-endpoint --auth="$TRANSCEND_API_KEY" --xdiLocation=https://cdn.your-site.com/xdi.js -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend consent build-xdi-sync-endpoint \ - --auth="$TRANSCEND_API_KEY" \ - --xdiLocation=https://cdn.your-site.com/xdi.js \ - --transcendUrl=https://api.us.transcend.io -``` - -**Pull to specific file location** - -```sh -transcend consent build-xdi-sync-endpoint \ - --auth="$TRANSCEND_API_KEY" \ - --xdiLocation=https://cdn.your-site.com/xdi.js \ - --file=./my-folder/sync-endpoint.html -``` - -**Don't filter out regular expressions** - -```sh -transcend consent build-xdi-sync-endpoint \ - --auth="$TRANSCEND_API_KEY" \ - --xdiLocation=https://cdn.your-site.com/xdi.js \ - --removeIpAddresses=false -``` - -**Filter out certain domains that should not be included in the sync endpoint definition** - -```sh -transcend consent build-xdi-sync-endpoint \ - --auth="$TRANSCEND_API_KEY" \ - --xdiLocation=https://cdn.your-site.com/xdi.js \ - --domainBlockList=ignored.com,localhost -``` - -**Override XDI allowed commands** - -```sh -transcend consent build-xdi-sync-endpoint \ - --auth="$TRANSCEND_API_KEY" \ - --xdiLocation=https://cdn.your-site.com/xdi.js \ - --xdiAllowedCommands=ExtractIdentifiers:Simple -``` - -**Configuring across multiple Transcend Instances** - -```sh -# Pull down API keys across all Transcend instances -transcend admin generate-api-keys \ - --email="$TRANSCEND_EMAIL" \ - --password="$TRANSCEND_PASSWORD" \ - --transcendUrl=https://api.us.transcend.io \ - --scopes="View Consent Manager" \ - --apiKeyTitle="[cli][$TRANSCEND_EMAIL] XDI Endpoint Construction" \ - --file=./api-keys.json \ - --parentOrganizationId=1821d872-6114-406e-90c3-73b4d5e246cf - -# Path list of API keys as authentication -transcend consent build-xdi-sync-endpoint \ - --auth=./api-keys.json \ - --xdiLocation=https://cdn.your-site.com/xdi.js \ - --transcendUrl=https://api.us.transcend.io -``` - -### `transcend consent pull-consent-metrics` - -```txt -USAGE - transcend consent pull-consent-metrics (--auth value) (--start value) [--end value] [--folder value] [--bin value] [--transcendUrl value] - transcend consent pull-consent-metrics --help - -This command allows for pulling consent manager metrics for a Transcend account, or a set of Transcend accounts. - -By default, the consent metrics will be written to a folder named `consent-metrics` within the directory where you run the command. You can override the location that these CSVs are written to using the flag `--folder=./my-folder/`. This folder will contain a set of CSV files: - -- `CONSENT_CHANGES_TIMESERIES_optIn.csv` -> this is a feed containing the number of explicit opt in events that happen - these are calls to `airgap.setConsent(event, { SaleOfInfo: true });` -- `CONSENT_CHANGES_TIMESERIES_optOut.csv` -> this is a feed containing the number of explicit opt out events that happen - these are calls to `airgap.setConsent(event, { SaleOfInfo: false });` -- `CONSENT_SESSIONS_BY_REGIME_Default.csv` -> this contains the number of sessions detected for the bin period -- `PRIVACY_SIGNAL_TIMESERIES_DNT.csv` -> the number of DNT signals detected. -- `PRIVACY_SIGNAL_TIMESERIES_GPC.csv` -> the number of GPC signals detected. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Consent Manager" - --start The start date to pull metrics from - [--end] The end date to pull metrics until - [--folder] The folder to save metrics to [default = ./consent-metrics/] - [--bin] The bin metric when pulling data (1h or 1d) [default = 1d] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Pull consent manager metrics for a Transcend account** - -```sh -transcend consent pull-consent-metrics --auth="$TRANSCEND_API_KEY" --start=2024-01-01T00:00:00.000Z -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend consent pull-consent-metrics \ - --auth="$TRANSCEND_API_KEY" \ - --start=2024-01-01T00:00:00.000Z \ - --transcendUrl=https://api.us.transcend.io -``` - -**Pull start and end date explicitly** - -```sh -transcend consent pull-consent-metrics \ - --auth="$TRANSCEND_API_KEY" \ - --start=2024-01-01T00:00:00.000Z \ - --end=2024-03-01T00:00:00.000Z -``` - -**Save to an explicit folder** - -```sh -transcend consent pull-consent-metrics \ - --auth="$TRANSCEND_API_KEY" \ - --start=2024-01-01T00:00:00.000Z \ - --end=2024-03-01T00:00:00.000Z \ - --folder=./my-folder/ -``` - -**Bin data hourly vs daily** - -```sh -transcend consent pull-consent-metrics --auth="$TRANSCEND_API_KEY" --start=2024-01-01T00:00:00.000Z --bin=1h -``` - -### `transcend consent pull-consent-preferences` - -```txt -USAGE - transcend consent pull-consent-preferences (--auth value) (--partition value) [--sombraAuth value] [--file value] [--transcendUrl value] [--timestampBefore value] [--timestampAfter value] [--identifiers value]... [--concurrency value] - transcend consent pull-consent-preferences --help - -This command allows for pull of consent preferences from the Managed Consent Database. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Managed Consent Database Admin API" - --partition The partition key to download consent preferences to - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--file] Path to the CSV file to save preferences to [default = ./preferences.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--timestampBefore] Filter for consents updated this time - [--timestampAfter] Filter for consents updated after this time - [--identifiers]... Filter for specific identifiers [separator = ,] - [--concurrency] The concurrency to use when downloading consents in parallel [default = 100] - -h --help Print help information and exit -``` - -#### Examples - -**Fetch all consent preferences from partition key** - -```sh -transcend consent pull-consent-preferences \ - --auth="$TRANSCEND_API_KEY" \ - --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 -``` - -**Fetch all consent preferences from partition key and save to ./consent.csv** - -```sh -transcend consent pull-consent-preferences \ - --auth="$TRANSCEND_API_KEY" \ - --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 \ - --file=./consent.csv -``` - -**Filter on consent updates before a date** - -```sh -transcend consent pull-consent-preferences \ - --auth="$TRANSCEND_API_KEY" \ - --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 \ - --timestampBefore=2024-04-03T00:00:00.000Z -``` - -**Filter on consent updates after a date** - -```sh -transcend consent pull-consent-preferences \ - --auth="$TRANSCEND_API_KEY" \ - --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 \ - --timestampAfter=2024-04-03T00:00:00.000Z -``` - -**For self-hosted sombras that use an internal key** - -```sh -transcend consent pull-consent-preferences \ - --auth="$TRANSCEND_API_KEY" \ - --sombraAuth="$SOMBRA_INTERNAL_KEY" \ - --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend consent pull-consent-preferences \ - --auth="$TRANSCEND_API_KEY" \ - --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 \ - --transcendUrl=https://api.us.transcend.io -``` - -### `transcend consent update-consent-manager` - -```txt -USAGE - transcend consent update-consent-manager (--auth value) (--bundleTypes PRODUCTION|TEST) [--deploy] [--transcendUrl value] - transcend consent update-consent-manager --help - -This command allows for updating Consent Manager to latest version. The Consent Manager bundle can also be deployed using this command. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Consent Manager Developer Settings" - --bundleTypes The bundle types to deploy. Defaults to PRODUCTION,TEST. [PRODUCTION|TEST, separator = ,] - [--deploy] When true, deploy the Consent Manager after updating the version [default = false] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Update Consent Manager to latest version** - -```sh -transcend consent update-consent-manager --auth="$TRANSCEND_API_KEY" -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend consent update-consent-manager --auth="$TRANSCEND_API_KEY" --transcendUrl=https://api.us.transcend.io -``` - -**Update version and deploy bundles** - -```sh -transcend consent update-consent-manager --auth="$TRANSCEND_API_KEY" --deploy -``` - -**Update just the TEST bundle** - -```sh -transcend consent update-consent-manager --auth="$TRANSCEND_API_KEY" --bundleTypes=TEST -``` - -**Update just the PRODUCTION bundle** - -```sh -transcend consent update-consent-manager --auth="$TRANSCEND_API_KEY" --bundleTypes=PRODUCTION -``` - -**Update multiple organizations at once** - -```sh -transcend admin generate-api-keys \ - --email=test@transcend.io \ - --password="$TRANSCEND_PASSWORD" \ - --scopes="Manage Consent Manager" \ - --apiKeyTitle="CLI Usage Cross Instance Sync" \ - --file=./transcend-api-keys.json -transcend consent update-consent-manager --auth=./transcend-api-keys.json --deploy -``` - -### `transcend consent upload-consent-preferences` - -```txt -USAGE - transcend consent upload-consent-preferences (--base64EncryptionKey value) (--base64SigningKey value) (--partition value) [--file value] [--consentUrl value] [--concurrency value] - transcend consent upload-consent-preferences --help - -This command allows for updating of consent preferences to the Managed Consent Database. - -FLAGS - --base64EncryptionKey The encryption key used to encrypt the userId - --base64SigningKey The signing key used to prove authentication of consent request - --partition The partition key to download consent preferences to - [--file] The file to pull consent preferences from [default = ./preferences.csv] - [--consentUrl] URL of the Transcend consent backend. Use https://consent.us.transcend.io for US hosting [default = https://consent.transcend.io] - [--concurrency] The concurrency to use when uploading requests in parallel [default = 100] - -h --help Print help information and exit -``` - -#### Examples - -**Upload consent preferences to partition key** - -```sh -transcend consent upload-consent-preferences \ - --base64EncryptionKey="$TRANSCEND_CONSENT_ENCRYPTION_KEY" \ - --base64SigningKey="$TRANSCEND_CONSENT_SIGNING_KEY" \ - --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 -``` - -**Upload consent preferences to partition key from file** - -```sh -transcend consent upload-consent-preferences \ - --base64EncryptionKey="$TRANSCEND_CONSENT_ENCRYPTION_KEY" \ - --base64SigningKey="$TRANSCEND_CONSENT_SIGNING_KEY" \ - --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 \ - --file=./consent.csv -``` - -**Upload consent preferences to partition key and set concurrency** - -```sh -transcend consent upload-consent-preferences \ - --base64EncryptionKey="$TRANSCEND_CONSENT_ENCRYPTION_KEY" \ - --base64SigningKey="$TRANSCEND_CONSENT_SIGNING_KEY" \ - --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 \ - --concurrency=200 -``` - -### `transcend consent upload-cookies-from-csv` - -```txt -USAGE - transcend consent upload-cookies-from-csv (--auth value) (--trackerStatus LIVE|NEEDS_REVIEW) [--file value] [--transcendUrl value] - transcend consent upload-cookies-from-csv --help - -Upload cookies from CSV. This command allows for uploading of cookies from CSV. - -Step 1) Download the CSV of cookies that you want to edit from the Admin Dashboard under [Consent Management -> Cookies](https://app.transcend.io/consent-manager/cookies). You can download cookies from both the "Triage" and "Approved" tabs. - -Step 2) You can edit the contents of the CSV file as needed. You may adjust the "Purpose" column, adjust the "Notes" column, add "Owners" and "Teams" or even add custom columns with additional metadata. - -Step 3) Upload the modified CSV file back into the dashboard with this command. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Data Flows" - --trackerStatus The status of the cookies you will upload. [LIVE|NEEDS_REVIEW] - [--file] Path to the CSV file to upload [default = ./cookies.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Upload the file of cookies in ./cookies.csv into the "Approved" tab** - -```sh -transcend consent upload-cookies-from-csv --auth="$TRANSCEND_API_KEY" --trackerStatus=LIVE -``` - -**Upload the file of cookies in ./cookies.csv into the "Triage" tab** - -```sh -transcend consent upload-cookies-from-csv --auth="$TRANSCEND_API_KEY" --trackerStatus=NEEDS_REVIEW -``` - -**Specifying the CSV file to read from** - -```sh -transcend consent upload-cookies-from-csv \ - --auth="$TRANSCEND_API_KEY" \ - --trackerStatus=LIVE \ - --file=./custom/my-cookies.csv -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend consent upload-cookies-from-csv \ - --auth="$TRANSCEND_API_KEY" \ - --trackerStatus=LIVE \ - --transcendUrl=https://api.us.transcend.io -``` - -### `transcend consent upload-data-flows-from-csv` - -```txt -USAGE - transcend consent upload-data-flows-from-csv (--auth value) (--trackerStatus LIVE|NEEDS_REVIEW) [--file value] [--classifyService] [--transcendUrl value] - transcend consent upload-data-flows-from-csv --help - -Upload data flows from CSV. This command allows for uploading of data flows from CSV. - -Step 1) Download the CSV of data flows that you want to edit from the Admin Dashboard under [Consent Management -> Data Flows](https://app.transcend.io/consent-manager/data-flows). You can download data flows from both the "Triage" and "Approved" tabs. - -Step 2) You can edit the contents of the CSV file as needed. You may adjust the "Purpose" column, adjust the "Notes" column, add "Owners" and "Teams" or even add custom columns with additional metadata. - -Step 3) Upload the modified CSV file back into the dashboard with this command. - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Data Flows" - --trackerStatus The status of the data flows you will upload. [LIVE|NEEDS_REVIEW] - [--file] Path to the CSV file to upload [default = ./data-flows.csv] - [--classifyService] When true, automatically assign the service for a data flow based on the domain that is specified [default = false] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -To get a CSV of data flows, you can download the data flows from the Admin Dashboard under [Consent Management -> Data Flows](https://app.transcend.io/consent-manager/data-flows). You can download data flows from both the "Triage" and "Approved" tabs. - -export-data-flows - -#### Examples - -**Upload the file of data flows in ./data-flows.csv into the "Approved" tab** - -```sh -transcend consent upload-data-flows-from-csv --auth="$TRANSCEND_API_KEY" --trackerStatus=LIVE -``` - -**Upload the file of data flows in ./data-flows.csv into the "Triage" tab** - -```sh -transcend consent upload-data-flows-from-csv --auth="$TRANSCEND_API_KEY" --trackerStatus=NEEDS_REVIEW -``` - -**Specifying the CSV file to read from** - -```sh -transcend consent upload-data-flows-from-csv \ - --auth="$TRANSCEND_API_KEY" \ - --trackerStatus=LIVE \ - --file=./custom/my-data-flows.csv -``` - -**Have Transcend automatically fill in the service names by looking up the data flow host in Transcend's database** - -```sh -transcend consent upload-data-flows-from-csv --auth="$TRANSCEND_API_KEY" --trackerStatus=LIVE --classifyService -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend consent upload-data-flows-from-csv \ - --auth="$TRANSCEND_API_KEY" \ - --trackerStatus=LIVE \ - --transcendUrl=https://api.us.transcend.io -``` - -### `transcend consent upload-preferences` - -```txt -USAGE - transcend consent upload-preferences (--auth value) (--partition value) [--sombraAuth value] [--transcendUrl value] [--file value] [--directory value] [--dryRun] [--skipExistingRecordCheck] [--receiptFileDir value] [--skipWorkflowTriggers] [--forceTriggerWorkflows] [--skipConflictUpdates] [--isSilent] [--attributes value] [--receiptFilepath value] [--concurrency value] - transcend consent upload-preferences --help - -Upload preference management data to your Preference Store. - -This command prompts you to map the shape of the CSV to the shape of the Transcend API. There is no requirement for the shape of the incoming CSV, as the script will handle the mapping process. - -The script will also produce a JSON cache file that allows for the mappings to be preserved between runs. - -FLAGS - --auth The Transcend API key. Requires scopes: "Modify User Stored Preferences", "View Managed Consent Database Admin API", "View Preference Store Settings" - --partition The partition key to download consent preferences to - [--sombraAuth] The Sombra internal key, use for additional authentication when self-hosting Sombra - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--file] Path to the CSV file to load preferences from - [--directory] Path to the directory of CSV files to load preferences from - [--dryRun] Whether to do a dry run only - will write results to receiptFilepath without updating Transcend [default = false] - [--skipExistingRecordCheck] Whether to skip the check for existing records. SHOULD ONLY BE USED FOR INITIAL UPLOAD [default = false] - [--receiptFileDir] Directory path where the response receipts should be saved [default = ./receipts] - [--skipWorkflowTriggers] Whether to skip workflow triggers when uploading to preference store [default = false] - [--forceTriggerWorkflows] Whether to force trigger workflows for existing consent records [default = false] - [--skipConflictUpdates] Whether to skip uploading of any records where the preference store and file have a hard conflict [default = false] - [--isSilent/--noIsSilent] Whether to skip sending emails in workflows [default = true] - [--attributes] Attributes to add to any DSR request if created. Comma-separated list of key:value pairs. [default = Tags:transcend-cli,Source:transcend-cli] - [--receiptFilepath] Store resulting, continuing where left off [default = ./preference-management-upload-receipts.json] - [--concurrency] The concurrency to use when uploading in parallel [default = 10] - -h --help Print help information and exit -``` - -A sample CSV can be found [here](./examples/cli-upload-preferences-example.csv). In this example, `Sales` and `Marketing` are two custom Purposes, and `SalesCommunications` and `MarketingCommunications` are Preference Topics. During the interactive CLI prompt, you can map these columns to the slugs stored in Transcend! - -#### Examples - -**Upload consent preferences to partition key `4d1c5daa-90b7-4d18-aa40-f86a43d2c726`** - -```sh -transcend consent upload-preferences \ - --auth="$TRANSCEND_API_KEY" \ - --file=./preferences.csv \ - --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 -``` - -**Upload consent preferences with additional options** - -```sh -transcend consent upload-preferences \ - --auth="$TRANSCEND_API_KEY" \ - --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 \ - --file=./preferences.csv \ - --dryRun \ - --skipWorkflowTriggers \ - --skipConflictUpdates \ - --isSilent=false \ - --attributes=Tags:transcend-cli,Source:transcend-cli \ - --receiptFilepath=./preference-management-upload-receipts.json -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend consent upload-preferences \ - --auth="$TRANSCEND_API_KEY" \ - --partition=4d1c5daa-90b7-4d18-aa40-f86a43d2c726 \ - --file=./preferences.csv \ - --transcendUrl=https://api.us.transcend.io -``` - -### `transcend inventory pull` - -```txt -USAGE - transcend inventory pull (--auth value) [--resources all|apiKeys|customFields|templates|dataSilos|enrichers|dataFlows|businessEntities|processingActivities|actions|dataSubjects|identifiers|cookies|consentManager|partitions|prompts|promptPartials|promptGroups|agents|agentFunctions|agentFiles|vendors|dataCategories|processingPurposes|actionItems|actionItemCollections|teams|privacyCenters|policies|messages|assessments|assessmentTemplates|purposes] [--file value] [--transcendUrl value] [--dataSiloIds value]... [--integrationNames value]... [--trackerStatuses LIVE|NEEDS_REVIEW] [--pageSize value] [--skipDatapoints] [--skipSubDatapoints] [--includeGuessedCategories] [--debug] - transcend inventory pull --help - -Generates a transcend.yml by pulling the configuration from your Transcend instance. - -The API key needs various scopes depending on the resources being pulled (see the CLI's README for more details). - -This command can be helpful if you are looking to: - -- Copy your data into another instance -- Generate a transcend.yml file as a starting point to maintain parts of your data inventory in code. - -FLAGS - --auth The Transcend API key. The scopes required will vary depending on the operation performed. If in doubt, the Full Admin scope will always work. - [--resources] The different resource types to pull in. Defaults to dataSilos,enrichers,templates,apiKeys. [all|apiKeys|customFields|templates|dataSilos|enrichers|dataFlows|businessEntities|processingActivities|actions|dataSubjects|identifiers|cookies|consentManager|partitions|prompts|promptPartials|promptGroups|agents|agentFunctions|agentFiles|vendors|dataCategories|processingPurposes|actionItems|actionItemCollections|teams|privacyCenters|policies|messages|assessments|assessmentTemplates|purposes, separator = ,] - [--file] Path to the YAML file to pull into [default = ./transcend.yml] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--dataSiloIds]... The UUIDs of the data silos that should be pulled into the YAML file [separator = ,] - [--integrationNames]... The types of integrations to pull down [separator = ,] - [--trackerStatuses] The statuses of consent manager trackers to pull down. Defaults to all statuses. [LIVE|NEEDS_REVIEW, separator = ,] - [--pageSize] The page size to use when paginating over the API [default = 50] - [--skipDatapoints] When true, skip pulling in datapoints alongside data silo resource [default = false] - [--skipSubDatapoints] When true, skip pulling in subDatapoints alongside data silo resource [default = false] - [--includeGuessedCategories] When true, included guessed data categories that came from the content classifier [default = false] - [--debug] Set to true to include debug logs while pulling the configuration [default = false] - -h --help Print help information and exit -``` - -#### Scopes - -The API key permissions for this command vary based on the `resources` argument: - -| Resource | Description | Scopes | Link | -| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| apiKeys | API Key definitions assigned to Data Silos. API keys cannot be created through the CLI, but you can map API key usage to Data Silos. | View API Keys | [Developer Tools -> API keys](https://app.transcend.io/infrastructure/api-keys) | -| customFields | Custom field definitions that define extra metadata for each table in the Admin Dashboard. | View Global Attributes | [Custom Fields](https://app.transcend.io/infrastructure/attributes) | -| templates | Email templates. Only template titles can be created and mapped to other resources. | View Email Templates | [DSR Automation -> Email Templates](https://app.transcend.io/privacy-requests/email-templates) | -| dataSilos | The Data Silo/Integration definitions. | View Data Map, View Data Subject Request Settings | [Data Inventory -> Data Silos](https://app.transcend.io/data-map/data-inventory/) and [Infrastucture -> Integrations](https://app.transcend.io/infrastructure/integrationsdata-silos) | -| enrichers | The Privacy Request enricher configurations. | View Identity Verification Settings | [DSR Automation -> Identifiers](https://app.transcend.io/privacy-requests/identifiers) | -| dataFlows | Consent Manager Data Flow definitions. | View Data Flows | [Consent Management -> Data Flows](https://app.transcend.io/consent-manager/data-flows/approved) | -| businessEntities | The business entities in the data inventory. | View Data Inventory | [Data Inventory -> Business Entities](https://app.transcend.io/data-map/data-inventory/business-entities) | -| processingActivities | The processing activities in the data inventory. | View Data Inventory | [Data Inventory -> Processing Activities](https://app.transcend.io/data-map/data-inventory/processing-activities) | -| actions | The Privacy Request action settings. | View Data Subject Request Settings | [DSR Automation -> Request Settings](https://app.transcend.io/privacy-requests/settings) | -| dataSubjects | The Privacy Request data subject settings. | View Data Subject Request Settings | [DSR Automation -> Request Settings](https://app.transcend.io/privacy-requests/settings) | -| identifiers | The Privacy Request identifier configurations. | View Identity Verification Settings | [DSR Automation -> Identifiers](https://app.transcend.io/privacy-requests/identifiers) | -| cookies | Consent Manager Cookie definitions. | View Data Flows | [Consent Management -> Cookies](https://app.transcend.io/consent-manager/cookies/approved) | -| consentManager | Consent Manager general settings, including domain list. | View Consent Manager | [Consent Management -> Developer Settings](https://app.transcend.io/consent-manager/developer-settings) | -| partitions | The partitions in the account (often representative of separate data controllers). | View Consent Manager | [Consent Management -> Developer Settings -> Advanced Settings](https://app.transcend.io/consent-manager/developer-settings/advanced-settings) | -| prompts | The Transcend AI prompts | View Prompts | [Prompt Manager -> Browse](https://app.transcend.io/prompts/browse) | -| promptPartials | The Transcend AI prompt partials | View Prompts | [Prompt Manager -> Partials](https://app.transcend.io/prompts/partialss) | -| promptGroups | The Transcend AI prompt groups | View Prompts | [Prompt Manager -> Groups](https://app.transcend.io/prompts/groups) | -| agents | The agents in the prompt manager. | View Prompts | [Prompt Manager -> Agents](https://app.transcend.io/prompts/agents) | -| agentFunctions | The agent functions in the prompt manager. | View Prompts | [Prompt Manager -> Agent Functions](https://app.transcend.io/prompts/agent-functions) | -| agentFiles | The agent files in the prompt manager. | View Prompts | [Prompt Manager -> Agent Files](https://app.transcend.io/prompts/agent-files) | -| vendors | The vendors in the data inventory. | View Data Inventory | [Data Inventory -> Vendors](https://app.transcend.io/data-map/data-inventory/vendors) | -| dataCategories | The data categories in the data inventory. | View Data Inventory | [Data Inventory -> Data Categories](https://app.transcend.io/data-map/data-inventory/data-categories) | -| processingPurposes | The processing purposes in the data inventory. | View Data Inventory | [Data Inventory -> Processing Purposes](https://app.transcend.io/data-map/data-inventory/purposes) | -| actionItems | Onboarding related action items | View All Action Items | [Action Items](https://app.transcend.io/action-items/all) | -| actionItemCollections | Onboarding related action item group names | View All Action Items | [Action Items](https://app.transcend.io/action-items/all) | -| teams | Team definitions of users and scope groupings | View Scopes | [Administration -> Teams](https://app.transcend.io/admin/teams) | -| privacyCenters | The privacy center configurations. | View Privacy Center Layout | [Privacy Center](https://app.transcend.io/privacy-center/general-settings) | -| policies | The privacy center policies. | View Policies | [Privacy Center -> Policies](https://app.transcend.io/privacy-center/policies) | -| messages | Message definitions used across consent, privacy center, email templates and more. | View Internationalization Messages | [Privacy Center -> Messages](https://app.transcend.io/privacy-center/messages-internationalization), [Consent Management -> Display Settings -> Messages](https://app.transcend.io/consent-manager/display-settings/messages) | -| assessments | Assessment responses. | View Assessments | [Assessments -> Assessments](https://app.transcend.io/assessments/groups) | -| assessmentTemplates | Assessment template configurations. | View Assessments | [Assessment -> Templates](https://app.transcend.io/assessments/form-templates) | -| purposes | Consent purposes and related preference management topics. | View Consent Manager, View Preference Store Settings | [Consent Management -> Regional Experiences -> Purposes](https://app.transcend.io/consent-manager/regional-experiences/purposes) | - -#### Examples - -**Write out file to ./transcend.yml** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" -``` - -**Write out file to custom location** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --file=./custom/location.yml -``` - -**Pull specific data silo by ID** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --dataSiloIds=710fec3c-7bcc-4c9e-baff-bf39f9bec43e -``` - -**Pull specific types of data silos** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --integrationNames=salesforce,snowflake -``` - -**Pull specific resource types** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=apiKeys,templates,dataSilos,enrichers -``` - -**Pull data flows and cookies with specific tracker statuses (see [this example](./examples/data-flows-cookies.yml))** - -```sh -transcend inventory pull \ - --auth="$TRANSCEND_API_KEY" \ - --resources=dataFlows,cookies \ - --trackerStatuses=NEEDS_REVIEW,LIVE -``` - -**Pull data silos without datapoint information** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=dataSilos --skipDatapoints -``` - -**Pull data silos without subdatapoint information** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=dataSilos --skipSubDatapoints -``` - -**Pull data silos with guessed categories** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=dataSilos --includeGuessedCategories -``` - -**Pull custom field definitions only (see [this example](./examples/attributes.yml))** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=customFields -``` - -**Pull business entities only (see [this example](./examples/business-entities.yml))** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=businessEntities -``` - -**Pull processing activities only (see [this example](./examples/processing-activities.yml))** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=processingActivities -``` - -**Pull enrichers and identifiers (see [this example](./examples/enrichers.yml))** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=enrichers,identifiers -``` - -**Pull onboarding action items (see [this example](./examples/action-items.yml))** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=actionItems,actionItemCollections -``` - -**Pull consent manager domain list (see [this example](./examples/consent-manager-domains.yml))** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=consentManager -``` - -**Pull identifier configurations (see [this example](./examples/identifiers.yml))** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=identifiers -``` - -**Pull request actions configurations (see [this example](./examples/actions.yml))** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=actions -``` - -**Pull consent manager purposes and preference management topics (see [this example](./examples/purposes.yml))** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=purposes -``` - -**Pull data subject configurations (see [this example](./examples/data-subjects.yml))** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=dataSubjects -``` - -**Pull assessments and assessment templates** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=assessments,assessmentTemplates -``` - -**Pull everything** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=all -``` - -**Pull configuration files across multiple instances** - -```sh -transcend admin generate-api-keys \ - --email=test@transcend.io \ - --password="$TRANSCEND_PASSWORD" \ - --scopes="View Consent Manager" \ - --apiKeyTitle="CLI Usage Cross Instance Sync" \ - --file=./transcend-api-keys.json -transcend inventory pull --auth=./transcend-api-keys.json --resources=consentManager --file=./transcend/ -``` - -Note: This command will overwrite the existing transcend.yml file that you have locally. - -### `transcend inventory push` - -```txt -USAGE - transcend inventory push (--auth value) [--file value] [--transcendUrl value] [--pageSize value] [--variables value] [--publishToPrivacyCenter] [--classifyService] [--deleteExtraAttributeValues] - transcend inventory push --help - -Given a transcend.yml file, sync the contents up to your Transcend instance. - -FLAGS - --auth The Transcend API key. The scopes required will vary depending on the operation performed. If in doubt, the Full Admin scope will always work. - [--file] Path to the YAML file to push from [default = ./transcend.yml] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--pageSize] The page size to use when paginating over the API [default = 50] - [--variables] The variables to template into the YAML file when pushing configuration. Comma-separated list of key:value pairs. [default = ""] - [--publishToPrivacyCenter] When true, publish the configuration to the Privacy Center [default = false] - [--classifyService] When true, automatically assign the service for a data flow based on the domain that is specified [default = false] - [--deleteExtraAttributeValues] When true and syncing attributes, delete any extra attributes instead of just upserting [default = false] - -h --help Print help information and exit -``` - -#### Scopes - -The scopes for `transcend inventory push` are the same as the scopes for [`transcend inventory pull`](#transcend-inventory-pull). - -#### Examples - -**Looks for file at ./transcend.yml** - -```sh -transcend inventory push --auth="$TRANSCEND_API_KEY" -``` - -**Looks for file at custom location ./custom/location.yml** - -```sh -transcend inventory push --auth="$TRANSCEND_API_KEY" --file=./custom/location.yml -``` - -**Apply service classifier to all data flows** - -```sh -transcend inventory push --auth="$TRANSCEND_API_KEY" --classifyService -``` - -**Push up attributes, deleting any attributes that are not specified in the transcend.yml file** - -```sh -transcend inventory push --auth="$TRANSCEND_API_KEY" --deleteExtraAttributeValues -``` - -**Use dynamic variables to fill out parameters in YAML files (see [./examples/multi-instance.yml](./examples/multi-instance.yml))** - -```sh -transcend inventory push --auth="$TRANSCEND_API_KEY" --variables=domain:acme.com,stage:staging -``` - -**Push a single .yml file configuration into multiple Transcend instances** - -This uses the output of [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys). - -```sh -transcend admin generate-api-keys \ - --email=test@transcend.io \ - --password="$TRANSCEND_PASSWORD" \ - --scopes="View Email Templates,View Data Map" \ - --apiKeyTitle="CLI Usage Cross Instance Sync" \ - --file=./transcend-api-keys.json -transcend inventory pull --auth="$TRANSCEND_API_KEY" -transcend inventory push --auth=./transcend-api-keys.json -``` - -**Push multiple .yml file configurations into multiple Transcend instances** - -This uses the output of [`transcend admin generate-api-keys`](#transcend-admin-generate-api-keys). - -```sh -transcend admin generate-api-keys \ - --email=test@transcend.io \ - --password="$TRANSCEND_PASSWORD" \ - --scopes="View Email Templates,View Data Map" \ - --apiKeyTitle="CLI Usage Cross Instance Sync" \ - --file=./transcend-api-keys.json -transcend inventory pull --auth=./transcend-api-keys.json --file=./transcend/ -# -transcend inventory push --auth=./transcend-api-keys.json --file=./transcend/ -``` - -**Apply service classifier to all data flows** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=dataFlows -transcend inventory push --auth="$TRANSCEND_API_KEY" --classifyService -``` - -**Push up attributes, deleting any attributes that are not specified in the transcend.yml file** - -```sh -transcend inventory pull --auth="$TRANSCEND_API_KEY" --resources=customFields -transcend inventory push --auth="$TRANSCEND_API_KEY" --deleteExtraAttributeValues -``` - -Some things to note about this sync process: - -1. Any field that is defined in your .yml file will be synced up to app.transcend.io. If any change was made on the Admin Dashboard, it will be overwritten. -2. If you omit a field from the .yml file, this field will not be synced. This gives you the ability to define as much or as little configuration in your transcend.yml file as you would like, and let the remainder of fields be labeled through the Admin Dashboard -3. If you define new data subjects, identifiers, data silos or datapoints that were not previously defined on the Admin Dashboard, the CLI will create these new resources automatically. -4. Currently, this CLI does not handle deleting or renaming of resources. If you need to delete or rename a data silo, identifier, enricher or API key, you should make the change on the Admin Dashboard. -5. The only resources that this CLI will not auto-generate are: - -- a) Data silo owners: If you assign an email address to a data silo, you must first make sure that user is invited into your Transcend instance (https://app.transcend.io/admin/users). -- b) API keys: This CLI will not create new API keys. You will need to first create the new API keys on the Admin Dashboard (https://app.transcend.io/infrastructure/api-keys). You can then list out the titles of the API keys that you generated in your transcend.yml file, after which the CLI is capable of updating that API key to be able to respond to different data silos in your Data Map - -#### CI Integration - -Once you have a workflow for creating your transcend.yml file, you will want to integrate your `transcend inventory push` command on your CI. - -Below is an example of how to set this up using a Github action: - -```yaml -name: Transcend Data Map Syncing -# See https://app.transcend.io/privacy-requests/connected-services - -on: - push: - branches: - - 'main' - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Setup Node.js - uses: actions/setup-node@v2 - with: - node-version: '16' - - - name: Install Transcend CLI - run: npm install --global @transcend-io/cli - - # If you have a script that generates your transcend.yml file from - # an ORM or infrastructure configuration, add that step here - # Leave this step commented out if you want to manage your transcend.yml manually - # - name: Generate transcend.yml - # run: ./scripts/generate_transcend_yml.py - - - name: Push Transcend config - run: transcend inventory push --auth=${{ secrets.TRANSCEND_API_KEY }} -``` - -#### Dynamic Variables - -If you are using this CLI to sync your Data Map between multiple Transcend instances, you may find the need to make minor modifications to your configurations between environments. The most notable difference would be the domain where your webhook URLs are hosted on. - -The `transcend inventory push` command takes in a parameter `variables`. This is a CSV of `key:value` pairs. - -This command could fill out multiple parameters in a YAML file like [./examples/multi-instance.yml](./examples/multi-instance.yml), copied below: - -```yml -api-keys: - - title: Webhook Key -enrichers: - - title: Basic Identity Enrichment - description: Enrich an email address to the userId and phone number - # The data silo webhook URL is the same in each environment, - # except for the base domain in the webhook URL. - url: https://example.<>/transcend-enrichment-webhook - input-identifier: email - output-identifiers: - - userId - - phone - - myUniqueIdentifier - - title: Fraud Check - description: Ensure the email address is not marked as fraudulent - url: https://example.<>/transcend-fraud-check - input-identifier: email - output-identifiers: - - email - privacy-actions: - - ERASURE -data-silos: - - title: Redshift Data Warehouse - integrationName: server - description: The mega-warehouse that contains a copy over all SQL backed databases - <> - url: https://example.<>/transcend-webhook - api-key-title: Webhook Key -``` - -### `transcend inventory scan-packages` - -```txt -USAGE - transcend inventory scan-packages (--auth value) [--scanPath value] [--ignoreDirs value]... [--repositoryName value] [--transcendUrl value] - transcend inventory scan-packages --help - -Transcend scans packages and dependencies for the following frameworks: - -- package.json -- requirements.txt & setup.py -- Podfile -- Package.resolved -- build.gradle -- pubspec.yaml -- Gemfile & .gemspec -- composer.json - -This command will scan the folder you point at to look for any of these files. Once found, the build file will be parsed in search of dependencies. Those code packages and dependencies will be uploaded to Transcend. The information uploaded to Transcend is: - -- repository name -- package names -- dependency names and versions -- package descriptions - -FLAGS - --auth The Transcend API key. Requires scopes: "Manage Code Scanning" - [--scanPath] File path in the project to scan [default = ./] - [--ignoreDirs]... List of directories to ignore in scan [separator = ,] - [--repositoryName] Name of the git repository that the package should be tied to - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Scan the current directory** - -```sh -transcend inventory scan-packages --auth="$TRANSCEND_API_KEY" -``` - -**Scan a specific directory** - -```sh -transcend inventory scan-packages --auth="$TRANSCEND_API_KEY" --scanPath=./examples/ -``` - -**Ignore certain folders** - -```sh -transcend inventory scan-packages --auth="$TRANSCEND_API_KEY" --ignoreDirs=./test,./build -``` - -**Specify the name of the repository** - -```sh -transcend inventory scan-packages --auth="$TRANSCEND_API_KEY" --repositoryName=transcend-io/test -``` - -### `transcend inventory discover-silos` - -```txt -USAGE - transcend inventory discover-silos (--scanPath value) (--dataSiloId value) (--auth value) [--fileGlobs value] [--ignoreDirs value] [--transcendUrl value] - transcend inventory discover-silos --help - -We support scanning for new data silos in JavaScript, Python, Gradle, and CocoaPods projects. - -To get started, add a data silo for the corresponding project type with the "silo discovery" plugin enabled. For example, if you want to scan a JavaScript project, add a package.json data silo. Then, specify the data silo ID in the "--dataSiloId" parameter. - -FLAGS - --scanPath File path in the project to scan - --dataSiloId The UUID of the corresponding data silo - --auth The Transcend API key. This key must be associated with the data silo(s) being operated on. Requires scopes: "Manage Assigned Data Inventory" - [--fileGlobs] You can pass a glob syntax pattern(s) to specify additional file paths to scan. Comma-separated list of globs. [default = ""] - [--ignoreDirs] Comma-separated list of directories to ignore. [default = ""] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Scan a JavaScript package.json** - -```sh -transcend inventory discover-silos \ - --scanPath=./myJavascriptProject \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=445ee241-5f2a-477b-9948-2a3682a43d0e -``` - -**Scan multiple file types (Podfile, Gradle, etc.) in examples directory** - -```sh -transcend inventory discover-silos \ - --scanPath=./examples/ \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloId=b6776589-0b7d-466f-8aad-4378ffd3a321 -``` - -This call will look for all the package.json files in the scan path `./myJavascriptProject`, parse each of the dependencies into their individual package names, and send it to our Transcend backend for classification. These classifications can then be viewed [here](https://app.transcend.io/data-map/data-inventory/silo-discovery/triage). The process is the same for scanning requirements.txt, podfiles and build.gradle files. - -Here are some examples of a [Podfile](./examples/code-scanning/test-cocoa-pods/Podfile) and [Gradle file](./examples/code-scanning/test-gradle/build.gradle). - -### `transcend inventory pull-datapoints` - -```txt -USAGE - transcend inventory pull-datapoints (--auth value) [--file value] [--transcendUrl value] [--dataSiloIds value]... [--includeAttributes] [--includeGuessedCategories] [--parentCategories FINANCIAL|HEALTH|CONTACT|LOCATION|DEMOGRAPHIC|ID|ONLINE_ACTIVITY|USER_PROFILE|SOCIAL_MEDIA|CONNECTION|TRACKING|DEVICE|SURVEY|OTHER|UNSPECIFIED|NOT_PERSONAL_DATA|INTEGRATION_IDENTIFIER] [--subCategories value]... - transcend inventory pull-datapoints --help - -Export the datapoints from your Data Inventory into a CSV. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Data Inventory" - [--file] The file to save datapoints to [default = ./datapoints.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--dataSiloIds]... List of data silo IDs to filter by [separator = ,] - [--includeAttributes] Whether to include attributes in the output [default = false] - [--includeGuessedCategories] Whether to include guessed categories in the output [default = false] - [--parentCategories] List of parent categories to filter by [FINANCIAL|HEALTH|CONTACT|LOCATION|DEMOGRAPHIC|ID|ONLINE_ACTIVITY|USER_PROFILE|SOCIAL_MEDIA|CONNECTION|TRACKING|DEVICE|SURVEY|OTHER|UNSPECIFIED|NOT_PERSONAL_DATA|INTEGRATION_IDENTIFIER, separator = ,] - [--subCategories]... List of subcategories to filter by [separator = ,] - -h --help Print help information and exit -``` - -#### Examples - -**All arguments** - -```sh -transcend inventory pull-datapoints \ - --auth="$TRANSCEND_API_KEY" \ - --file=./datapoints.csv \ - --includeGuessedCategories \ - --parentCategories=CONTACT,ID,LOCATION \ - --subCategories=79d998b7-45dd-481c-ae3a-856fd93458b2,9ecc213a-cd46-46d6-afd9-46cea713f5d1 \ - --dataSiloIds=f956ccce-5534-4328-a78d-3a924b1fe429 -``` - -**Pull datapoints for specific data silos** - -```sh -transcend inventory pull-datapoints \ - --auth="$TRANSCEND_API_KEY" \ - --file=./datapoints.csv \ - --dataSiloIds=f956ccce-5534-4328-a78d-3a924b1fe429 -``` - -**Include attributes in the output** - -```sh -transcend inventory pull-datapoints --auth="$TRANSCEND_API_KEY" --file=./datapoints.csv --includeAttributes -``` - -**Include guessed categories in the output** - -```sh -transcend inventory pull-datapoints --auth="$TRANSCEND_API_KEY" --file=./datapoints.csv --includeGuessedCategories -``` - -**Filter by parent categories** - -```sh -transcend inventory pull-datapoints \ - --auth="$TRANSCEND_API_KEY" \ - --file=./datapoints.csv \ - --parentCategories=ID,LOCATION -``` - -**Filter by subcategories** - -```sh -transcend inventory pull-datapoints \ - --auth="$TRANSCEND_API_KEY" \ - --file=./datapoints.csv \ - --subCategories=79d998b7-45dd-481c-ae3a-856fd93458b2,9ecc213a-cd46-46d6-afd9-46cea713f5d1 -``` - -**Specify the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend inventory pull-datapoints \ - --auth="$TRANSCEND_API_KEY" \ - --file=./datapoints.csv \ - --transcendUrl=https://api.us.transcend.io -``` - -### `transcend inventory pull-unstructured-discovery-files` - -```txt -USAGE - transcend inventory pull-unstructured-discovery-files (--auth value) [--file value] [--transcendUrl value] [--dataSiloIds value]... [--subCategories value]... [--status MANUALLY_ADDED|CORRECTED|VALIDATED|CLASSIFIED|REJECTED] [--includeEncryptedSnippets] - transcend inventory pull-unstructured-discovery-files --help - -This command allows for pulling Unstructured Discovery into a CSV. - -FLAGS - --auth The Transcend API key. Requires scopes: "View Data Inventory" - [--file] The file to save datapoints to [default = ./unstructured-discovery-files.csv] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--dataSiloIds]... List of data silo IDs to filter by [separator = ,] - [--subCategories]... List of data categories to filter by [separator = ,] - [--status] List of classification statuses to filter by [MANUALLY_ADDED|CORRECTED|VALIDATED|CLASSIFIED|REJECTED, separator = ,] - [--includeEncryptedSnippets] Whether to include encrypted snippets of the entries classified [default = false] - -h --help Print help information and exit -``` - -#### Examples - -**All arguments** - -```sh -transcend inventory pull-unstructured-discovery-files \ - --auth="$TRANSCEND_API_KEY" \ - --file=./unstructured-discovery-files.csv \ - --transcendUrl=https://api.us.transcend.io \ - --dataSiloIds=f956ccce-5534-4328-a78d-3a924b1fe429 \ - --subCategories=79d998b7-45dd-481c-ae3a-856fd93458b2,9ecc213a-cd46-46d6-afd9-46cea713f5d1 \ - --status=VALIDATED,MANUALLY_ADDED,CORRECTED \ - --includeEncryptedSnippets -``` - -**Specify the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend inventory pull-unstructured-discovery-files \ - --auth="$TRANSCEND_API_KEY" \ - --transcendUrl=https://api.us.transcend.io -``` - -**Pull entries for specific data silos** - -```sh -transcend inventory pull-unstructured-discovery-files \ - --auth="$TRANSCEND_API_KEY" \ - --dataSiloIds=f956ccce-5534-4328-a78d-3a924b1fe429 -``` - -**Filter by data categories** - -```sh -transcend inventory pull-unstructured-discovery-files \ - --auth="$TRANSCEND_API_KEY" \ - --subCategories=79d998b7-45dd-481c-ae3a-856fd93458b2,9ecc213a-cd46-46d6-afd9-46cea713f5d1 -``` - -**Filter by classification status (exclude unconfirmed recommendations)** - -```sh -transcend inventory pull-unstructured-discovery-files \ - --auth="$TRANSCEND_API_KEY" \ - --status=VALIDATED,MANUALLY_ADDED,CORRECTED -``` - -**Filter by classification status (include rejected recommendations)** - -```sh -transcend inventory pull-unstructured-discovery-files --auth="$TRANSCEND_API_KEY" --status=REJECTED -``` - -### `transcend inventory derive-data-silos-from-data-flows` - -```txt -USAGE - transcend inventory derive-data-silos-from-data-flows (--auth value) (--dataFlowsYmlFolder value) (--dataSilosYmlFolder value) [--ignoreYmls value]... [--transcendUrl value] - transcend inventory derive-data-silos-from-data-flows --help - -Given a folder of data flow transcend.yml configurations, convert those configurations to set of data silo transcend.yml configurations. - -FLAGS - --auth The Transcend API key. No scopes are required for this command. - --dataFlowsYmlFolder The folder that contains data flow yml files - --dataSilosYmlFolder The folder that contains data silo yml files - [--ignoreYmls]... The set of yml files that should be skipped when uploading [separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Convert data flow configurations in folder to data silo configurations in folder** - -```sh -transcend inventory derive-data-silos-from-data-flows \ - --auth="$TRANSCEND_API_KEY" \ - --dataFlowsYmlFolder=./working/data-flows/ \ - --dataSilosYmlFolder=./working/data-silos/ -``` - -**Use with US backend** - -```sh -transcend inventory derive-data-silos-from-data-flows \ - --auth="$TRANSCEND_API_KEY" \ - --dataFlowsYmlFolder=./working/data-flows/ \ - --dataSilosYmlFolder=./working/data-silos/ \ - --transcendUrl=https://api.us.transcend.io -``` - -**Skip a set of yml files** - -```sh -transcend inventory derive-data-silos-from-data-flows \ - --auth="$TRANSCEND_API_KEY" \ - --dataFlowsYmlFolder=./working/data-flows/ \ - --dataSilosYmlFolder=./working/data-silos/ \ - --ignoreYmls=Skip.yml,Other.yml -``` - -### `transcend inventory derive-data-silos-from-data-flows-cross-instance` - -```txt -USAGE - transcend inventory derive-data-silos-from-data-flows-cross-instance (--auth value) (--dataFlowsYmlFolder value) [--output value] [--ignoreYmls value]... [--transcendUrl value] - transcend inventory derive-data-silos-from-data-flows-cross-instance --help - -Given a folder of data flow transcend.yml configurations, convert those configurations to a single transcend.yml configurations of all related data silos. - -FLAGS - --auth The Transcend API key. No scopes are required for this command. - --dataFlowsYmlFolder The folder that contains data flow yml files - [--output] The output transcend.yml file containing the data silo configurations [default = ./transcend.yml] - [--ignoreYmls]... The set of yml files that should be skipped when uploading [separator = ,] - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Convert data flow configurations in folder to data silo configurations in file** - -```sh -transcend inventory derive-data-silos-from-data-flows-cross-instance \ - --auth="$TRANSCEND_API_KEY" \ - --dataFlowsYmlFolder=./working/data-flows/ -``` - -**Use with US backend** - -```sh -transcend inventory derive-data-silos-from-data-flows-cross-instance \ - --auth="$TRANSCEND_API_KEY" \ - --dataFlowsYmlFolder=./working/data-flows/ \ - --transcendUrl=https://api.us.transcend.io -``` - -**Skip a set of yml files** - -```sh -transcend inventory derive-data-silos-from-data-flows-cross-instance \ - --auth="$TRANSCEND_API_KEY" \ - --dataFlowsYmlFolder=./working/data-flows/ \ - --ignoreYmls=Skip.yml,Other.yml -``` - -**Convert data flow configurations in folder to data silo configurations in file** - -```sh -transcend inventory derive-data-silos-from-data-flows-cross-instance \ - --auth="$TRANSCEND_API_KEY" \ - --dataFlowsYmlFolder=./working/data-flows/ \ - --output=./output.yml -``` - -### `transcend inventory consent-manager-service-json-to-yml` - -```txt -USAGE - transcend inventory consent-manager-service-json-to-yml [--file value] [--output value] - transcend inventory consent-manager-service-json-to-yml --help - -Import the services from an airgap.js file into a Transcend instance. - -1. Run `await airgap.getMetadata()` on a site with airgap -2. Right click on the printed object, and click `Copy object` -3. Place output of file in a file named `services.json` -4. Run: - - transcend inventory consent-manager-service-json-to-yml --file=./services.json --output=./transcend.yml - -5. Run: - - transcend inventory push --auth="$TRANSCEND_API_KEY" --file=./transcend.yml --classifyService - -FLAGS - [--file] Path to the services.json file, output of await airgap.getMetadata() [default = ./services.json] - [--output] Path to the output transcend.yml to write to [default = ./transcend.yml] - -h --help Print help information and exit -``` - -#### Examples - -**Convert data flow configurations in folder to yml in ./transcend.yml** - -```sh -transcend inventory consent-manager-service-json-to-yml -``` - -**With file locations** - -```sh -transcend inventory consent-manager-service-json-to-yml --file=./folder/services.json --output=./folder/transcend.yml -``` - -### `transcend inventory consent-managers-to-business-entities` - -```txt -USAGE - transcend inventory consent-managers-to-business-entities (--consentManagerYmlFolder value) [--output value] - transcend inventory consent-managers-to-business-entities --help - -This command allows for converting a folder or Consent Manager transcend.yml files into a single transcend.yml file where each consent manager configuration is a Business Entity in the data inventory. - -FLAGS - --consentManagerYmlFolder Path to the folder of Consent Manager transcend.yml files to combine - [--output] Path to the output transcend.yml with business entity configuration [default = ./combined-business-entities.yml] - -h --help Print help information and exit -``` - -#### Examples - -**Combine files in folder to file ./combined-business-entities.yml** - -```sh -transcend inventory consent-managers-to-business-entities --consentManagerYmlFolder=./working/consent-managers/ -``` - -**Specify custom output file** - -```sh -transcend inventory consent-managers-to-business-entities \ - --consentManagerYmlFolder=./working/consent-managers/ \ - --output=./custom.yml -``` - -### `transcend admin generate-api-keys` - -```txt -USAGE - transcend admin generate-api-keys (--email value) (--password value) (--apiKeyTitle value) (--file value) (--scopes View Only|Full Admin|Rotate Hosted Sombra keys|Manage Global Attributes|Manage Access Controls|Manage Billing|Manage SSO|Manage API Keys|Manage Organization Information|Manage Email Domains|Manage Data Sub Categories|View Customer Data in Privacy Requests|View Customer Data in Data Mapping|View API Keys|View Audit Events|View SSO|View Scopes|View All Action Items|Manage All Action Items|View Employees|View Email Domains|View Global Attributes|View Legal Hold|Manage Legal Holds|Manage Request Security|Manage Request Compilation|Manage Assigned Privacy Requests|Submit New Data Subject Request|Manage Data Subject Request Settings|Manage Email Templates|Manage Request Identity Verification|Publish Privacy Center|Manage Data Map|Manage Privacy Center Layout|Manage Policies|View Policies|Manage Internationalization Messages|View Internationalization Messages|Request Approval and Communication|View Data Subject Request Settings|View the Request Compilation|View Identity Verification Settings|View Incoming Requests|View Assigned Privacy Requests|View Privacy Center Layout|View Email Templates|Connect Data Silos|Manage Data Inventory|Manage Assigned Data Inventory|Manage Assigned Integrations|View Data Map|View Assigned Integrations|View Assigned Data Inventory|View Data Inventory|Manage Consent Manager|Manage Consent Manager Developer Settings|Manage Consent Manager Display Settings|Deploy Test Consent Manager|Deploy Consent Manager|Manage Assigned Consent Manager|Manage Data Flows|View Data Flows|View Assigned Consent Manager|View Consent Manager|View Assessments|Manage Assessments|View Assigned Assessments|Manage Assigned Assessments|View Pathfinder|Manage Pathfinder|View Contract Scanning|Manage Contract Scanning|View Prompts|Manage Prompts|View Prompt Runs|Manage Prompt Runs|View Code Scanning|Manage Code Scanning|Execute Prompt|View Auditor Runs|Manage Auditor Runs and Schedules|Execute Auditor|Approve Prompts|Manage Action Item Collections|View Managed Consent Database Admin API|Modify User Stored Preferences|Manage Preference Store Settings|View Preference Store Settings|LLM Log Transfer|Manage Workflows|View Data Sub Categories) [--deleteExistingApiKey] [--createNewApiKey] [--parentOrganizationId value] [--transcendUrl value] - transcend admin generate-api-keys --help - -This command allows for creating API keys across multiple Transcend instances. This is useful for customers that are managing many Transcend instances and need to regularly create, cycle or delete API keys across all of their instances. - -Unlike the other commands that rely on API key authentication, this command relies upon username/password authentication. This command will spit out the API keys into a JSON file, and that JSON file can be used in subsequent CLI commands. - -Authentication requires your email and password for the Transcend account. This command will only generate API keys for Transcend instances where you have the permission to "Manage API Keys". - -FLAGS - --email The email address that you use to log into Transcend - --password The password for your account login - --apiKeyTitle The title of the API key being generated or destroyed - --file The file where API keys should be written to - --scopes The list of scopes that should be given to the API key [View Only|Full Admin|Rotate Hosted Sombra keys|Manage Global Attributes|Manage Access Controls|Manage Billing|Manage SSO|Manage API Keys|Manage Organization Information|Manage Email Domains|Manage Data Sub Categories|View Customer Data in Privacy Requests|View Customer Data in Data Mapping|View API Keys|View Audit Events|View SSO|View Scopes|View All Action Items|Manage All Action Items|View Employees|View Email Domains|View Global Attributes|View Legal Hold|Manage Legal Holds|Manage Request Security|Manage Request Compilation|Manage Assigned Privacy Requests|Submit New Data Subject Request|Manage Data Subject Request Settings|Manage Email Templates|Manage Request Identity Verification|Publish Privacy Center|Manage Data Map|Manage Privacy Center Layout|Manage Policies|View Policies|Manage Internationalization Messages|View Internationalization Messages|Request Approval and Communication|View Data Subject Request Settings|View the Request Compilation|View Identity Verification Settings|View Incoming Requests|View Assigned Privacy Requests|View Privacy Center Layout|View Email Templates|Connect Data Silos|Manage Data Inventory|Manage Assigned Data Inventory|Manage Assigned Integrations|View Data Map|View Assigned Integrations|View Assigned Data Inventory|View Data Inventory|Manage Consent Manager|Manage Consent Manager Developer Settings|Manage Consent Manager Display Settings|Deploy Test Consent Manager|Deploy Consent Manager|Manage Assigned Consent Manager|Manage Data Flows|View Data Flows|View Assigned Consent Manager|View Consent Manager|View Assessments|Manage Assessments|View Assigned Assessments|Manage Assigned Assessments|View Pathfinder|Manage Pathfinder|View Contract Scanning|Manage Contract Scanning|View Prompts|Manage Prompts|View Prompt Runs|Manage Prompt Runs|View Code Scanning|Manage Code Scanning|Execute Prompt|View Auditor Runs|Manage Auditor Runs and Schedules|Execute Auditor|Approve Prompts|Manage Action Item Collections|View Managed Consent Database Admin API|Modify User Stored Preferences|Manage Preference Store Settings|View Preference Store Settings|LLM Log Transfer|Manage Workflows|View Data Sub Categories, separator = ,] - [--deleteExistingApiKey/--noDeleteExistingApiKey] When true, if an API key exists with the specified apiKeyTitle, the existing API key is deleted [default = true] - [--createNewApiKey/--noCreateNewApiKey] When true, new API keys will be created. Set to false if you simply want to delete all API keys with a title [default = true] - [--parentOrganizationId] Filter for only a specific organization by ID, returning all child accounts associated with that organization - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - -h --help Print help information and exit -``` - -#### Examples - -**Generate API keys for cross-instance usage** - -```sh -transcend admin generate-api-keys \ - --email=test@transcend.io \ - --password="$TRANSCEND_PASSWORD" \ - --scopes="View Email Templates,View Data Map" \ - --apiKeyTitle="CLI Usage Cross Instance Sync" \ - --file=./working/auth.json -``` - -**Specifying the backend URL, needed for US hosted backend infrastructure** - -```sh -transcend admin generate-api-keys \ - --email=test@transcend.io \ - --password="$TRANSCEND_PASSWORD" \ - --scopes="View Email Templates,View Data Map" \ - --apiKeyTitle="CLI Usage Cross Instance Sync" \ - --file=./working/auth.json \ - --transcendUrl=https://api.us.transcend.io -``` - -**Filter for only a specific organization by ID, returning all child accounts associated with that organization** - -```sh -transcend admin generate-api-keys \ - --email=test@transcend.io \ - --password="$TRANSCEND_PASSWORD" \ - --scopes="View Email Templates,View Data Map" \ - --apiKeyTitle="CLI Usage Cross Instance Sync" \ - --file=./working/auth.json \ - --parentOrganizationId=7098bb38-070d-4f26-8fa4-1b61b9cdef77 -``` - -**Delete all API keys with a certain title** - -```sh -transcend admin generate-api-keys \ - --email=test@transcend.io \ - --password="$TRANSCEND_PASSWORD" \ - --scopes="View Email Templates,View Data Map" \ - --apiKeyTitle="CLI Usage Cross Instance Sync" \ - --file=./working/auth.json \ - --createNewApiKey=false -``` - -**Throw error if an API key already exists with that title, default behavior is to delete the existing API key and create a new one with that same title** - -```sh -transcend admin generate-api-keys \ - --email=test@transcend.io \ - --password="$TRANSCEND_PASSWORD" \ - --scopes="View Email Templates,View Data Map" \ - --apiKeyTitle="CLI Usage Cross Instance Sync" \ - --file=./working/auth.json \ - --deleteExistingApiKey=false -``` - -**Find your organization ID** - -You can use the following GQL query on the [EU GraphQL Playground](https://api.us.transcend.io/graphql) or [US GraphQL Playground](https://api.us.transcend.io/graphql) to get your organization IDs and their parent/child relationships. - -```gql -query { - user { - organization { - id - parentOrganizationId - } - } -} -``` - -### `transcend admin chunk-csv` - -```txt -USAGE - transcend admin chunk-csv (--directory value) [--outputDir value] [--clearOutputDir] [--chunkSizeMB value] [--concurrency value] [--viewerMode] - transcend admin chunk-csv --help - -Streams every CSV in --directory and writes chunked files of approximately N MB each. -- Runs files in parallel across worker processes (configurable via --concurrency). -- Validates row-length consistency against the header row; logs periodic progress and memory usage. - -FLAGS - --directory Directory containing CSV files to split (required) - [--outputDir] Directory to write chunk files (defaults to each input file's directory) - [--clearOutputDir/--noClearOutputDir] Clear the output directory before writing chunks [default = true] - [--chunkSizeMB] Approximate chunk size in megabytes. Keep well under JS string size limits [default = 10] - [--concurrency] Max number of worker processes (defaults based on CPU and file count) - [--viewerMode] Run in non-interactive viewer mode (no attach UI, auto-artifacts) [default = false] - -h --help Print help information and exit -``` - -#### Examples - -**Chunk a file into smaller CSV files** - -```sh -transcend admin chunk-csv --directory=./working/files --outputDir=./working/chunks -``` - -**Specify chunk size in MB** - -```sh -transcend admin chunk-csv --directory=./working/files --outputDir=./working/chunks --chunkSizeMB=250 -``` - -**Specify concurrency (pool size)** - -```sh -transcend admin chunk-csv --directory=./working/files --outputDir=./working/chunks --concurrency=4 -``` - -**Viewer mode - no ability to switch between files** - -```sh -transcend admin chunk-csv --directory=./working/files --outputDir=./working/chunks --viewerMode -``` - -**Clear output directory before writing chunks** - -```sh -transcend admin chunk-csv --directory=./working/files --outputDir=./working/chunks --clearOutputDir -``` - -**Run with all options** - -```sh -transcend admin chunk-csv \ - --directory=./working/files \ - --outputDir=./working/chunks \ - --chunkSizeMB=100 \ - --concurrency=2 \ - --viewerMode=false \ - --clearOutputDir -``` - -**Run with no output directory specified (defaults to input directory)** - -```sh -transcend admin chunk-csv --directory=./working/files -``` - -### `transcend migration sync-ot` - -```txt -USAGE - transcend migration sync-ot [--hostname value] [--oneTrustAuth value] [--source oneTrust|file] [--transcendAuth value] [--transcendUrl value] [--file value] [--resource assessments] [--dryRun] [--debug] - transcend migration sync-ot --help - -Pulls resources from a OneTrust and syncs them to a Transcend instance. For now, it only supports retrieving OneTrust Assessments. - -This command can be helpful if you are looking to: -- Pull resources from your OneTrust account. -- Migrate your resources from your OneTrust account to Transcend. - -OneTrust authentication requires an OAuth Token with scope for accessing the assessment endpoints. -If syncing the resources to Transcend, you will also need to generate an API key on the Transcend Admin Dashboard. - -FLAGS - [--hostname] The domain of the OneTrust environment from which to pull the resource - [--oneTrustAuth] The OAuth access token with the scopes necessary to access the OneTrust Public APIs - [--source] Whether to read the assessments from OneTrust or from a file [oneTrust|file, default = oneTrust] - [--transcendAuth] The Transcend API key. Requires scopes: "Manage Assessments" - [--transcendUrl] URL of the Transcend backend. Use https://api.us.transcend.io for US hosting [default = https://api.transcend.io] - [--file] Path to the file to pull the resource into. Must be a json file! - [--resource] The resource to pull from OneTrust. For now, only assessments is supported [assessments, default = assessments] - [--dryRun] Whether to export the resource to a file rather than sync to Transcend [default = false] - [--debug] Whether to print detailed logs in case of error [default = false] - -h --help Print help information and exit -``` - -#### Authentication - -In order to use this command, you will need to generate a OneTrust OAuth Token with scope for accessing the following endpoints: - -- [GET /v2/assessments](https://developer.onetrust.com/onetrust/reference/getallassessmentbasicdetailsusingget) -- [GET /v2/assessments/{assessmentId}/export](https://developer.onetrust.com/onetrust/reference/exportassessmentusingget) -- [GET /risks/{riskId}](https://developer.onetrust.com/onetrust/reference/getriskusingget) -- [GET /v2/Users/{userId}](https://developer.onetrust.com/onetrust/reference/getuserusingget) - -To learn how to generate the token, see the [OAuth 2.0 Scopes](https://developer.onetrust.com/onetrust/reference/oauth-20-scopes) and [Generate Access Token](https://developer.onetrust.com/onetrust/reference/getoauthtoken) pages. - -#### Examples - -**Syncs all assessments from the OneTrust instance to Transcend** - -```sh -transcend migration sync-ot \ - --hostname=trial.onetrust.com \ - --oneTrustAuth="$ONE_TRUST_OAUTH_TOKEN" \ - --transcendAuth="$TRANSCEND_API_KEY" -``` - -**Set dryRun to true and sync the resource to disk (writes out file to ./oneTrustAssessments.json)** - -```sh -transcend migration sync-ot \ - --hostname=trial.onetrust.com \ - --oneTrustAuth="$ONE_TRUST_OAUTH_TOKEN" \ - --dryRun \ - --file=./oneTrustAssessments.json -``` - -**Sync to Transcend by reading from file instead of OneTrust** - -```sh -transcend migration sync-ot --source=file --file=./oneTrustAssessments.json --transcendAuth="$TRANSCEND_API_KEY" -``` - - - -## Prompt Manager - -If you are integrating Transcend's Prompt Manager into your code, it may look like: - -```ts -import * as t from 'io-ts'; -import { TranscendPromptManager } from '@transcend-io/cli'; -import { - ChatCompletionMessage, - PromptRunProductArea, -} from '@transcend-io/privacy-types'; - -/** - * Example prompt integration - */ -export async function main(): Promise { - // Instantiate the Transcend Prompt Manager instance - const promptManager = new TranscendPromptManager({ - // API key - transcendApiKey: process.env.TRANSCEND_API_KEY, - // Define the prompts that are stored in Transcend - prompts: { - test: { - // identify by ID - id: '30bcaa79-889a-4af3-842d-2e8ba443d36d', - // no runtime variables - paramCodec: t.type({}), - // response is list of strings - outputCodec: t.array(t.string), - }, - json: { - // identify by title - title: 'test', - // one runtime variable "test" - paramCodec: t.type({ test: t.string }), - // runtime is json object - outputCodec: t.record(t.string, t.string), - // response is stored in atg - extractFromTag: 'json', - }, - predictProductLine: { - // identify by title - title: 'Predict Product Line', - // runtime parameter for slack channel name - paramCodec: t.type({ - slackChannelName: t.string, - }), - // response is specific JSON shape - outputCodec: t.type({ - product: t.union([t.string, t.null]), - clarification: t.union([t.string, t.null]), - }), - // response is stored in atg - extractFromTag: 'json', - }, - }, - // Optional arguments - // transcendUrl: 'https://api.us.transcend.io', // defaults to 'https://api.transcend.io' - // requireApproval: false, // defaults to true - // cacheDuration: 1000 * 60 * 60, // defaults to undefined, no cache - // defaultVariables: { myVariable: 'this is custom', other: [{ name: 'custom' }] }, // defaults to {} - // handlebarsOptions: { helpers, templates }, // defaults to {} - }); - - // Fetch the prompt from Transcend and template any variables - // in this case, we template the slack channel name in the LLM prompt - const systemPrompt = await promptManager.compilePrompt('predictProductLine', { - slackChannelName: channelName, - }); - - // Parameters to pass to the LLM - const input: ChatCompletionMessage[] = [ - { - role: 'system', - content: systemPrompt, - }, - { - role: 'user', - content: input, - }, - ]; - const largeLanguageModel = { - name: 'gpt-4', - client: 'openai' as const, - }; - const temperature = 1; - const topP = 1; - const maxTokensToSample = 1000; - - // Run prompt against LLM - let response: string; - const t0 = new Date().getTime(); - try { - response = await openai.createCompletion(input, { - temperature, - top_p: topP, - max_tokens: maxTokensToSample, - }); - } catch (err) { - // report error upon failure - await promptManager.reportPromptRunError('predictProductLine', { - promptRunMessages: input, - duration: new Date().getTime() - t0, - temperature, - topP, - error: err.message, - maxTokensToSample, - largeLanguageModel, - }); - } - const t1 = new Date().getTime(); - - // Parsed response as JSON and do not report to Transcend - // const parsedResponse = promptManager.parseAiResponse( - // 'predictProductLine', - // response, - // ); - - // Parsed response as JSON and report output to Transcend - const parsedResponse = await promptManager.reportAndParsePromptRun( - 'predictProductLine', - { - promptRunMessages: [ - ...input, - { - role: 'assistant', - content: response, - }, - ], - duration: t1 - t0, - temperature, - topP, - maxTokensToSample, - largeLanguageModel, - // Optional parameters - // name, // unique identifier for this run - // productArea, // Transcend product area that the prompt relates to - // runByEmployeeEmail, // Employee email that is executing the request - // promptGroupId, // The prompt group being reported - }, - ); -} -``` - -## Proxy usage - -If you are trying to use the CLI inside a corporate firewall and need to send traffic through a proxy, you can do so via the `http_proxy` environment variable,with a command like `http_proxy=http://localhost:5051 transcend inventory pull --auth=$TRANSCEND_API_KEY`. diff --git a/examples/chats/generate_chats.ts b/examples/chats/generate_chats.ts new file mode 100644 index 00000000..f19bdac2 --- /dev/null +++ b/examples/chats/generate_chats.ts @@ -0,0 +1,145 @@ +#!/usr/bin/env node +// generate-support-chats.mjs +// +// Generates fake support chat transcripts with personal data categories. +// Creates N .txt files in the specified directory. +// +// Run with: +// node generate-support-chats.mjs --out ./support_chats --count 1000 + +import { mkdirSync, writeFileSync, existsSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +/** + * Generates a random integer between min and max (inclusive). + * + * @param min - Minimum value (inclusive) + * @param max - Maximum value (inclusive) + * @returns - Random integer between min and max + */ +function randomInt(min: number, max: number): number { + return Math.floor(Math.random() * (max - min + 1)) + min; +} +/** + * Picks a random element from an array. + * + * @param arr - Array to pick from + * @returns - Random element from the array + */ +function pick(arr: T[]): T { + return arr[randomInt(0, arr.length - 1)]; +} + +// ----- Data pools ----- +const agentNames = ['Sofia', 'Liam', 'Ava', 'Noah', 'Maya', 'Ethan']; +const customerNames = ['Jordan', 'Taylor', 'Sam', 'Alex', 'Casey', 'Riley']; + +const openings = [ + 'Hi, I need help with', + "Hello—I'm having trouble with", + 'Can you assist me with', + 'Good morning, question about', +]; + +const issues = [ + 'a double charge on my card', + 'logging into my account', + 'a late delivery', + 'resetting my password', + 'updating my subscription', + 'firmware update errors', +]; + +// Example personal data categories to sprinkle into chats +const personalData = [ + 'Social Security Number: 123-45-6789', + 'User ID: U123456', + 'Username: jordan_92', + 'Declared Interests: hiking, cooking', + 'Profile Picture URL: https://example.com/img/profile123.jpg', + 'User Preferences: marketing_emails=false', + 'Profile URL: https://social.example.com/jordan', + 'IP Address: 192.168.1.42', + 'MAC Address: 00:1B:44:11:3A:B7', + 'User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)', + 'Cookies: sessionid=abcd1234', + 'Serial Number: SN-1234567890', + 'Survey Data: Q1=Yes, Q2=No', + 'Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----', + 'Email: jordan@example.com', + 'Phone: +1-202-555-0172', + 'Approximate Geolocation: San Francisco, CA', + 'Date of Birth: 1990-05-12', + 'Gender: Non-binary', + 'Political Affiliation: Independent', + 'Race/Ethnicity: Hispanic', + 'Religion: None', + 'Sexual Orientation: Heterosexual', + 'Union Membership: None', + 'Biometric Identifier: FaceID registered', +]; + +/** + * Generate a single chat + * + * @param chatId - Unique identifier for the chat + * @returns - Formatted chat transcript as a string + */ +function generateChat(chatId: number): string { + const customer = pick(customerNames); + const agent = pick(agentNames); + const turns = randomInt(8, 18); + + const lines = []; + lines.push(`Support Chat Transcript #${chatId}`); + lines.push('=================================='); + + for (let i = 0; i < turns; i += 1) { + if (i % 2 === 0) { + // customer + const msg = + i === 0 + ? `${pick(openings)} ${pick(issues)}.` + : chance(0.2) + ? `Here is my ${pick(personalData)}` + : 'Can you check on that?'; + lines.push(`${customer}: ${msg}`); + } else { + // agent + const msg = chance(0.3) + ? `Thanks, can you also provide your ${pick(personalData)}?` + : 'Let me look that up for you.'; + lines.push(`${agent}: ${msg}`); + } + } + + return lines.join('\n'); +} + +/** + * Chance function to simulate probability. + * + * @param p - Probability of true (0-1) + * @returns - True with probability p + */ +function chance(p: number): boolean { + return Math.random() < p; +} + +const filename = fileURLToPath(import.meta.url); +const dir = dirname(filename); + +const OUT_DIR = join(dir, './support_chats'); +const COUNT = Number(process.env.COUNT || 1000); + +if (!existsSync(OUT_DIR)) mkdirSync(OUT_DIR, { recursive: true }); + +for (let i = 1; i <= COUNT; i += 1) { + const chat = generateChat(i); + const filename = join(OUT_DIR, `chat_${i.toString().padStart(4, '0')}.txt`); + writeFileSync(filename, chat, 'utf8'); +} + +const logger = console; +logger.log(`✅ Generated ${COUNT} support chat transcripts in ${OUT_DIR}`); diff --git a/examples/chats/support_chats/chat_0001.txt b/examples/chats/support_chats/chat_0001.txt new file mode 100644 index 00000000..ae61fafd --- /dev/null +++ b/examples/chats/support_chats/chat_0001.txt @@ -0,0 +1,16 @@ +Support Chat Transcript #1 +================================== +Taylor: Hi, I need help with logging into my account. +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Profile URL: https://social.example.com/jordan? +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Union Membership: None? +Taylor: Can you check on that? +Sofia: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0002.txt b/examples/chats/support_chats/chat_0002.txt new file mode 100644 index 00000000..ffc2cadb --- /dev/null +++ b/examples/chats/support_chats/chat_0002.txt @@ -0,0 +1,17 @@ +Support Chat Transcript #2 +================================== +Casey: Good morning, question about a late delivery. +Sofia: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7? +Casey: Can you check on that? +Sofia: Let me look that up for you. +Casey: Here is my Profile Picture URL: https://example.com/img/profile123.jpg +Sofia: Let me look that up for you. +Casey: Can you check on that? +Sofia: Let me look that up for you. +Casey: Can you check on that? +Sofia: Let me look that up for you. +Casey: Can you check on that? +Sofia: Let me look that up for you. +Casey: Can you check on that? +Sofia: Let me look that up for you. +Casey: Here is my Profile URL: https://social.example.com/jordan \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0003.txt b/examples/chats/support_chats/chat_0003.txt new file mode 100644 index 00000000..ba18f573 --- /dev/null +++ b/examples/chats/support_chats/chat_0003.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #3 +================================== +Sam: Good morning, question about logging into my account. +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0004.txt b/examples/chats/support_chats/chat_0004.txt new file mode 100644 index 00000000..7278eb14 --- /dev/null +++ b/examples/chats/support_chats/chat_0004.txt @@ -0,0 +1,10 @@ +Support Chat Transcript #4 +================================== +Sam: Good morning, question about updating my subscription. +Ava: Let me look that up for you. +Sam: Can you check on that? +Ava: Let me look that up for you. +Sam: Here is my IP Address: 192.168.1.42 +Ava: Let me look that up for you. +Sam: Can you check on that? +Ava: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0005.txt b/examples/chats/support_chats/chat_0005.txt new file mode 100644 index 00000000..322c9a85 --- /dev/null +++ b/examples/chats/support_chats/chat_0005.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #5 +================================== +Taylor: Hi, I need help with a late delivery. +Liam: Let me look that up for you. +Taylor: Here is my Survey Data: Q1=Yes, Q2=No +Liam: Let me look that up for you. +Taylor: Can you check on that? +Liam: Thanks, can you also provide your Profile URL: https://social.example.com/jordan? +Taylor: Can you check on that? +Liam: Let me look that up for you. +Taylor: Can you check on that? +Liam: Thanks, can you also provide your Race/Ethnicity: Hispanic? +Taylor: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0006.txt b/examples/chats/support_chats/chat_0006.txt new file mode 100644 index 00000000..8cfc6c69 --- /dev/null +++ b/examples/chats/support_chats/chat_0006.txt @@ -0,0 +1,11 @@ +Support Chat Transcript #6 +================================== +Riley: Hello—I'm having trouble with a double charge on my card. +Liam: Thanks, can you also provide your User ID: U123456? +Riley: Can you check on that? +Liam: Thanks, can you also provide your Sexual Orientation: Heterosexual? +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Here is my Race/Ethnicity: Hispanic +Liam: Let me look that up for you. +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0007.txt b/examples/chats/support_chats/chat_0007.txt new file mode 100644 index 00000000..7e2352bd --- /dev/null +++ b/examples/chats/support_chats/chat_0007.txt @@ -0,0 +1,15 @@ +Support Chat Transcript #7 +================================== +Riley: Hi, I need help with a double charge on my card. +Sofia: Let me look that up for you. +Riley: Can you check on that? +Sofia: Thanks, can you also provide your Cookies: sessionid=abcd1234? +Riley: Here is my Approximate Geolocation: San Francisco, CA +Sofia: Let me look that up for you. +Riley: Here is my Race/Ethnicity: Hispanic +Sofia: Let me look that up for you. +Riley: Can you check on that? +Sofia: Let me look that up for you. +Riley: Can you check on that? +Sofia: Thanks, can you also provide your Serial Number: SN-1234567890? +Riley: Here is my User Preferences: marketing_emails=false \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0008.txt b/examples/chats/support_chats/chat_0008.txt new file mode 100644 index 00000000..2681f7db --- /dev/null +++ b/examples/chats/support_chats/chat_0008.txt @@ -0,0 +1,10 @@ +Support Chat Transcript #8 +================================== +Casey: Hello—I'm having trouble with a late delivery. +Ethan: Thanks, can you also provide your Profile URL: https://social.example.com/jordan? +Casey: Can you check on that? +Ethan: Let me look that up for you. +Casey: Can you check on that? +Ethan: Thanks, can you also provide your Phone: +1-202-555-0172? +Casey: Can you check on that? +Ethan: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0009.txt b/examples/chats/support_chats/chat_0009.txt new file mode 100644 index 00000000..73464764 --- /dev/null +++ b/examples/chats/support_chats/chat_0009.txt @@ -0,0 +1,17 @@ +Support Chat Transcript #9 +================================== +Taylor: Good morning, question about firmware update errors. +Sofia: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----? +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Serial Number: SN-1234567890? +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Here is my MAC Address: 00:1B:44:11:3A:B7 +Sofia: Let me look that up for you. +Taylor: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0010.txt b/examples/chats/support_chats/chat_0010.txt new file mode 100644 index 00000000..048e1561 --- /dev/null +++ b/examples/chats/support_chats/chat_0010.txt @@ -0,0 +1,12 @@ +Support Chat Transcript #10 +================================== +Sam: Can you assist me with updating my subscription. +Ethan: Let me look that up for you. +Sam: Can you check on that? +Ethan: Let me look that up for you. +Sam: Can you check on that? +Ethan: Thanks, can you also provide your Survey Data: Q1=Yes, Q2=No? +Sam: Can you check on that? +Ethan: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? +Sam: Can you check on that? +Ethan: Thanks, can you also provide your Declared Interests: hiking, cooking? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0011.txt b/examples/chats/support_chats/chat_0011.txt new file mode 100644 index 00000000..1e5fa7f7 --- /dev/null +++ b/examples/chats/support_chats/chat_0011.txt @@ -0,0 +1,20 @@ +Support Chat Transcript #11 +================================== +Sam: Hi, I need help with firmware update errors. +Maya: Thanks, can you also provide your Religion: None? +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)? +Sam: Here is my Approximate Geolocation: San Francisco, CA +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Here is my Date of Birth: 1990-05-12 +Maya: Thanks, can you also provide your User ID: U123456? +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Here is my Email: jordan@example.com +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Thanks, can you also provide your Email: jordan@example.com? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0012.txt b/examples/chats/support_chats/chat_0012.txt new file mode 100644 index 00000000..b21ee812 --- /dev/null +++ b/examples/chats/support_chats/chat_0012.txt @@ -0,0 +1,12 @@ +Support Chat Transcript #12 +================================== +Sam: Good morning, question about a double charge on my card. +Ava: Thanks, can you also provide your Email: jordan@example.com? +Sam: Can you check on that? +Ava: Thanks, can you also provide your Political Affiliation: Independent? +Sam: Can you check on that? +Ava: Let me look that up for you. +Sam: Can you check on that? +Ava: Let me look that up for you. +Sam: Can you check on that? +Ava: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0013.txt b/examples/chats/support_chats/chat_0013.txt new file mode 100644 index 00000000..c44b0e1c --- /dev/null +++ b/examples/chats/support_chats/chat_0013.txt @@ -0,0 +1,19 @@ +Support Chat Transcript #13 +================================== +Sam: Good morning, question about updating my subscription. +Noah: Thanks, can you also provide your Profile URL: https://social.example.com/jordan? +Sam: Can you check on that? +Noah: Thanks, can you also provide your Union Membership: None? +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Here is my Serial Number: SN-1234567890 +Noah: Thanks, can you also provide your Union Membership: None? +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Here is my Social Security Number: 123-45-6789 +Noah: Thanks, can you also provide your IP Address: 192.168.1.42? +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Here is my MAC Address: 00:1B:44:11:3A:B7 \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0014.txt b/examples/chats/support_chats/chat_0014.txt new file mode 100644 index 00000000..ac4c0207 --- /dev/null +++ b/examples/chats/support_chats/chat_0014.txt @@ -0,0 +1,17 @@ +Support Chat Transcript #14 +================================== +Casey: Can you assist me with logging into my account. +Ava: Thanks, can you also provide your Race/Ethnicity: Hispanic? +Casey: Can you check on that? +Ava: Let me look that up for you. +Casey: Here is my Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY----- +Ava: Let me look that up for you. +Casey: Can you check on that? +Ava: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)? +Casey: Here is my Profile Picture URL: https://example.com/img/profile123.jpg +Ava: Let me look that up for you. +Casey: Here is my Race/Ethnicity: Hispanic +Ava: Let me look that up for you. +Casey: Can you check on that? +Ava: Let me look that up for you. +Casey: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0015.txt b/examples/chats/support_chats/chat_0015.txt new file mode 100644 index 00000000..6016f6e0 --- /dev/null +++ b/examples/chats/support_chats/chat_0015.txt @@ -0,0 +1,12 @@ +Support Chat Transcript #15 +================================== +Taylor: Hello—I'm having trouble with logging into my account. +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Phone: +1-202-555-0172? +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0016.txt b/examples/chats/support_chats/chat_0016.txt new file mode 100644 index 00000000..737438e3 --- /dev/null +++ b/examples/chats/support_chats/chat_0016.txt @@ -0,0 +1,12 @@ +Support Chat Transcript #16 +================================== +Riley: Can you assist me with updating my subscription. +Ethan: Let me look that up for you. +Riley: Can you check on that? +Ethan: Thanks, can you also provide your Profile Picture URL: https://example.com/img/profile123.jpg? +Riley: Can you check on that? +Ethan: Let me look that up for you. +Riley: Can you check on that? +Ethan: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)? +Riley: Can you check on that? +Ethan: Thanks, can you also provide your Social Security Number: 123-45-6789? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0017.txt b/examples/chats/support_chats/chat_0017.txt new file mode 100644 index 00000000..278f29f9 --- /dev/null +++ b/examples/chats/support_chats/chat_0017.txt @@ -0,0 +1,16 @@ +Support Chat Transcript #17 +================================== +Taylor: Hello—I'm having trouble with updating my subscription. +Maya: Let me look that up for you. +Taylor: Can you check on that? +Maya: Let me look that up for you. +Taylor: Here is my Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY----- +Maya: Thanks, can you also provide your Political Affiliation: Independent? +Taylor: Can you check on that? +Maya: Let me look that up for you. +Taylor: Can you check on that? +Maya: Let me look that up for you. +Taylor: Can you check on that? +Maya: Let me look that up for you. +Taylor: Can you check on that? +Maya: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0018.txt b/examples/chats/support_chats/chat_0018.txt new file mode 100644 index 00000000..67382fcc --- /dev/null +++ b/examples/chats/support_chats/chat_0018.txt @@ -0,0 +1,15 @@ +Support Chat Transcript #18 +================================== +Alex: Hi, I need help with resetting my password. +Ava: Let me look that up for you. +Alex: Can you check on that? +Ava: Let me look that up for you. +Alex: Here is my Sexual Orientation: Heterosexual +Ava: Let me look that up for you. +Alex: Can you check on that? +Ava: Let me look that up for you. +Alex: Can you check on that? +Ava: Let me look that up for you. +Alex: Can you check on that? +Ava: Let me look that up for you. +Alex: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0019.txt b/examples/chats/support_chats/chat_0019.txt new file mode 100644 index 00000000..0c5ad711 --- /dev/null +++ b/examples/chats/support_chats/chat_0019.txt @@ -0,0 +1,16 @@ +Support Chat Transcript #19 +================================== +Casey: Can you assist me with firmware update errors. +Noah: Let me look that up for you. +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Can you check on that? +Noah: Thanks, can you also provide your Sexual Orientation: Heterosexual? +Casey: Can you check on that? +Noah: Thanks, can you also provide your Email: jordan@example.com? +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Can you check on that? +Noah: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0020.txt b/examples/chats/support_chats/chat_0020.txt new file mode 100644 index 00000000..ad398f97 --- /dev/null +++ b/examples/chats/support_chats/chat_0020.txt @@ -0,0 +1,10 @@ +Support Chat Transcript #20 +================================== +Casey: Can you assist me with updating my subscription. +Maya: Thanks, can you also provide your Sexual Orientation: Heterosexual? +Casey: Can you check on that? +Maya: Let me look that up for you. +Casey: Can you check on that? +Maya: Thanks, can you also provide your Gender: Non-binary? +Casey: Can you check on that? +Maya: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0021.txt b/examples/chats/support_chats/chat_0021.txt new file mode 100644 index 00000000..de387e0d --- /dev/null +++ b/examples/chats/support_chats/chat_0021.txt @@ -0,0 +1,16 @@ +Support Chat Transcript #21 +================================== +Sam: Can you assist me with firmware update errors. +Sofia: Thanks, can you also provide your Political Affiliation: Independent? +Sam: Here is my Serial Number: SN-1234567890 +Sofia: Let me look that up for you. +Sam: Here is my Username: jordan_92 +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Here is my Union Membership: None +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0022.txt b/examples/chats/support_chats/chat_0022.txt new file mode 100644 index 00000000..788c595a --- /dev/null +++ b/examples/chats/support_chats/chat_0022.txt @@ -0,0 +1,11 @@ +Support Chat Transcript #22 +================================== +Sam: Hello—I'm having trouble with a double charge on my card. +Ethan: Let me look that up for you. +Sam: Can you check on that? +Ethan: Let me look that up for you. +Sam: Can you check on that? +Ethan: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7? +Sam: Can you check on that? +Ethan: Thanks, can you also provide your Username: jordan_92? +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0023.txt b/examples/chats/support_chats/chat_0023.txt new file mode 100644 index 00000000..6323f876 --- /dev/null +++ b/examples/chats/support_chats/chat_0023.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #23 +================================== +Taylor: Hello—I'm having trouble with firmware update errors. +Ethan: Let me look that up for you. +Taylor: Can you check on that? +Ethan: Thanks, can you also provide your IP Address: 192.168.1.42? +Taylor: Here is my Race/Ethnicity: Hispanic +Ethan: Let me look that up for you. +Taylor: Can you check on that? +Ethan: Let me look that up for you. +Taylor: Can you check on that? +Ethan: Let me look that up for you. +Taylor: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0024.txt b/examples/chats/support_chats/chat_0024.txt new file mode 100644 index 00000000..a2b9b7bd --- /dev/null +++ b/examples/chats/support_chats/chat_0024.txt @@ -0,0 +1,15 @@ +Support Chat Transcript #24 +================================== +Sam: Hello—I'm having trouble with updating my subscription. +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Here is my Gender: Non-binary +Sofia: Thanks, can you also provide your Political Affiliation: Independent? +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Here is my User ID: U123456 +Sofia: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)? +Sam: Can you check on that? +Sofia: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0025.txt b/examples/chats/support_chats/chat_0025.txt new file mode 100644 index 00000000..aec0b252 --- /dev/null +++ b/examples/chats/support_chats/chat_0025.txt @@ -0,0 +1,17 @@ +Support Chat Transcript #25 +================================== +Taylor: Can you assist me with logging into my account. +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Here is my Email: jordan@example.com +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Thanks, can you also provide your User Preferences: marketing_emails=false? +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0026.txt b/examples/chats/support_chats/chat_0026.txt new file mode 100644 index 00000000..3319bbf6 --- /dev/null +++ b/examples/chats/support_chats/chat_0026.txt @@ -0,0 +1,17 @@ +Support Chat Transcript #26 +================================== +Alex: Can you assist me with logging into my account. +Ava: Let me look that up for you. +Alex: Here is my Gender: Non-binary +Ava: Let me look that up for you. +Alex: Can you check on that? +Ava: Let me look that up for you. +Alex: Here is my MAC Address: 00:1B:44:11:3A:B7 +Ava: Let me look that up for you. +Alex: Can you check on that? +Ava: Let me look that up for you. +Alex: Can you check on that? +Ava: Let me look that up for you. +Alex: Here is my Approximate Geolocation: San Francisco, CA +Ava: Let me look that up for you. +Alex: Here is my Union Membership: None \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0027.txt b/examples/chats/support_chats/chat_0027.txt new file mode 100644 index 00000000..5b2821ea --- /dev/null +++ b/examples/chats/support_chats/chat_0027.txt @@ -0,0 +1,16 @@ +Support Chat Transcript #27 +================================== +Taylor: Can you assist me with resetting my password. +Sofia: Thanks, can you also provide your Declared Interests: hiking, cooking? +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Religion: None? +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Profile URL: https://social.example.com/jordan? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0028.txt b/examples/chats/support_chats/chat_0028.txt new file mode 100644 index 00000000..41c143a9 --- /dev/null +++ b/examples/chats/support_chats/chat_0028.txt @@ -0,0 +1,19 @@ +Support Chat Transcript #28 +================================== +Alex: Hi, I need help with logging into my account. +Sofia: Thanks, can you also provide your User Preferences: marketing_emails=false? +Alex: Can you check on that? +Sofia: Thanks, can you also provide your Phone: +1-202-555-0172? +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Thanks, can you also provide your Race/Ethnicity: Hispanic? +Alex: Here is my Survey Data: Q1=Yes, Q2=No +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Thanks, can you also provide your Biometric Identifier: FaceID registered? +Alex: Here is my Approximate Geolocation: San Francisco, CA +Sofia: Thanks, can you also provide your Social Security Number: 123-45-6789? +Alex: Here is my Username: jordan_92 \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0029.txt b/examples/chats/support_chats/chat_0029.txt new file mode 100644 index 00000000..a705b5a2 --- /dev/null +++ b/examples/chats/support_chats/chat_0029.txt @@ -0,0 +1,20 @@ +Support Chat Transcript #29 +================================== +Jordan: Hello—I'm having trouble with a double charge on my card. +Ava: Thanks, can you also provide your Serial Number: SN-1234567890? +Jordan: Can you check on that? +Ava: Let me look that up for you. +Jordan: Here is my Approximate Geolocation: San Francisco, CA +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? +Jordan: Can you check on that? +Ava: Thanks, can you also provide your Declared Interests: hiking, cooking? +Jordan: Can you check on that? +Ava: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0030.txt b/examples/chats/support_chats/chat_0030.txt new file mode 100644 index 00000000..4e57fea0 --- /dev/null +++ b/examples/chats/support_chats/chat_0030.txt @@ -0,0 +1,18 @@ +Support Chat Transcript #30 +================================== +Jordan: Hi, I need help with updating my subscription. +Maya: Let me look that up for you. +Jordan: Can you check on that? +Maya: Let me look that up for you. +Jordan: Can you check on that? +Maya: Let me look that up for you. +Jordan: Here is my MAC Address: 00:1B:44:11:3A:B7 +Maya: Let me look that up for you. +Jordan: Here is my Biometric Identifier: FaceID registered +Maya: Let me look that up for you. +Jordan: Can you check on that? +Maya: Let me look that up for you. +Jordan: Can you check on that? +Maya: Let me look that up for you. +Jordan: Can you check on that? +Maya: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0031.txt b/examples/chats/support_chats/chat_0031.txt new file mode 100644 index 00000000..f89f8278 --- /dev/null +++ b/examples/chats/support_chats/chat_0031.txt @@ -0,0 +1,16 @@ +Support Chat Transcript #31 +================================== +Taylor: Hi, I need help with logging into my account. +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Username: jordan_92? +Taylor: Here is my Gender: Non-binary +Sofia: Thanks, can you also provide your User ID: U123456? +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0032.txt b/examples/chats/support_chats/chat_0032.txt new file mode 100644 index 00000000..fcf5204d --- /dev/null +++ b/examples/chats/support_chats/chat_0032.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #32 +================================== +Sam: Can you assist me with updating my subscription. +Ava: Let me look that up for you. +Sam: Can you check on that? +Ava: Let me look that up for you. +Sam: Can you check on that? +Ava: Thanks, can you also provide your Gender: Non-binary? +Sam: Can you check on that? +Ava: Let me look that up for you. +Sam: Can you check on that? +Ava: Thanks, can you also provide your Biometric Identifier: FaceID registered? +Sam: Here is my Race/Ethnicity: Hispanic \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0033.txt b/examples/chats/support_chats/chat_0033.txt new file mode 100644 index 00000000..6c08ab3b --- /dev/null +++ b/examples/chats/support_chats/chat_0033.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #33 +================================== +Jordan: Can you assist me with firmware update errors. +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Thanks, can you also provide your Profile URL: https://social.example.com/jordan? +Jordan: Can you check on that? +Ava: Let me look that up for you. +Jordan: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0034.txt b/examples/chats/support_chats/chat_0034.txt new file mode 100644 index 00000000..30b48836 --- /dev/null +++ b/examples/chats/support_chats/chat_0034.txt @@ -0,0 +1,19 @@ +Support Chat Transcript #34 +================================== +Alex: Can you assist me with a late delivery. +Noah: Thanks, can you also provide your Date of Birth: 1990-05-12? +Alex: Can you check on that? +Noah: Thanks, can you also provide your Serial Number: SN-1234567890? +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Thanks, can you also provide your User ID: U123456? +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Here is my Email: jordan@example.com +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Here is my Serial Number: SN-1234567890 +Noah: Let me look that up for you. +Alex: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0035.txt b/examples/chats/support_chats/chat_0035.txt new file mode 100644 index 00000000..80eb6b77 --- /dev/null +++ b/examples/chats/support_chats/chat_0035.txt @@ -0,0 +1,12 @@ +Support Chat Transcript #35 +================================== +Jordan: Can you assist me with a double charge on my card. +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Thanks, can you also provide your Phone: +1-202-555-0172? +Jordan: Here is my Username: jordan_92 +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Thanks, can you also provide your Gender: Non-binary? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0036.txt b/examples/chats/support_chats/chat_0036.txt new file mode 100644 index 00000000..7dfef667 --- /dev/null +++ b/examples/chats/support_chats/chat_0036.txt @@ -0,0 +1,16 @@ +Support Chat Transcript #36 +================================== +Casey: Can you assist me with a double charge on my card. +Noah: Let me look that up for you. +Casey: Can you check on that? +Noah: Thanks, can you also provide your Gender: Non-binary? +Casey: Can you check on that? +Noah: Thanks, can you also provide your Sexual Orientation: Heterosexual? +Casey: Here is my Survey Data: Q1=Yes, Q2=No +Noah: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7? +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Here is my User ID: U123456 +Noah: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0037.txt b/examples/chats/support_chats/chat_0037.txt new file mode 100644 index 00000000..eeb63a6f --- /dev/null +++ b/examples/chats/support_chats/chat_0037.txt @@ -0,0 +1,19 @@ +Support Chat Transcript #37 +================================== +Taylor: Can you assist me with a double charge on my card. +Ethan: Thanks, can you also provide your Union Membership: None? +Taylor: Can you check on that? +Ethan: Let me look that up for you. +Taylor: Can you check on that? +Ethan: Thanks, can you also provide your Political Affiliation: Independent? +Taylor: Can you check on that? +Ethan: Let me look that up for you. +Taylor: Can you check on that? +Ethan: Thanks, can you also provide your User Preferences: marketing_emails=false? +Taylor: Here is my Approximate Geolocation: San Francisco, CA +Ethan: Let me look that up for you. +Taylor: Here is my Approximate Geolocation: San Francisco, CA +Ethan: Thanks, can you also provide your Username: jordan_92? +Taylor: Can you check on that? +Ethan: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7? +Taylor: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0038.txt b/examples/chats/support_chats/chat_0038.txt new file mode 100644 index 00000000..c5ccfc0c --- /dev/null +++ b/examples/chats/support_chats/chat_0038.txt @@ -0,0 +1,12 @@ +Support Chat Transcript #38 +================================== +Jordan: Hello—I'm having trouble with a late delivery. +Ethan: Let me look that up for you. +Jordan: Can you check on that? +Ethan: Let me look that up for you. +Jordan: Can you check on that? +Ethan: Let me look that up for you. +Jordan: Here is my Date of Birth: 1990-05-12 +Ethan: Let me look that up for you. +Jordan: Can you check on that? +Ethan: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0039.txt b/examples/chats/support_chats/chat_0039.txt new file mode 100644 index 00000000..79a77a62 --- /dev/null +++ b/examples/chats/support_chats/chat_0039.txt @@ -0,0 +1,19 @@ +Support Chat Transcript #39 +================================== +Riley: Good morning, question about firmware update errors. +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Thanks, can you also provide your Serial Number: SN-1234567890? +Riley: Can you check on that? +Liam: Thanks, can you also provide your Profile Picture URL: https://example.com/img/profile123.jpg? +Riley: Here is my Approximate Geolocation: San Francisco, CA +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Here is my Religion: None +Liam: Thanks, can you also provide your Username: jordan_92? +Riley: Can you check on that? +Liam: Thanks, can you also provide your Social Security Number: 123-45-6789? +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0040.txt b/examples/chats/support_chats/chat_0040.txt new file mode 100644 index 00000000..0a4c3f63 --- /dev/null +++ b/examples/chats/support_chats/chat_0040.txt @@ -0,0 +1,15 @@ +Support Chat Transcript #40 +================================== +Riley: Hello—I'm having trouble with firmware update errors. +Ethan: Thanks, can you also provide your Sexual Orientation: Heterosexual? +Riley: Here is my Approximate Geolocation: San Francisco, CA +Ethan: Let me look that up for you. +Riley: Here is my Survey Data: Q1=Yes, Q2=No +Ethan: Let me look that up for you. +Riley: Can you check on that? +Ethan: Let me look that up for you. +Riley: Can you check on that? +Ethan: Let me look that up for you. +Riley: Can you check on that? +Ethan: Thanks, can you also provide your Serial Number: SN-1234567890? +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0041.txt b/examples/chats/support_chats/chat_0041.txt new file mode 100644 index 00000000..5e0b9f75 --- /dev/null +++ b/examples/chats/support_chats/chat_0041.txt @@ -0,0 +1,19 @@ +Support Chat Transcript #41 +================================== +Alex: Good morning, question about a late delivery. +Liam: Let me look that up for you. +Alex: Can you check on that? +Liam: Let me look that up for you. +Alex: Can you check on that? +Liam: Let me look that up for you. +Alex: Can you check on that? +Liam: Let me look that up for you. +Alex: Can you check on that? +Liam: Let me look that up for you. +Alex: Can you check on that? +Liam: Let me look that up for you. +Alex: Can you check on that? +Liam: Thanks, can you also provide your Union Membership: None? +Alex: Can you check on that? +Liam: Thanks, can you also provide your Declared Interests: hiking, cooking? +Alex: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0042.txt b/examples/chats/support_chats/chat_0042.txt new file mode 100644 index 00000000..e474ea5f --- /dev/null +++ b/examples/chats/support_chats/chat_0042.txt @@ -0,0 +1,19 @@ +Support Chat Transcript #42 +================================== +Sam: Can you assist me with a double charge on my card. +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Here is my Username: jordan_92 +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0043.txt b/examples/chats/support_chats/chat_0043.txt new file mode 100644 index 00000000..5e8f3746 --- /dev/null +++ b/examples/chats/support_chats/chat_0043.txt @@ -0,0 +1,14 @@ +Support Chat Transcript #43 +================================== +Sam: Hi, I need help with updating my subscription. +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Here is my Email: jordan@example.com +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0044.txt b/examples/chats/support_chats/chat_0044.txt new file mode 100644 index 00000000..d8971bc0 --- /dev/null +++ b/examples/chats/support_chats/chat_0044.txt @@ -0,0 +1,15 @@ +Support Chat Transcript #44 +================================== +Sam: Good morning, question about a double charge on my card. +Liam: Let me look that up for you. +Sam: Here is my Gender: Non-binary +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Thanks, can you also provide your Political Affiliation: Independent? +Sam: Can you check on that? +Liam: Thanks, can you also provide your Political Affiliation: Independent? +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0045.txt b/examples/chats/support_chats/chat_0045.txt new file mode 100644 index 00000000..bb17b65b --- /dev/null +++ b/examples/chats/support_chats/chat_0045.txt @@ -0,0 +1,17 @@ +Support Chat Transcript #45 +================================== +Alex: Good morning, question about firmware update errors. +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Here is my Email: jordan@example.com +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Thanks, can you also provide your Email: jordan@example.com? +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Here is my Survey Data: Q1=Yes, Q2=No +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0046.txt b/examples/chats/support_chats/chat_0046.txt new file mode 100644 index 00000000..34d7639a --- /dev/null +++ b/examples/chats/support_chats/chat_0046.txt @@ -0,0 +1,18 @@ +Support Chat Transcript #46 +================================== +Alex: Can you assist me with updating my subscription. +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Here is my Social Security Number: 123-45-6789 +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0047.txt b/examples/chats/support_chats/chat_0047.txt new file mode 100644 index 00000000..6e1ca08c --- /dev/null +++ b/examples/chats/support_chats/chat_0047.txt @@ -0,0 +1,15 @@ +Support Chat Transcript #47 +================================== +Riley: Hi, I need help with a double charge on my card. +Maya: Let me look that up for you. +Riley: Can you check on that? +Maya: Thanks, can you also provide your Union Membership: None? +Riley: Can you check on that? +Maya: Let me look that up for you. +Riley: Can you check on that? +Maya: Let me look that up for you. +Riley: Here is my Approximate Geolocation: San Francisco, CA +Maya: Let me look that up for you. +Riley: Can you check on that? +Maya: Let me look that up for you. +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0048.txt b/examples/chats/support_chats/chat_0048.txt new file mode 100644 index 00000000..10000059 --- /dev/null +++ b/examples/chats/support_chats/chat_0048.txt @@ -0,0 +1,16 @@ +Support Chat Transcript #48 +================================== +Alex: Can you assist me with a double charge on my card. +Ava: Thanks, can you also provide your Political Affiliation: Independent? +Alex: Can you check on that? +Ava: Let me look that up for you. +Alex: Can you check on that? +Ava: Let me look that up for you. +Alex: Can you check on that? +Ava: Let me look that up for you. +Alex: Can you check on that? +Ava: Let me look that up for you. +Alex: Can you check on that? +Ava: Let me look that up for you. +Alex: Here is my Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY----- +Ava: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0049.txt b/examples/chats/support_chats/chat_0049.txt new file mode 100644 index 00000000..902872cc --- /dev/null +++ b/examples/chats/support_chats/chat_0049.txt @@ -0,0 +1,20 @@ +Support Chat Transcript #49 +================================== +Alex: Can you assist me with resetting my password. +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Thanks, can you also provide your Race/Ethnicity: Hispanic? +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Thanks, can you also provide your Biometric Identifier: FaceID registered? +Alex: Here is my Social Security Number: 123-45-6789 +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Thanks, can you also provide your Religion: None? +Alex: Can you check on that? +Noah: Thanks, can you also provide your Gender: Non-binary? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0050.txt b/examples/chats/support_chats/chat_0050.txt new file mode 100644 index 00000000..785565d3 --- /dev/null +++ b/examples/chats/support_chats/chat_0050.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #50 +================================== +Jordan: Good morning, question about firmware update errors. +Maya: Let me look that up for you. +Jordan: Here is my Cookies: sessionid=abcd1234 +Maya: Let me look that up for you. +Jordan: Can you check on that? +Maya: Let me look that up for you. +Jordan: Can you check on that? +Maya: Let me look that up for you. +Jordan: Can you check on that? +Maya: Let me look that up for you. +Jordan: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0051.txt b/examples/chats/support_chats/chat_0051.txt new file mode 100644 index 00000000..fa053c8f --- /dev/null +++ b/examples/chats/support_chats/chat_0051.txt @@ -0,0 +1,15 @@ +Support Chat Transcript #51 +================================== +Alex: Good morning, question about a late delivery. +Maya: Let me look that up for you. +Alex: Can you check on that? +Maya: Let me look that up for you. +Alex: Can you check on that? +Maya: Let me look that up for you. +Alex: Can you check on that? +Maya: Thanks, can you also provide your User ID: U123456? +Alex: Can you check on that? +Maya: Let me look that up for you. +Alex: Can you check on that? +Maya: Let me look that up for you. +Alex: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0052.txt b/examples/chats/support_chats/chat_0052.txt new file mode 100644 index 00000000..afc8d68e --- /dev/null +++ b/examples/chats/support_chats/chat_0052.txt @@ -0,0 +1,15 @@ +Support Chat Transcript #52 +================================== +Taylor: Hi, I need help with updating my subscription. +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Thanks, can you also provide your Sexual Orientation: Heterosexual? +Taylor: Can you check on that? +Ava: Thanks, can you also provide your Cookies: sessionid=abcd1234? +Taylor: Can you check on that? +Ava: Thanks, can you also provide your User ID: U123456? +Taylor: Can you check on that? +Ava: Thanks, can you also provide your Email: jordan@example.com? +Taylor: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0053.txt b/examples/chats/support_chats/chat_0053.txt new file mode 100644 index 00000000..dd795395 --- /dev/null +++ b/examples/chats/support_chats/chat_0053.txt @@ -0,0 +1,12 @@ +Support Chat Transcript #53 +================================== +Jordan: Good morning, question about resetting my password. +Noah: Thanks, can you also provide your Declared Interests: hiking, cooking? +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Thanks, can you also provide your Biometric Identifier: FaceID registered? +Jordan: Can you check on that? +Noah: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0054.txt b/examples/chats/support_chats/chat_0054.txt new file mode 100644 index 00000000..086fc774 --- /dev/null +++ b/examples/chats/support_chats/chat_0054.txt @@ -0,0 +1,16 @@ +Support Chat Transcript #54 +================================== +Riley: Hi, I need help with logging into my account. +Sofia: Let me look that up for you. +Riley: Can you check on that? +Sofia: Thanks, can you also provide your User Preferences: marketing_emails=false? +Riley: Can you check on that? +Sofia: Let me look that up for you. +Riley: Can you check on that? +Sofia: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)? +Riley: Can you check on that? +Sofia: Let me look that up for you. +Riley: Can you check on that? +Sofia: Let me look that up for you. +Riley: Here is my Phone: +1-202-555-0172 +Sofia: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0055.txt b/examples/chats/support_chats/chat_0055.txt new file mode 100644 index 00000000..6c1b4cba --- /dev/null +++ b/examples/chats/support_chats/chat_0055.txt @@ -0,0 +1,17 @@ +Support Chat Transcript #55 +================================== +Sam: Hi, I need help with logging into my account. +Ethan: Thanks, can you also provide your Biometric Identifier: FaceID registered? +Sam: Can you check on that? +Ethan: Let me look that up for you. +Sam: Can you check on that? +Ethan: Let me look that up for you. +Sam: Can you check on that? +Ethan: Let me look that up for you. +Sam: Can you check on that? +Ethan: Let me look that up for you. +Sam: Here is my User ID: U123456 +Ethan: Let me look that up for you. +Sam: Can you check on that? +Ethan: Let me look that up for you. +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0056.txt b/examples/chats/support_chats/chat_0056.txt new file mode 100644 index 00000000..1ed2c54b --- /dev/null +++ b/examples/chats/support_chats/chat_0056.txt @@ -0,0 +1,15 @@ +Support Chat Transcript #56 +================================== +Riley: Hello—I'm having trouble with a late delivery. +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Thanks, can you also provide your Race/Ethnicity: Hispanic? +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0057.txt b/examples/chats/support_chats/chat_0057.txt new file mode 100644 index 00000000..02fad409 --- /dev/null +++ b/examples/chats/support_chats/chat_0057.txt @@ -0,0 +1,19 @@ +Support Chat Transcript #57 +================================== +Riley: Good morning, question about updating my subscription. +Liam: Thanks, can you also provide your Political Affiliation: Independent? +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Thanks, can you also provide your Sexual Orientation: Heterosexual? +Riley: Can you check on that? +Liam: Thanks, can you also provide your Race/Ethnicity: Hispanic? +Riley: Here is my Sexual Orientation: Heterosexual +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Here is my Profile Picture URL: https://example.com/img/profile123.jpg +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0058.txt b/examples/chats/support_chats/chat_0058.txt new file mode 100644 index 00000000..13b2dd91 --- /dev/null +++ b/examples/chats/support_chats/chat_0058.txt @@ -0,0 +1,11 @@ +Support Chat Transcript #58 +================================== +Riley: Hello—I'm having trouble with firmware update errors. +Ethan: Thanks, can you also provide your Profile URL: https://social.example.com/jordan? +Riley: Can you check on that? +Ethan: Let me look that up for you. +Riley: Here is my Race/Ethnicity: Hispanic +Ethan: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7? +Riley: Can you check on that? +Ethan: Let me look that up for you. +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0059.txt b/examples/chats/support_chats/chat_0059.txt new file mode 100644 index 00000000..9cc973c8 --- /dev/null +++ b/examples/chats/support_chats/chat_0059.txt @@ -0,0 +1,18 @@ +Support Chat Transcript #59 +================================== +Taylor: Hi, I need help with updating my subscription. +Ava: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Here is my User Preferences: marketing_emails=false +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Here is my MAC Address: 00:1B:44:11:3A:B7 +Ava: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0060.txt b/examples/chats/support_chats/chat_0060.txt new file mode 100644 index 00000000..dc7dd37f --- /dev/null +++ b/examples/chats/support_chats/chat_0060.txt @@ -0,0 +1,12 @@ +Support Chat Transcript #60 +================================== +Jordan: Good morning, question about resetting my password. +Ethan: Let me look that up for you. +Jordan: Can you check on that? +Ethan: Thanks, can you also provide your Serial Number: SN-1234567890? +Jordan: Here is my Phone: +1-202-555-0172 +Ethan: Let me look that up for you. +Jordan: Can you check on that? +Ethan: Thanks, can you also provide your Declared Interests: hiking, cooking? +Jordan: Can you check on that? +Ethan: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0061.txt b/examples/chats/support_chats/chat_0061.txt new file mode 100644 index 00000000..c5f24b1b --- /dev/null +++ b/examples/chats/support_chats/chat_0061.txt @@ -0,0 +1,18 @@ +Support Chat Transcript #61 +================================== +Taylor: Hi, I need help with logging into my account. +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Here is my Date of Birth: 1990-05-12 +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Religion: None? +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0062.txt b/examples/chats/support_chats/chat_0062.txt new file mode 100644 index 00000000..99f76f07 --- /dev/null +++ b/examples/chats/support_chats/chat_0062.txt @@ -0,0 +1,14 @@ +Support Chat Transcript #62 +================================== +Riley: Can you assist me with a late delivery. +Ethan: Let me look that up for you. +Riley: Can you check on that? +Ethan: Let me look that up for you. +Riley: Can you check on that? +Ethan: Let me look that up for you. +Riley: Can you check on that? +Ethan: Let me look that up for you. +Riley: Can you check on that? +Ethan: Let me look that up for you. +Riley: Can you check on that? +Ethan: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0063.txt b/examples/chats/support_chats/chat_0063.txt new file mode 100644 index 00000000..3f7d7e85 --- /dev/null +++ b/examples/chats/support_chats/chat_0063.txt @@ -0,0 +1,15 @@ +Support Chat Transcript #63 +================================== +Casey: Can you assist me with a double charge on my card. +Noah: Thanks, can you also provide your Political Affiliation: Independent? +Casey: Can you check on that? +Noah: Thanks, can you also provide your Union Membership: None? +Casey: Here is my Date of Birth: 1990-05-12 +Noah: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----? +Casey: Can you check on that? +Noah: Thanks, can you also provide your Serial Number: SN-1234567890? +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Can you check on that? +Noah: Thanks, can you also provide your Profile Picture URL: https://example.com/img/profile123.jpg? +Casey: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0064.txt b/examples/chats/support_chats/chat_0064.txt new file mode 100644 index 00000000..e188231f --- /dev/null +++ b/examples/chats/support_chats/chat_0064.txt @@ -0,0 +1,16 @@ +Support Chat Transcript #64 +================================== +Casey: Hello—I'm having trouble with a double charge on my card. +Ava: Let me look that up for you. +Casey: Can you check on that? +Ava: Thanks, can you also provide your Social Security Number: 123-45-6789? +Casey: Can you check on that? +Ava: Thanks, can you also provide your Sexual Orientation: Heterosexual? +Casey: Here is my MAC Address: 00:1B:44:11:3A:B7 +Ava: Thanks, can you also provide your Profile URL: https://social.example.com/jordan? +Casey: Can you check on that? +Ava: Let me look that up for you. +Casey: Can you check on that? +Ava: Let me look that up for you. +Casey: Can you check on that? +Ava: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0065.txt b/examples/chats/support_chats/chat_0065.txt new file mode 100644 index 00000000..455117ea --- /dev/null +++ b/examples/chats/support_chats/chat_0065.txt @@ -0,0 +1,10 @@ +Support Chat Transcript #65 +================================== +Casey: Good morning, question about a double charge on my card. +Sofia: Let me look that up for you. +Casey: Can you check on that? +Sofia: Thanks, can you also provide your Email: jordan@example.com? +Casey: Here is my Survey Data: Q1=Yes, Q2=No +Sofia: Thanks, can you also provide your Religion: None? +Casey: Can you check on that? +Sofia: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0066.txt b/examples/chats/support_chats/chat_0066.txt new file mode 100644 index 00000000..e92b3786 --- /dev/null +++ b/examples/chats/support_chats/chat_0066.txt @@ -0,0 +1,11 @@ +Support Chat Transcript #66 +================================== +Sam: Good morning, question about a double charge on my card. +Noah: Thanks, can you also provide your IP Address: 192.168.1.42? +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0067.txt b/examples/chats/support_chats/chat_0067.txt new file mode 100644 index 00000000..6f2f8cce --- /dev/null +++ b/examples/chats/support_chats/chat_0067.txt @@ -0,0 +1,15 @@ +Support Chat Transcript #67 +================================== +Taylor: Can you assist me with logging into my account. +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Here is my Profile URL: https://social.example.com/jordan +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Can you check on that? +Ava: Let me look that up for you. +Taylor: Here is my User Preferences: marketing_emails=false \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0068.txt b/examples/chats/support_chats/chat_0068.txt new file mode 100644 index 00000000..5f4f324e --- /dev/null +++ b/examples/chats/support_chats/chat_0068.txt @@ -0,0 +1,20 @@ +Support Chat Transcript #68 +================================== +Sam: Hello—I'm having trouble with updating my subscription. +Ava: Thanks, can you also provide your Survey Data: Q1=Yes, Q2=No? +Sam: Can you check on that? +Ava: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7? +Sam: Can you check on that? +Ava: Let me look that up for you. +Sam: Here is my Serial Number: SN-1234567890 +Ava: Let me look that up for you. +Sam: Can you check on that? +Ava: Let me look that up for you. +Sam: Here is my Cookies: sessionid=abcd1234 +Ava: Let me look that up for you. +Sam: Can you check on that? +Ava: Let me look that up for you. +Sam: Can you check on that? +Ava: Let me look that up for you. +Sam: Can you check on that? +Ava: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0069.txt b/examples/chats/support_chats/chat_0069.txt new file mode 100644 index 00000000..0f43f6d4 --- /dev/null +++ b/examples/chats/support_chats/chat_0069.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #69 +================================== +Sam: Hello—I'm having trouble with a late delivery. +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Thanks, can you also provide your Phone: +1-202-555-0172? +Sam: Here is my Date of Birth: 1990-05-12 \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0070.txt b/examples/chats/support_chats/chat_0070.txt new file mode 100644 index 00000000..2eb1f679 --- /dev/null +++ b/examples/chats/support_chats/chat_0070.txt @@ -0,0 +1,19 @@ +Support Chat Transcript #70 +================================== +Riley: Can you assist me with a late delivery. +Noah: Let me look that up for you. +Riley: Can you check on that? +Noah: Let me look that up for you. +Riley: Can you check on that? +Noah: Let me look that up for you. +Riley: Can you check on that? +Noah: Thanks, can you also provide your Religion: None? +Riley: Can you check on that? +Noah: Let me look that up for you. +Riley: Can you check on that? +Noah: Let me look that up for you. +Riley: Can you check on that? +Noah: Let me look that up for you. +Riley: Can you check on that? +Noah: Let me look that up for you. +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0071.txt b/examples/chats/support_chats/chat_0071.txt new file mode 100644 index 00000000..27cb33df --- /dev/null +++ b/examples/chats/support_chats/chat_0071.txt @@ -0,0 +1,11 @@ +Support Chat Transcript #71 +================================== +Riley: Good morning, question about updating my subscription. +Noah: Thanks, can you also provide your Declared Interests: hiking, cooking? +Riley: Can you check on that? +Noah: Thanks, can you also provide your Serial Number: SN-1234567890? +Riley: Can you check on that? +Noah: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7? +Riley: Can you check on that? +Noah: Thanks, can you also provide your Cookies: sessionid=abcd1234? +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0072.txt b/examples/chats/support_chats/chat_0072.txt new file mode 100644 index 00000000..a5b1b4c6 --- /dev/null +++ b/examples/chats/support_chats/chat_0072.txt @@ -0,0 +1,11 @@ +Support Chat Transcript #72 +================================== +Taylor: Can you assist me with a double charge on my card. +Ethan: Thanks, can you also provide your Profile Picture URL: https://example.com/img/profile123.jpg? +Taylor: Can you check on that? +Ethan: Let me look that up for you. +Taylor: Here is my Approximate Geolocation: San Francisco, CA +Ethan: Thanks, can you also provide your Gender: Non-binary? +Taylor: Here is my Date of Birth: 1990-05-12 +Ethan: Let me look that up for you. +Taylor: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0073.txt b/examples/chats/support_chats/chat_0073.txt new file mode 100644 index 00000000..90524738 --- /dev/null +++ b/examples/chats/support_chats/chat_0073.txt @@ -0,0 +1,12 @@ +Support Chat Transcript #73 +================================== +Alex: Hi, I need help with a late delivery. +Ethan: Thanks, can you also provide your User ID: U123456? +Alex: Can you check on that? +Ethan: Thanks, can you also provide your Gender: Non-binary? +Alex: Can you check on that? +Ethan: Let me look that up for you. +Alex: Can you check on that? +Ethan: Let me look that up for you. +Alex: Can you check on that? +Ethan: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0074.txt b/examples/chats/support_chats/chat_0074.txt new file mode 100644 index 00000000..1af28da7 --- /dev/null +++ b/examples/chats/support_chats/chat_0074.txt @@ -0,0 +1,17 @@ +Support Chat Transcript #74 +================================== +Taylor: Hello—I'm having trouble with a late delivery. +Liam: Thanks, can you also provide your Profile URL: https://social.example.com/jordan? +Taylor: Can you check on that? +Liam: Let me look that up for you. +Taylor: Here is my Approximate Geolocation: San Francisco, CA +Liam: Let me look that up for you. +Taylor: Can you check on that? +Liam: Let me look that up for you. +Taylor: Can you check on that? +Liam: Let me look that up for you. +Taylor: Can you check on that? +Liam: Thanks, can you also provide your Biometric Identifier: FaceID registered? +Taylor: Can you check on that? +Liam: Let me look that up for you. +Taylor: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0075.txt b/examples/chats/support_chats/chat_0075.txt new file mode 100644 index 00000000..1b6c3dc2 --- /dev/null +++ b/examples/chats/support_chats/chat_0075.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #75 +================================== +Riley: Hi, I need help with logging into my account. +Noah: Let me look that up for you. +Riley: Here is my Profile Picture URL: https://example.com/img/profile123.jpg +Noah: Thanks, can you also provide your Email: jordan@example.com? +Riley: Can you check on that? +Noah: Thanks, can you also provide your Profile URL: https://social.example.com/jordan? +Riley: Can you check on that? +Noah: Let me look that up for you. +Riley: Here is my Phone: +1-202-555-0172 +Noah: Let me look that up for you. +Riley: Here is my Gender: Non-binary \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0076.txt b/examples/chats/support_chats/chat_0076.txt new file mode 100644 index 00000000..b1a0ed02 --- /dev/null +++ b/examples/chats/support_chats/chat_0076.txt @@ -0,0 +1,20 @@ +Support Chat Transcript #76 +================================== +Jordan: Good morning, question about updating my subscription. +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----? +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)? +Jordan: Can you check on that? +Noah: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0077.txt b/examples/chats/support_chats/chat_0077.txt new file mode 100644 index 00000000..124dfef5 --- /dev/null +++ b/examples/chats/support_chats/chat_0077.txt @@ -0,0 +1,14 @@ +Support Chat Transcript #77 +================================== +Jordan: Hi, I need help with firmware update errors. +Ava: Let me look that up for you. +Jordan: Here is my Serial Number: SN-1234567890 +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Let me look that up for you. +Jordan: Here is my Gender: Non-binary +Ava: Thanks, can you also provide your Declared Interests: hiking, cooking? +Jordan: Can you check on that? +Ava: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0078.txt b/examples/chats/support_chats/chat_0078.txt new file mode 100644 index 00000000..2bc428da --- /dev/null +++ b/examples/chats/support_chats/chat_0078.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #78 +================================== +Jordan: Good morning, question about logging into my account. +Ethan: Let me look that up for you. +Jordan: Here is my Religion: None +Ethan: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)? +Jordan: Can you check on that? +Ethan: Let me look that up for you. +Jordan: Can you check on that? +Ethan: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7? +Jordan: Can you check on that? +Ethan: Let me look that up for you. +Jordan: Here is my Declared Interests: hiking, cooking \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0079.txt b/examples/chats/support_chats/chat_0079.txt new file mode 100644 index 00000000..69ad10ff --- /dev/null +++ b/examples/chats/support_chats/chat_0079.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #79 +================================== +Jordan: Hi, I need help with a late delivery. +Ethan: Thanks, can you also provide your Political Affiliation: Independent? +Jordan: Can you check on that? +Ethan: Thanks, can you also provide your User Preferences: marketing_emails=false? +Jordan: Here is my Phone: +1-202-555-0172 +Ethan: Thanks, can you also provide your User Preferences: marketing_emails=false? +Jordan: Can you check on that? +Ethan: Let me look that up for you. +Jordan: Can you check on that? +Ethan: Let me look that up for you. +Jordan: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0080.txt b/examples/chats/support_chats/chat_0080.txt new file mode 100644 index 00000000..de36cc70 --- /dev/null +++ b/examples/chats/support_chats/chat_0080.txt @@ -0,0 +1,11 @@ +Support Chat Transcript #80 +================================== +Jordan: Good morning, question about logging into my account. +Maya: Thanks, can you also provide your Gender: Non-binary? +Jordan: Can you check on that? +Maya: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? +Jordan: Can you check on that? +Maya: Let me look that up for you. +Jordan: Can you check on that? +Maya: Let me look that up for you. +Jordan: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0081.txt b/examples/chats/support_chats/chat_0081.txt new file mode 100644 index 00000000..61154f91 --- /dev/null +++ b/examples/chats/support_chats/chat_0081.txt @@ -0,0 +1,16 @@ +Support Chat Transcript #81 +================================== +Riley: Hi, I need help with firmware update errors. +Noah: Let me look that up for you. +Riley: Here is my Profile URL: https://social.example.com/jordan +Noah: Let me look that up for you. +Riley: Can you check on that? +Noah: Let me look that up for you. +Riley: Can you check on that? +Noah: Let me look that up for you. +Riley: Here is my User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X) +Noah: Let me look that up for you. +Riley: Here is my Social Security Number: 123-45-6789 +Noah: Let me look that up for you. +Riley: Can you check on that? +Noah: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0082.txt b/examples/chats/support_chats/chat_0082.txt new file mode 100644 index 00000000..51b0fb51 --- /dev/null +++ b/examples/chats/support_chats/chat_0082.txt @@ -0,0 +1,10 @@ +Support Chat Transcript #82 +================================== +Riley: Good morning, question about a late delivery. +Maya: Let me look that up for you. +Riley: Can you check on that? +Maya: Thanks, can you also provide your Race/Ethnicity: Hispanic? +Riley: Here is my Username: jordan_92 +Maya: Let me look that up for you. +Riley: Can you check on that? +Maya: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0083.txt b/examples/chats/support_chats/chat_0083.txt new file mode 100644 index 00000000..1b07bd11 --- /dev/null +++ b/examples/chats/support_chats/chat_0083.txt @@ -0,0 +1,20 @@ +Support Chat Transcript #83 +================================== +Sam: Hi, I need help with logging into my account. +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)? +Sam: Can you check on that? +Sofia: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----? +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Here is my Race/Ethnicity: Hispanic +Sofia: Let me look that up for you. +Sam: Here is my Serial Number: SN-1234567890 +Sofia: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0084.txt b/examples/chats/support_chats/chat_0084.txt new file mode 100644 index 00000000..e49b840f --- /dev/null +++ b/examples/chats/support_chats/chat_0084.txt @@ -0,0 +1,18 @@ +Support Chat Transcript #84 +================================== +Taylor: Hello—I'm having trouble with logging into my account. +Noah: Thanks, can you also provide your Sexual Orientation: Heterosexual? +Taylor: Can you check on that? +Noah: Let me look that up for you. +Taylor: Can you check on that? +Noah: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)? +Taylor: Here is my Survey Data: Q1=Yes, Q2=No +Noah: Let me look that up for you. +Taylor: Can you check on that? +Noah: Let me look that up for you. +Taylor: Can you check on that? +Noah: Let me look that up for you. +Taylor: Can you check on that? +Noah: Let me look that up for you. +Taylor: Here is my Survey Data: Q1=Yes, Q2=No +Noah: Thanks, can you also provide your Gender: Non-binary? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0085.txt b/examples/chats/support_chats/chat_0085.txt new file mode 100644 index 00000000..abee7b35 --- /dev/null +++ b/examples/chats/support_chats/chat_0085.txt @@ -0,0 +1,19 @@ +Support Chat Transcript #85 +================================== +Riley: Hi, I need help with updating my subscription. +Maya: Let me look that up for you. +Riley: Can you check on that? +Maya: Let me look that up for you. +Riley: Can you check on that? +Maya: Let me look that up for you. +Riley: Can you check on that? +Maya: Let me look that up for you. +Riley: Can you check on that? +Maya: Let me look that up for you. +Riley: Here is my IP Address: 192.168.1.42 +Maya: Thanks, can you also provide your Email: jordan@example.com? +Riley: Can you check on that? +Maya: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7? +Riley: Can you check on that? +Maya: Let me look that up for you. +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0086.txt b/examples/chats/support_chats/chat_0086.txt new file mode 100644 index 00000000..188cc113 --- /dev/null +++ b/examples/chats/support_chats/chat_0086.txt @@ -0,0 +1,11 @@ +Support Chat Transcript #86 +================================== +Alex: Good morning, question about a double charge on my card. +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Thanks, can you also provide your Race/Ethnicity: Hispanic? +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0087.txt b/examples/chats/support_chats/chat_0087.txt new file mode 100644 index 00000000..9956d10a --- /dev/null +++ b/examples/chats/support_chats/chat_0087.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #87 +================================== +Jordan: Hello—I'm having trouble with firmware update errors. +Liam: Let me look that up for you. +Jordan: Can you check on that? +Liam: Let me look that up for you. +Jordan: Can you check on that? +Liam: Let me look that up for you. +Jordan: Can you check on that? +Liam: Let me look that up for you. +Jordan: Can you check on that? +Liam: Let me look that up for you. +Jordan: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0088.txt b/examples/chats/support_chats/chat_0088.txt new file mode 100644 index 00000000..e80db9a9 --- /dev/null +++ b/examples/chats/support_chats/chat_0088.txt @@ -0,0 +1,20 @@ +Support Chat Transcript #88 +================================== +Sam: Can you assist me with a double charge on my card. +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----? +Sam: Can you check on that? +Sofia: Thanks, can you also provide your Biometric Identifier: FaceID registered? +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Here is my Date of Birth: 1990-05-12 +Sofia: Let me look that up for you. +Sam: Here is my IP Address: 192.168.1.42 +Sofia: Thanks, can you also provide your Date of Birth: 1990-05-12? +Sam: Can you check on that? +Sofia: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0089.txt b/examples/chats/support_chats/chat_0089.txt new file mode 100644 index 00000000..741b1ed3 --- /dev/null +++ b/examples/chats/support_chats/chat_0089.txt @@ -0,0 +1,16 @@ +Support Chat Transcript #89 +================================== +Taylor: Can you assist me with resetting my password. +Liam: Let me look that up for you. +Taylor: Here is my Declared Interests: hiking, cooking +Liam: Thanks, can you also provide your Race/Ethnicity: Hispanic? +Taylor: Here is my Union Membership: None +Liam: Let me look that up for you. +Taylor: Can you check on that? +Liam: Let me look that up for you. +Taylor: Can you check on that? +Liam: Let me look that up for you. +Taylor: Can you check on that? +Liam: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7? +Taylor: Can you check on that? +Liam: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0090.txt b/examples/chats/support_chats/chat_0090.txt new file mode 100644 index 00000000..5a80169b --- /dev/null +++ b/examples/chats/support_chats/chat_0090.txt @@ -0,0 +1,11 @@ +Support Chat Transcript #90 +================================== +Sam: Hi, I need help with firmware update errors. +Ava: Let me look that up for you. +Sam: Can you check on that? +Ava: Thanks, can you also provide your Union Membership: None? +Sam: Can you check on that? +Ava: Thanks, can you also provide your Email: jordan@example.com? +Sam: Can you check on that? +Ava: Let me look that up for you. +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0091.txt b/examples/chats/support_chats/chat_0091.txt new file mode 100644 index 00000000..889b15cf --- /dev/null +++ b/examples/chats/support_chats/chat_0091.txt @@ -0,0 +1,14 @@ +Support Chat Transcript #91 +================================== +Jordan: Good morning, question about logging into my account. +Maya: Let me look that up for you. +Jordan: Can you check on that? +Maya: Thanks, can you also provide your Union Membership: None? +Jordan: Can you check on that? +Maya: Let me look that up for you. +Jordan: Can you check on that? +Maya: Let me look that up for you. +Jordan: Here is my Approximate Geolocation: San Francisco, CA +Maya: Let me look that up for you. +Jordan: Can you check on that? +Maya: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0092.txt b/examples/chats/support_chats/chat_0092.txt new file mode 100644 index 00000000..522fea25 --- /dev/null +++ b/examples/chats/support_chats/chat_0092.txt @@ -0,0 +1,11 @@ +Support Chat Transcript #92 +================================== +Riley: Hi, I need help with firmware update errors. +Ethan: Thanks, can you also provide your Religion: None? +Riley: Can you check on that? +Ethan: Thanks, can you also provide your Sexual Orientation: Heterosexual? +Riley: Can you check on that? +Ethan: Thanks, can you also provide your IP Address: 192.168.1.42? +Riley: Can you check on that? +Ethan: Let me look that up for you. +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0093.txt b/examples/chats/support_chats/chat_0093.txt new file mode 100644 index 00000000..fd43ccd0 --- /dev/null +++ b/examples/chats/support_chats/chat_0093.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #93 +================================== +Taylor: Hi, I need help with a late delivery. +Maya: Let me look that up for you. +Taylor: Can you check on that? +Maya: Thanks, can you also provide your Username: jordan_92? +Taylor: Can you check on that? +Maya: Let me look that up for you. +Taylor: Here is my Political Affiliation: Independent +Maya: Let me look that up for you. +Taylor: Can you check on that? +Maya: Let me look that up for you. +Taylor: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0094.txt b/examples/chats/support_chats/chat_0094.txt new file mode 100644 index 00000000..deff8ce5 --- /dev/null +++ b/examples/chats/support_chats/chat_0094.txt @@ -0,0 +1,20 @@ +Support Chat Transcript #94 +================================== +Alex: Hi, I need help with firmware update errors. +Ethan: Let me look that up for you. +Alex: Can you check on that? +Ethan: Let me look that up for you. +Alex: Here is my Biometric Identifier: FaceID registered +Ethan: Let me look that up for you. +Alex: Can you check on that? +Ethan: Let me look that up for you. +Alex: Can you check on that? +Ethan: Thanks, can you also provide your Username: jordan_92? +Alex: Here is my Union Membership: None +Ethan: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7? +Alex: Can you check on that? +Ethan: Let me look that up for you. +Alex: Can you check on that? +Ethan: Thanks, can you also provide your User Preferences: marketing_emails=false? +Alex: Can you check on that? +Ethan: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0095.txt b/examples/chats/support_chats/chat_0095.txt new file mode 100644 index 00000000..7b9a134b --- /dev/null +++ b/examples/chats/support_chats/chat_0095.txt @@ -0,0 +1,19 @@ +Support Chat Transcript #95 +================================== +Sam: Can you assist me with logging into my account. +Liam: Thanks, can you also provide your Date of Birth: 1990-05-12? +Sam: Here is my Race/Ethnicity: Hispanic +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Thanks, can you also provide your Union Membership: None? +Sam: Can you check on that? +Liam: Thanks, can you also provide your Phone: +1-202-555-0172? +Sam: Here is my User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X) +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Here is my Survey Data: Q1=Yes, Q2=No \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0096.txt b/examples/chats/support_chats/chat_0096.txt new file mode 100644 index 00000000..b8baff39 --- /dev/null +++ b/examples/chats/support_chats/chat_0096.txt @@ -0,0 +1,14 @@ +Support Chat Transcript #96 +================================== +Riley: Good morning, question about logging into my account. +Ava: Let me look that up for you. +Riley: Can you check on that? +Ava: Let me look that up for you. +Riley: Can you check on that? +Ava: Let me look that up for you. +Riley: Can you check on that? +Ava: Let me look that up for you. +Riley: Can you check on that? +Ava: Thanks, can you also provide your Sexual Orientation: Heterosexual? +Riley: Can you check on that? +Ava: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0097.txt b/examples/chats/support_chats/chat_0097.txt new file mode 100644 index 00000000..2875d2df --- /dev/null +++ b/examples/chats/support_chats/chat_0097.txt @@ -0,0 +1,13 @@ +Support Chat Transcript #97 +================================== +Sam: Hi, I need help with firmware update errors. +Maya: Thanks, can you also provide your Serial Number: SN-1234567890? +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0098.txt b/examples/chats/support_chats/chat_0098.txt new file mode 100644 index 00000000..745fb609 --- /dev/null +++ b/examples/chats/support_chats/chat_0098.txt @@ -0,0 +1,19 @@ +Support Chat Transcript #98 +================================== +Taylor: Hi, I need help with resetting my password. +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Here is my Sexual Orientation: Heterosexual +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Political Affiliation: Independent? +Taylor: Here is my Profile URL: https://social.example.com/jordan +Sofia: Let me look that up for you. +Taylor: Here is my Race/Ethnicity: Hispanic \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0099.txt b/examples/chats/support_chats/chat_0099.txt new file mode 100644 index 00000000..04ed393c --- /dev/null +++ b/examples/chats/support_chats/chat_0099.txt @@ -0,0 +1,15 @@ +Support Chat Transcript #99 +================================== +Taylor: Good morning, question about a late delivery. +Maya: Thanks, can you also provide your User Preferences: marketing_emails=false? +Taylor: Can you check on that? +Maya: Let me look that up for you. +Taylor: Can you check on that? +Maya: Thanks, can you also provide your Email: jordan@example.com? +Taylor: Can you check on that? +Maya: Let me look that up for you. +Taylor: Here is my User ID: U123456 +Maya: Let me look that up for you. +Taylor: Here is my User ID: U123456 +Maya: Let me look that up for you. +Taylor: Here is my Gender: Non-binary \ No newline at end of file diff --git a/examples/chats/support_chats/chat_0100.txt b/examples/chats/support_chats/chat_0100.txt new file mode 100644 index 00000000..29d36daf --- /dev/null +++ b/examples/chats/support_chats/chat_0100.txt @@ -0,0 +1,11 @@ +Support Chat Transcript #100 +================================== +Casey: Can you assist me with resetting my password. +Noah: Let me look that up for you. +Casey: Here is my User ID: U123456 +Noah: Thanks, can you also provide your Declared Interests: hiking, cooking? +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0001.txt.redacted b/examples/chats/support_chats/redacted/chat_0001.txt.redacted new file mode 100644 index 00000000..9d29cd9a --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0001.txt.redacted @@ -0,0 +1,16 @@ +Support Chat Transcript #1 +================================== +Taylor: Hi, I need help with logging into my account. +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Profile URL: <<>>? +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your <<>>: <<>>? +Taylor: Can you check on that? +Sofia: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0001.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0001.txt.redacted.redaction.json new file mode 100644 index 00000000..9d3b46fe --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0001.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0001.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0001.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Profile URL", + "value": "https://social.example.com/jordan", + "confidence": 0.949, + "snippet": "a: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\nTaylor: Can you check on that?\nSofia: Let me loo" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.78, + "snippet": "on that?\nSofia: Thanks, can you also provide your Union Membership: None?\nTaylor: Can you check on that?\nSofia: Let " + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.979, + "snippet": "anks, can you also provide your Union Membership: None?\nTaylor: Can you check on that?\nSofia: Let me loo" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0002.txt.redacted b/examples/chats/support_chats/redacted/chat_0002.txt.redacted new file mode 100644 index 00000000..10899b20 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0002.txt.redacted @@ -0,0 +1,17 @@ +Support Chat Transcript #2 +================================== +Casey: Good morning, question about a late delivery. +Sofia: Thanks, can you also provide your MAC Address: <<>>? +Casey: Can you check on that? +Sofia: Let me look that up for you. +Casey: Here is my Profile Picture URL: https://example.com/img/profile123.jpg +Sofia: Let me look that up for you. +Casey: Can you check on that? +Sofia: Let me look that up for you. +Casey: Can you check on that? +Sofia: Let me look that up for you. +Casey: Can you check on that? +Sofia: Let me look that up for you. +Casey: Can you check on that? +Sofia: Let me look that up for you. +Casey: Here is my Profile URL: <<>> \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0002.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0002.txt.redacted.redaction.json new file mode 100644 index 00000000..512ab066 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0002.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0002.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0002.txt.redacted", + "totalRedactions": 2, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.997, + "snippet": "a: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\nCasey: Can you check on that?\nSofia: Let me look" + }, + { + "type": "Profile URL", + "value": "https://social.example.com/jordan", + "confidence": 0.849, + "snippet": "k that up for you.\nCasey: Here is my Profile URL: https://social.example.com/jordan" + }, + { + "type": "Profile Picture URL", + "value": "https://social.example.com/jordan", + "confidence": 0.523, + "snippet": "k that up for you.\nCasey: Here is my Profile URL: https://social.example.com/jordan" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0003.txt.redacted b/examples/chats/support_chats/redacted/chat_0003.txt.redacted new file mode 100644 index 00000000..b6d69625 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0003.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #3 +================================== +Sam: Good morning, question about logging into my account. +<<>>: Let me look that up for you. +Sam: Can you check on that? +<<>>: Let me look that up for you. +Sam: Can you check on that? +<<>>: Let me look that up for you. +Sam: Can you check on that? +<<>>: Let me look that up for you. +Sam: Can you check on that? +<<>>: Let me look that up for you. +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0003.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0003.txt.redacted.redaction.json new file mode 100644 index 00000000..064151d1 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0003.txt.redacted.redaction.json @@ -0,0 +1,57 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0003.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0003.txt.redacted", + "totalRedactions": 1, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.564, + "snippet": " morning, question about logging into my account.\nLiam: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.53, + "snippet": "look that up for you.\nSam: Can you check on that?\nLiam: Let me look that up for you.\nSam: Can you check " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0004.txt.redacted b/examples/chats/support_chats/redacted/chat_0004.txt.redacted new file mode 100644 index 00000000..6bf25a9f --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0004.txt.redacted @@ -0,0 +1,10 @@ +Support Chat Transcript #4 +================================== +<<>>: Good morning, question about updating my subscription. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my IP Address: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0004.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0004.txt.redacted.redaction.json new file mode 100644 index 00000000..b838417e --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0004.txt.redacted.redaction.json @@ -0,0 +1,99 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0004.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0004.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.985, + "snippet": " Transcript #4\n==================================\nSam: Good morning, question about updating my subscri" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.98, + "snippet": "morning, question about updating my subscription.\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.98, + "snippet": "y subscription.\nAva: Let me look that up for you.\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.979, + "snippet": "look that up for you.\nSam: Can you check on that?\nAva: Let me look that up for you.\nSam: Here is my IP " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.985, + "snippet": " check on that?\nAva: Let me look that up for you.\nSam: Here is my IP Address: 192.168.1.42\nAva: Let me " + }, + { + "type": "IP Address", + "value": "192.168.1.42", + "confidence": 0.997, + "snippet": "look that up for you.\nSam: Here is my IP Address: 192.168.1.42\nAva: Let me look that up for you.\nSam: Can you ch" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.969, + "snippet": "for you.\nSam: Here is my IP Address: 192.168.1.42\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.983, + "snippet": "s: 192.168.1.42\nAva: Let me look that up for you.\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.984, + "snippet": "look that up for you.\nSam: Can you check on that?\nAva: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0005.txt.redacted b/examples/chats/support_chats/redacted/chat_0005.txt.redacted new file mode 100644 index 00000000..610d192e --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0005.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #5 +================================== +Taylor: Hi, I need help with a late delivery. +<<>>: Let me look that up for you. +Taylor: Here is my Survey Data: Q1=Yes, Q2=No +<<>>: Let me look that up for you. +Taylor: Can you check on that? +<<>>: Thanks, can you also provide your Profile URL: <<>>? +Taylor: Can you check on that? +<<>>: Let me look that up for you. +Taylor: Can you check on that? +<<>>: Thanks, can you also provide your Race/Ethnicity: <<>>? +Taylor: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0005.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0005.txt.redacted.redaction.json new file mode 100644 index 00000000..89e654f6 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0005.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0005.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0005.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.538, + "snippet": "===\nTaylor: Hi, I need help with a late delivery.\nLiam: Let me look that up for you.\nTaylor: Here is my " + }, + { + "type": "Profile URL", + "value": "https://social.example.com/jordan", + "confidence": 0.981, + "snippet": "m: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\nTaylor: Can you check on that?\nLiam: Let me look" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.995, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\nTaylor: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0006.txt.redacted b/examples/chats/support_chats/redacted/chat_0006.txt.redacted new file mode 100644 index 00000000..5c343548 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0006.txt.redacted @@ -0,0 +1,11 @@ +Support Chat Transcript #6 +================================== +Riley: Hello—I'm having trouble with a double charge on my card. +Liam: Thanks, can you also provide your User ID: <<>>? +Riley: Can you check on that? +Liam: Thanks, can you also provide your Sexual Orientation: <<>>? +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Here is my Race/Ethnicity: <<>> +Liam: Let me look that up for you. +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0006.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0006.txt.redacted.redaction.json new file mode 100644 index 00000000..d1a8fe34 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0006.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0006.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0006.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User ID", + "value": "U123456", + "confidence": 0.981, + "snippet": "\nLiam: Thanks, can you also provide your User ID: U123456?\nRiley: Can you check on that?\nLiam: Thanks, can " + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.992, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\nRiley: Can you check on that?\nLiam: Let me look " + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.997, + "snippet": "hat up for you.\nRiley: Here is my Race/Ethnicity: Hispanic\nLiam: Let me look that up for you.\nRiley: Can you" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0007.txt.redacted b/examples/chats/support_chats/redacted/chat_0007.txt.redacted new file mode 100644 index 00000000..7f714017 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0007.txt.redacted @@ -0,0 +1,15 @@ +Support Chat Transcript #7 +================================== +Riley: Hi, I need help with a double charge on my card. +Sofia: Let me look that up for you. +Riley: Can you check on that? +Sofia: Thanks, can you also provide your Cookies: sessionid=abcd1234? +Riley: Here is my Approximate Geolocation: <<>> +Sofia: Let me look that up for you. +Riley: Here is my Race/Ethnicity: <<>> +Sofia: Let me look that up for you. +Riley: Can you check on that? +Sofia: Let me look that up for you. +Riley: Can you check on that? +Sofia: Thanks, can you also provide your Serial Number: <<>>? +Riley: Here is my User Preferences: marketing_emails=<<>> \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0007.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0007.txt.redacted.redaction.json new file mode 100644 index 00000000..224fd79c --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0007.txt.redacted.redaction.json @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0007.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0007.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Region", + "value": "San Francisco, CA", + "confidence": 0.525, + "snippet": "d1234?\nRiley: Here is my Approximate Geolocation: San Francisco, CA\nSofia: Let me look that up for you.\nRiley: Here i" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.997, + "snippet": "hat up for you.\nRiley: Here is my Race/Ethnicity: Hispanic\nSofia: Let me look that up for you.\nRiley: Can yo" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.997, + "snippet": " Thanks, can you also provide your Serial Number: SN-1234567890?\nRiley: Here is my User Preferences: marketing_em" + }, + { + "type": "User Preferences", + "value": "false", + "confidence": 0.681, + "snippet": "ey: Here is my User Preferences: marketing_emails=false" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0008.txt.redacted b/examples/chats/support_chats/redacted/chat_0008.txt.redacted new file mode 100644 index 00000000..c58986d7 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0008.txt.redacted @@ -0,0 +1,10 @@ +Support Chat Transcript #8 +================================== +Casey: Hello—I'm having trouble with a late delivery. +Ethan: Thanks, can you also provide your Profile URL: <<>>? +Casey: Can you check on that? +Ethan: Let me look that up for you. +Casey: Can you check on that? +Ethan: Thanks, can you also provide your Phone: <<>>? +Casey: Can you check on that? +Ethan: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0008.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0008.txt.redacted.redaction.json new file mode 100644 index 00000000..c9b1efff --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0008.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0008.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0008.txt.redacted", + "totalRedactions": 2, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Profile URL", + "value": "https://social.example.com/jordan", + "confidence": 0.98, + "snippet": "n: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\nCasey: Can you check on that?\nEthan: Let me look" + }, + { + "type": "Profile Picture URL", + "value": "https://social.example.com/jordan", + "confidence": 0.524, + "snippet": "n: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\nCasey: Can you check on that?\nEthan: Let me look" + }, + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.999, + "snippet": "?\nEthan: Thanks, can you also provide your Phone: +1-202-555-0172?\nCasey: Can you check on that?\nEthan: Let me look" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0009.txt.redacted b/examples/chats/support_chats/redacted/chat_0009.txt.redacted new file mode 100644 index 00000000..165279ff --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0009.txt.redacted @@ -0,0 +1,17 @@ +Support Chat Transcript #9 +================================== +<<>>: Good morning, question about firmware update errors. +<<>>: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Serial Number: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my MAC Address: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0009.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0009.txt.redacted.redaction.json new file mode 100644 index 00000000..1f5c36b6 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0009.txt.redacted.redaction.json @@ -0,0 +1,147 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0009.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0009.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.688, + "snippet": " Transcript #9\n==================================\nTaylor: Good morning, question about firmware update err" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.736, + "snippet": "d morning, question about firmware update errors.\nSofia: Thanks, can you also provide your Private Key: -" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.71, + "snippet": "TE KEY----- [redacted] -----END PRIVATE KEY-----?\nTaylor: Can you check on that?\nSofia: Let me look that u" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.682, + "snippet": " PRIVATE KEY-----?\nTaylor: Can you check on that?\nSofia: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.605, + "snippet": "heck on that?\nSofia: Let me look that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that u" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.66, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.659, + "snippet": "heck on that?\nSofia: Let me look that up for you.\nTaylor: Can you check on that?\nSofia: Thanks, can you al" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.616, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nSofia: Thanks, can you also provide your Serial Number:" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.998, + "snippet": " Thanks, can you also provide your Serial Number: SN-1234567890?\nTaylor: Can you check on that?\nSofia: Let me loo" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.613, + "snippet": "u also provide your Serial Number: SN-1234567890?\nTaylor: Can you check on that?\nSofia: Let me look that u" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.504, + "snippet": "er: SN-1234567890?\nTaylor: Can you check on that?\nSofia: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.717, + "snippet": "heck on that?\nSofia: Let me look that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that u" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.585, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that up for you.\nTaylor: Here is my " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.773, + "snippet": "heck on that?\nSofia: Let me look that up for you.\nTaylor: Here is my MAC Address: 00:1B:44:11:3A:B7\nSofia:" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": " that up for you.\nTaylor: Here is my MAC Address: 00:1B:44:11:3A:B7\nSofia: Let me look that up for you.\nTaylor: Can y" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.631, + "snippet": "Taylor: Here is my MAC Address: 00:1B:44:11:3A:B7\nSofia: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.7, + "snippet": "B:44:11:3A:B7\nSofia: Let me look that up for you.\nTaylor: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0010.txt.redacted b/examples/chats/support_chats/redacted/chat_0010.txt.redacted new file mode 100644 index 00000000..4d90b7c3 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0010.txt.redacted @@ -0,0 +1,12 @@ +Support Chat Transcript #10 +================================== +Sam: Can you assist me with updating my subscription. +<<>>: Let me look that up for you. +Sam: Can you check on that? +<<>>: Let me look that up for you. +Sam: Can you check on that? +<<>>: Thanks, can you also provide your Survey Data: Q1=Yes, Q2=No? +Sam: Can you check on that? +<<>>: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? +Sam: Can you check on that? +<<>>: Thanks, can you also provide your <<>>: <<>>? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0010.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0010.txt.redacted.redaction.json new file mode 100644 index 00000000..75a91b0b --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0010.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0010.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0010.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.651, + "snippet": " Can you assist me with updating my subscription.\nEthan: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.838, + "snippet": "on that?\nEthan: Thanks, can you also provide your Declared Interests: hiking, cooking?" + }, + { + "type": "Declared Interests", + "value": "hiking, cooking", + "confidence": 0.898, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0011.txt.redacted b/examples/chats/support_chats/redacted/chat_0011.txt.redacted new file mode 100644 index 00000000..d8f6f66b --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0011.txt.redacted @@ -0,0 +1,20 @@ +Support Chat Transcript #11 +================================== +Sam: Hi, I need help with firmware update errors. +Maya: Thanks, can you also provide your <<>>: <<>>? +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Thanks, can you also provide your User Agent: <<>> (Macintosh; Intel Mac OS X)? +Sam: Here is my Approximate Geolocation: <<>> +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Here is my Date of Birth: <<>> +Maya: Thanks, can you also provide your User ID: <<>>? +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Here is my Email: <<>> +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Thanks, can you also provide your Email: <<>>? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0011.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0011.txt.redacted.redaction.json new file mode 100644 index 00000000..ab7fa9c1 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0011.txt.redacted.redaction.json @@ -0,0 +1,93 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0011.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0011.txt.redacted", + "totalRedactions": 7, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Religion", + "value": "Religion", + "confidence": 0.959, + "snippet": "e errors.\nMaya: Thanks, can you also provide your Religion: None?\nSam: Can you check on that?\nMaya: Let me l" + }, + { + "type": "Religion", + "value": "None", + "confidence": 0.947, + "snippet": "Maya: Thanks, can you also provide your Religion: None?\nSam: Can you check on that?\nMaya: Let me look th" + }, + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.965, + "snippet": "ya: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nSam: Here is my Appr" + }, + { + "type": "Region", + "value": "San Francisco, CA", + "confidence": 0.564, + "snippet": "c OS X)?\nSam: Here is my Approximate Geolocation: San Francisco, CA\nMaya: Let me look that up for you.\nSam: Can you c" + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.991, + "snippet": "k that up for you.\nSam: Here is my Date of Birth: 1990-05-12\nMaya: Thanks, can you also provide your User ID: " + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.992, + "snippet": "\nMaya: Thanks, can you also provide your User ID: U123456?\nSam: Can you check on that?\nMaya: Let me look th" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.999, + "snippet": "t me look that up for you.\nSam: Here is my Email: jordan@example.com\nMaya: Let me look that up for you.\nSam: Can you c" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.997, + "snippet": "t?\nMaya: Thanks, can you also provide your Email: jordan@example.com?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0012.txt.redacted b/examples/chats/support_chats/redacted/chat_0012.txt.redacted new file mode 100644 index 00000000..aeb372e2 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0012.txt.redacted @@ -0,0 +1,12 @@ +Support Chat Transcript #12 +================================== +<<>>: Good morning, question about a double charge on my card. +<<>>: Thanks, can you also provide your Email: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Political Affiliation: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0012.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0012.txt.redacted.redaction.json new file mode 100644 index 00000000..2e913854 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0012.txt.redacted.redaction.json @@ -0,0 +1,117 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0012.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0012.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.803, + "snippet": "Transcript #12\n==================================\nSam: Good morning, question about a double charge on " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.758, + "snippet": "rning, question about a double charge on my card.\nAva: Thanks, can you also provide your Email: jordan@" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.995, + "snippet": "rd.\nAva: Thanks, can you also provide your Email: jordan@example.com?\nSam: Can you check on that?\nAva: Thanks, can you" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.707, + "snippet": " you also provide your Email: jordan@example.com?\nSam: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.727, + "snippet": ": jordan@example.com?\nSam: Can you check on that?\nAva: Thanks, can you also provide your Political Affi" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.984, + "snippet": " can you also provide your Political Affiliation: Independent?\nSam: Can you check on that?\nAva: Let me look tha" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.705, + "snippet": " provide your Political Affiliation: Independent?\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.658, + "snippet": "liation: Independent?\nSam: Can you check on that?\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.728, + "snippet": " check on that?\nAva: Let me look that up for you.\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.631, + "snippet": "look that up for you.\nSam: Can you check on that?\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.808, + "snippet": " check on that?\nAva: Let me look that up for you.\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.721, + "snippet": "look that up for you.\nSam: Can you check on that?\nAva: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0013.txt.redacted b/examples/chats/support_chats/redacted/chat_0013.txt.redacted new file mode 100644 index 00000000..da4decff --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0013.txt.redacted @@ -0,0 +1,19 @@ +Support Chat Transcript #13 +================================== +Sam: Good morning, question about updating my subscription. +Noah: Thanks, can you also provide your Profile URL: <<>>? +Sam: Can you check on that? +Noah: Thanks, can you also provide your <<>>: <<>>? +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Here is my Serial Number: <<>> +Noah: Thanks, can you also provide your <<>>: <<>>? +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Here is my Social Security Number: <<>> +Noah: Thanks, can you also provide your IP Address: <<>>? +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Here is my MAC Address: <<>> \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0013.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0013.txt.redacted.redaction.json new file mode 100644 index 00000000..51e7737a --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0013.txt.redacted.redaction.json @@ -0,0 +1,99 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0013.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0013.txt.redacted", + "totalRedactions": 7, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Profile URL", + "value": "https://social.example.com/jordan", + "confidence": 0.984, + "snippet": "h: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\nSam: Can you check on that?\nNoah: Thanks, can yo" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.78, + "snippet": " on that?\nNoah: Thanks, can you also provide your Union Membership: None?\nSam: Can you check on that?\nNoah: Let me l" + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.964, + "snippet": "anks, can you also provide your Union Membership: None?\nSam: Can you check on that?\nNoah: Let me look th" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.995, + "snippet": "k that up for you.\nSam: Here is my Serial Number: SN-1234567890\nNoah: Thanks, can you also provide your Union Mem" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.795, + "snippet": "234567890\nNoah: Thanks, can you also provide your Union Membership: None?\nSam: Can you check on that?\nNoah: Let me l" + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.955, + "snippet": "anks, can you also provide your Union Membership: None?\nSam: Can you check on that?\nNoah: Let me look th" + }, + { + "type": "Social Security Number", + "value": "123-45-6789", + "confidence": 0.998, + "snippet": " for you.\nSam: Here is my Social Security Number: 123-45-6789\nNoah: Thanks, can you also provide your IP Addres" + }, + { + "type": "IP Address", + "value": "192.168.1.42", + "confidence": 0.998, + "snippet": "ah: Thanks, can you also provide your IP Address: 192.168.1.42?\nSam: Can you check on that?\nNoah: Let me look th" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.972, + "snippet": "ook that up for you.\nSam: Here is my MAC Address: 00:1B:44:11:3A:B7" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0014.txt.redacted b/examples/chats/support_chats/redacted/chat_0014.txt.redacted new file mode 100644 index 00000000..dac7eafe --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0014.txt.redacted @@ -0,0 +1,17 @@ +Support Chat Transcript #14 +================================== +Casey: Can you assist me with logging into my account. +Ava: Thanks, can you also provide your Race/Ethnicity: <<>>? +Casey: Can you check on that? +Ava: Let me look that up for you. +Casey: Here is my Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY----- +Ava: Let me look that up for you. +Casey: Can you check on that? +Ava: Thanks, can you also provide your User Agent: <<>> (Macintosh; Intel Mac OS X)? +Casey: Here is my Profile Picture URL: https://example.com/img/profile123.jpg +Ava: Let me look that up for you. +Casey: Here is my Race/Ethnicity: <<>> +Ava: Let me look that up for you. +Casey: Can you check on that? +Ava: Let me look that up for you. +Casey: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0014.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0014.txt.redacted.redaction.json new file mode 100644 index 00000000..e6473544 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0014.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0014.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0014.txt.redacted", + "totalRedactions": 2, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.969, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\nCasey: Can you check on that?\nAva: Let me look t" + }, + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.944, + "snippet": "va: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nCasey: Here is my Pr" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.979, + "snippet": "hat up for you.\nCasey: Here is my Race/Ethnicity: Hispanic\nAva: Let me look that up for you.\nCasey: Can you " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0015.txt.redacted b/examples/chats/support_chats/redacted/chat_0015.txt.redacted new file mode 100644 index 00000000..ffdd0f54 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0015.txt.redacted @@ -0,0 +1,12 @@ +Support Chat Transcript #15 +================================== +Taylor: Hello—I'm having trouble with logging into my account. +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Phone: <<>>? +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your User Agent: <<>> (Macintosh; <<>>)? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0015.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0015.txt.redacted.redaction.json new file mode 100644 index 00000000..a09d5744 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0015.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0015.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0015.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.999, + "snippet": "?\nSofia: Thanks, can you also provide your Phone: +1-202-555-0172?\nTaylor: Can you check on that?\nSofia: Thanks, ca" + }, + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.976, + "snippet": "ia: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?" + }, + { + "type": "MAC Address", + "value": "Intel Mac OS X", + "confidence": 0.559, + "snippet": " provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0016.txt.redacted b/examples/chats/support_chats/redacted/chat_0016.txt.redacted new file mode 100644 index 00000000..ec036d09 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0016.txt.redacted @@ -0,0 +1,12 @@ +Support Chat Transcript #16 +================================== +Riley: Can you assist me with updating my subscription. +Ethan: Let me look that up for you. +Riley: Can you check on that? +Ethan: Thanks, can you also provide your Profile Picture URL: https://example.com/img/profile123.jpg? +Riley: Can you check on that? +Ethan: Let me look that up for you. +Riley: Can you check on that? +Ethan: Thanks, can you also provide your User Agent: <<>> (Macintosh; Intel Mac OS X)? +Riley: Can you check on that? +Ethan: Thanks, can you also provide your Social Security Number: <<>>? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0016.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0016.txt.redacted.redaction.json new file mode 100644 index 00000000..348eaf68 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0016.txt.redacted.redaction.json @@ -0,0 +1,57 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0016.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0016.txt.redacted", + "totalRedactions": 2, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.951, + "snippet": "an: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nRiley: Can you check" + }, + { + "type": "Social Security Number", + "value": "123-45-6789", + "confidence": 0.999, + "snippet": "can you also provide your Social Security Number: 123-45-6789?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0017.txt.redacted b/examples/chats/support_chats/redacted/chat_0017.txt.redacted new file mode 100644 index 00000000..3097232f --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0017.txt.redacted @@ -0,0 +1,16 @@ +Support Chat Transcript #17 +================================== +<<>>: Hello—I'm having trouble with updating my subscription. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY----- +<<>>: Thanks, can you also provide your Political Affiliation: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0017.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0017.txt.redacted.redaction.json new file mode 100644 index 00000000..376267ec --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0017.txt.redacted.redaction.json @@ -0,0 +1,135 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0017.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0017.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.861, + "snippet": "Transcript #17\n==================================\nTaylor: Hello—I'm having trouble with updating my subscr" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.878, + "snippet": "I'm having trouble with updating my subscription.\nMaya: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.857, + "snippet": " subscription.\nMaya: Let me look that up for you.\nTaylor: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.906, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nMaya: Let me look that up for you.\nTaylor: Here is my " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.887, + "snippet": "check on that?\nMaya: Let me look that up for you.\nTaylor: Here is my Private Key: -----BEGIN PRIVATE KEY--" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.902, + "snippet": "ATE KEY----- [redacted] -----END PRIVATE KEY-----\nMaya: Thanks, can you also provide your Political Affi" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.965, + "snippet": " can you also provide your Political Affiliation: Independent?\nTaylor: Can you check on that?\nMaya: Let me look" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.83, + "snippet": " provide your Political Affiliation: Independent?\nTaylor: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.853, + "snippet": "tion: Independent?\nTaylor: Can you check on that?\nMaya: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.784, + "snippet": "check on that?\nMaya: Let me look that up for you.\nTaylor: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.809, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nMaya: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.826, + "snippet": "check on that?\nMaya: Let me look that up for you.\nTaylor: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.807, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nMaya: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.824, + "snippet": "check on that?\nMaya: Let me look that up for you.\nTaylor: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.865, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nMaya: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0018.txt.redacted b/examples/chats/support_chats/redacted/chat_0018.txt.redacted new file mode 100644 index 00000000..eca18e17 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0018.txt.redacted @@ -0,0 +1,15 @@ +Support Chat Transcript #18 +================================== +<<>>: Hi, I need help with resetting my password. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Sexual Orientation: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0018.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0018.txt.redacted.redaction.json new file mode 100644 index 00000000..ed40d5f8 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0018.txt.redacted.redaction.json @@ -0,0 +1,129 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0018.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0018.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.955, + "snippet": "Transcript #18\n==================================\nAlex: Hi, I need help with resetting my password.\nAva:" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.949, + "snippet": "Alex: Hi, I need help with resetting my password.\nAva: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.951, + "snippet": "ng my password.\nAva: Let me look that up for you.\nAlex: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.953, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nAva: Let me look that up for you.\nAlex: Here is my Se" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.957, + "snippet": " check on that?\nAva: Let me look that up for you.\nAlex: Here is my Sexual Orientation: Heterosexual\nAva:" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.991, + "snippet": " up for you.\nAlex: Here is my Sexual Orientation: Heterosexual\nAva: Let me look that up for you.\nAlex: Can you c" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.909, + "snippet": "Alex: Here is my Sexual Orientation: Heterosexual\nAva: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.947, + "snippet": "n: Heterosexual\nAva: Let me look that up for you.\nAlex: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.934, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nAva: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.941, + "snippet": " check on that?\nAva: Let me look that up for you.\nAlex: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.938, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nAva: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.949, + "snippet": " check on that?\nAva: Let me look that up for you.\nAlex: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.958, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nAva: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.959, + "snippet": " check on that?\nAva: Let me look that up for you.\nAlex: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0019.txt.redacted b/examples/chats/support_chats/redacted/chat_0019.txt.redacted new file mode 100644 index 00000000..baf61760 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0019.txt.redacted @@ -0,0 +1,16 @@ +Support Chat Transcript #19 +================================== +<<>>: Can you assist me with firmware update errors. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Sexual Orientation: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Email: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0019.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0019.txt.redacted.redaction.json new file mode 100644 index 00000000..9eaf7321 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0019.txt.redacted.redaction.json @@ -0,0 +1,141 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0019.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0019.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.671, + "snippet": "Transcript #19\n==================================\nCasey: Can you assist me with firmware update errors.\nN" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.59, + "snippet": "y: Can you assist me with firmware update errors.\nNoah: Let me look that up for you.\nCasey: Can you chec" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.657, + "snippet": "update errors.\nNoah: Let me look that up for you.\nCasey: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.608, + "snippet": "ok that up for you.\nCasey: Can you check on that?\nNoah: Let me look that up for you.\nCasey: Can you chec" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.547, + "snippet": "check on that?\nNoah: Let me look that up for you.\nCasey: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.672, + "snippet": "ok that up for you.\nCasey: Can you check on that?\nNoah: Let me look that up for you.\nCasey: Can you chec" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.584, + "snippet": "check on that?\nNoah: Let me look that up for you.\nCasey: Can you check on that?\nNoah: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.642, + "snippet": "ok that up for you.\nCasey: Can you check on that?\nNoah: Thanks, can you also provide your Sexual Orienta" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.995, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\nCasey: Can you check on that?\nNoah: Thanks, can " + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.555, + "snippet": "so provide your Sexual Orientation: Heterosexual?\nCasey: Can you check on that?\nNoah: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.595, + "snippet": "tion: Heterosexual?\nCasey: Can you check on that?\nNoah: Thanks, can you also provide your Email: jordan@" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.996, + "snippet": "t?\nNoah: Thanks, can you also provide your Email: jordan@example.com?\nCasey: Can you check on that?\nNoah: Let me look " + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.576, + "snippet": " you also provide your Email: jordan@example.com?\nCasey: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.511, + "snippet": "jordan@example.com?\nCasey: Can you check on that?\nNoah: Let me look that up for you.\nCasey: Can you chec" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.671, + "snippet": "check on that?\nNoah: Let me look that up for you.\nCasey: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.574, + "snippet": "ok that up for you.\nCasey: Can you check on that?\nNoah: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0020.txt.redacted b/examples/chats/support_chats/redacted/chat_0020.txt.redacted new file mode 100644 index 00000000..c7327bf7 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0020.txt.redacted @@ -0,0 +1,10 @@ +Support Chat Transcript #20 +================================== +<<>>: Can you assist me with updating my subscription. +<<>>: Thanks, can you also provide your Sexual Orientation: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Gender: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0020.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0020.txt.redacted.redaction.json new file mode 100644 index 00000000..5025a915 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0020.txt.redacted.redaction.json @@ -0,0 +1,105 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0020.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0020.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.674, + "snippet": "Transcript #20\n==================================\nCasey: Can you assist me with updating my subscription." + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.716, + "snippet": " Can you assist me with updating my subscription.\nMaya: Thanks, can you also provide your Sexual Orienta" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.988, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\nCasey: Can you check on that?\nMaya: Let me look " + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.742, + "snippet": "so provide your Sexual Orientation: Heterosexual?\nCasey: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.665, + "snippet": "tion: Heterosexual?\nCasey: Can you check on that?\nMaya: Let me look that up for you.\nCasey: Can you chec" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.705, + "snippet": "check on that?\nMaya: Let me look that up for you.\nCasey: Can you check on that?\nMaya: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.651, + "snippet": "ok that up for you.\nCasey: Can you check on that?\nMaya: Thanks, can you also provide your Gender: Non-bi" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.975, + "snippet": "?\nMaya: Thanks, can you also provide your Gender: Non-binary?\nCasey: Can you check on that?\nMaya: Thanks, can " + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.687, + "snippet": "ks, can you also provide your Gender: Non-binary?\nCasey: Can you check on that?\nMaya: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.661, + "snippet": "Gender: Non-binary?\nCasey: Can you check on that?\nMaya: Thanks, can you also provide your Approximate Ge" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0021.txt.redacted b/examples/chats/support_chats/redacted/chat_0021.txt.redacted new file mode 100644 index 00000000..370126fc --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0021.txt.redacted @@ -0,0 +1,16 @@ +Support Chat Transcript #21 +================================== +<<>>: Can you assist me with firmware update errors. +<<>>: Thanks, can you also provide your Political Affiliation: <<>>? +<<>>: Here is my Serial Number: <<>> +<<>>: Let me look that up for you. +<<>>: Here is my Username: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Union Membership: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0021.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0021.txt.redacted.redaction.json new file mode 100644 index 00000000..6ad75bdc --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0021.txt.redacted.redaction.json @@ -0,0 +1,99 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0021.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0021.txt.redacted", + "totalRedactions": 6, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Username", + "value": "Sam", + "confidence": 0.711, + "snippet": "Transcript #21\n==================================\nSam: Can you assist me with firmware update errors.\nS" + }, + { + "type": "Username", + "value": "Sofia", + "confidence": 0.559, + "snippet": "m: Can you assist me with firmware update errors.\nSofia: Thanks, can you also provide your Political Affi" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.974, + "snippet": " can you also provide your Political Affiliation: Independent?\nSam: Here is my Serial Number: SN-1234567890\nSof" + }, + { + "type": "Username", + "value": "Sam", + "confidence": 0.734, + "snippet": " provide your Political Affiliation: Independent?\nSam: Here is my Serial Number: SN-1234567890\nSofia: L" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.996, + "snippet": "tion: Independent?\nSam: Here is my Serial Number: SN-1234567890\nSofia: Let me look that up for you.\nSam: Here is " + }, + { + "type": "Username", + "value": "Sam", + "confidence": 0.571, + "snippet": "SN-1234567890\nSofia: Let me look that up for you.\nSam: Here is my Username: jordan_92\nSofia: Let me loo" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.976, + "snippet": "e look that up for you.\nSam: Here is my Username: jordan_92\nSofia: Let me look that up for you.\nSam: Can you " + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.98, + "snippet": "hat up for you.\nSam: Here is my Union Membership: None\nSofia: Let me look that up for you.\nSam: Can you " + }, + { + "type": "Username", + "value": "Sofia", + "confidence": 0.549, + "snippet": "look that up for you.\nSam: Can you check on that?\nSofia: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0022.txt.redacted b/examples/chats/support_chats/redacted/chat_0022.txt.redacted new file mode 100644 index 00000000..d28302ad --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0022.txt.redacted @@ -0,0 +1,11 @@ +Support Chat Transcript #22 +================================== +<<>>: Hello—I'm having trouble with a double charge on my card. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your MAC Address: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Username: <<>>? +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0022.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0022.txt.redacted.redaction.json new file mode 100644 index 00000000..ad98d6bd --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0022.txt.redacted.redaction.json @@ -0,0 +1,87 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0022.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0022.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Username", + "value": "Ethan", + "confidence": 0.762, + "snippet": "m having trouble with a double charge on my card.\nEthan: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.892, + "snippet": "look that up for you.\nSam: Can you check on that?\nEthan: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.872, + "snippet": "look that up for you.\nSam: Can you check on that?\nEthan: Thanks, can you also provide your MAC Address: 0" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.999, + "snippet": "n: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\nSam: Can you check on that?\nEthan: Thanks, can y" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.891, + "snippet": "s: 00:1B:44:11:3A:B7?\nSam: Can you check on that?\nEthan: Thanks, can you also provide your Username: jord" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.976, + "snippet": "than: Thanks, can you also provide your Username: jordan_92?\nSam: Can you check on that?" + }, + { + "type": "Username", + "value": "Sam", + "confidence": 0.589, + "snippet": "s, can you also provide your Username: jordan_92?\nSam: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0023.txt.redacted b/examples/chats/support_chats/redacted/chat_0023.txt.redacted new file mode 100644 index 00000000..6965bd8e --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0023.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #23 +================================== +<<>>: Hello—I'm having trouble with firmware update errors. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your IP Address: <<>>? +<<>>: Here is my Race/Ethnicity: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0023.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0023.txt.redacted.redaction.json new file mode 100644 index 00000000..a4c9b84f --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0023.txt.redacted.redaction.json @@ -0,0 +1,123 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0023.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0023.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.875, + "snippet": "Transcript #23\n==================================\nTaylor: Hello—I'm having trouble with firmware update er" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.922, + "snippet": "o—I'm having trouble with firmware update errors.\nEthan: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.881, + "snippet": "pdate errors.\nEthan: Let me look that up for you.\nTaylor: Can you check on that?\nEthan: Thanks, can you al" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.917, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nEthan: Thanks, can you also provide your IP Address: 19" + }, + { + "type": "IP Address", + "value": "192.168.1.42", + "confidence": 0.998, + "snippet": "an: Thanks, can you also provide your IP Address: 192.168.1.42?\nTaylor: Here is my Race/Ethnicity: Hispanic\nEtha" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.909, + "snippet": "n you also provide your IP Address: 192.168.1.42?\nTaylor: Here is my Race/Ethnicity: Hispanic\nEthan: Let m" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.997, + "snippet": " 192.168.1.42?\nTaylor: Here is my Race/Ethnicity: Hispanic\nEthan: Let me look that up for you.\nTaylor: Can y" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.833, + "snippet": "1.42?\nTaylor: Here is my Race/Ethnicity: Hispanic\nEthan: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.898, + "snippet": "ity: Hispanic\nEthan: Let me look that up for you.\nTaylor: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.899, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nEthan: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.889, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nTaylor: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.925, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nEthan: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.891, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nTaylor: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0024.txt.redacted b/examples/chats/support_chats/redacted/chat_0024.txt.redacted new file mode 100644 index 00000000..9e7207a6 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0024.txt.redacted @@ -0,0 +1,15 @@ +Support Chat Transcript #24 +================================== +Sam: Hello—I'm having trouble with updating my subscription. +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Here is my Gender: <<>> +Sofia: Thanks, can you also provide your Political Affiliation: <<>>? +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Here is my User ID: <<>> +Sofia: Thanks, can you also provide your User Agent: <<>> (Macintosh; Intel Mac OS X)? +Sam: Can you check on that? +Sofia: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0024.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0024.txt.redacted.redaction.json new file mode 100644 index 00000000..ab72b028 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0024.txt.redacted.redaction.json @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0024.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0024.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.974, + "snippet": " me look that up for you.\nSam: Here is my Gender: Non-binary\nSofia: Thanks, can you also provide your Politica" + }, + { + "type": "Sexual Orientation", + "value": "Non-binary", + "confidence": 0.55, + "snippet": " me look that up for you.\nSam: Here is my Gender: Non-binary\nSofia: Thanks, can you also provide your Politica" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.992, + "snippet": " can you also provide your Political Affiliation: Independent?\nSam: Can you check on that?\nSofia: Let me look t" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.976, + "snippet": "me look that up for you.\nSam: Here is my User ID: U123456\nSofia: Thanks, can you also provide your User Age" + }, + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.953, + "snippet": "ia: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nSam: Can you check o" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0025.txt.redacted b/examples/chats/support_chats/redacted/chat_0025.txt.redacted new file mode 100644 index 00000000..83147757 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0025.txt.redacted @@ -0,0 +1,17 @@ +Support Chat Transcript #25 +================================== +<<>>: Can you assist me with logging into my account. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Email: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your User Preferences: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0025.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0025.txt.redacted.redaction.json new file mode 100644 index 00000000..41dbbf75 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0025.txt.redacted.redaction.json @@ -0,0 +1,147 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0025.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0025.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.943, + "snippet": "Transcript #25\n==================================\nTaylor: Can you assist me with logging into my account.\n" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.936, + "snippet": ": Can you assist me with logging into my account.\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.945, + "snippet": "nto my account.\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.944, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.96, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.938, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Here is my " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.96, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Here is my Email: jordan@example.com\nAva: Let me" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.998, + "snippet": "e look that up for you.\nTaylor: Here is my Email: jordan@example.com\nAva: Let me look that up for you.\nTaylor: Can you" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.915, + "snippet": "you.\nTaylor: Here is my Email: jordan@example.com\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.936, + "snippet": "dan@example.com\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.945, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Thanks, can you also provide your User Preferenc" + }, + { + "type": "User Preferences", + "value": "marketing_emails=false", + "confidence": 0.96, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\nTaylor: Can you check on that?\nAva: Let me look " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.939, + "snippet": "de your User Preferences: marketing_emails=false?\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.933, + "snippet": "ting_emails=false?\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.946, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.954, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.944, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0026.txt.redacted b/examples/chats/support_chats/redacted/chat_0026.txt.redacted new file mode 100644 index 00000000..0b5c62d6 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0026.txt.redacted @@ -0,0 +1,17 @@ +Support Chat Transcript #26 +================================== +<<>>: Can you assist me with logging into my account. +<<>>: Let me look that up for you. +<<>>: Here is my Gender: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my MAC Address: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Approximate Geolocation: San Francisco, CA +<<>>: Let me look that up for you. +<<>>: Here is my Union Membership: <<>> \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0026.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0026.txt.redacted.redaction.json new file mode 100644 index 00000000..e972bcf8 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0026.txt.redacted.redaction.json @@ -0,0 +1,153 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0026.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0026.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.941, + "snippet": "Transcript #26\n==================================\nAlex: Can you assist me with logging into my account.\n" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.91, + "snippet": ": Can you assist me with logging into my account.\nAva: Let me look that up for you.\nAlex: Here is my Ge" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.937, + "snippet": "nto my account.\nAva: Let me look that up for you.\nAlex: Here is my Gender: Non-binary\nAva: Let me look t" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.949, + "snippet": "me look that up for you.\nAlex: Here is my Gender: Non-binary\nAva: Let me look that up for you.\nAlex: Can you c" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.862, + "snippet": "t up for you.\nAlex: Here is my Gender: Non-binary\nAva: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.92, + "snippet": "der: Non-binary\nAva: Let me look that up for you.\nAlex: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.898, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nAva: Let me look that up for you.\nAlex: Here is my MA" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.931, + "snippet": " check on that?\nAva: Let me look that up for you.\nAlex: Here is my MAC Address: 00:1B:44:11:3A:B7\nAva: L" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": "ok that up for you.\nAlex: Here is my MAC Address: 00:1B:44:11:3A:B7\nAva: Let me look that up for you.\nAlex: Can you c" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.85, + "snippet": ".\nAlex: Here is my MAC Address: 00:1B:44:11:3A:B7\nAva: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.887, + "snippet": ":1B:44:11:3A:B7\nAva: Let me look that up for you.\nAlex: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.896, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nAva: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.879, + "snippet": " check on that?\nAva: Let me look that up for you.\nAlex: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.904, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nAva: Let me look that up for you.\nAlex: Here is my Ap" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.924, + "snippet": " check on that?\nAva: Let me look that up for you.\nAlex: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.89, + "snippet": " is my Approximate Geolocation: San Francisco, CA\nAva: Let me look that up for you.\nAlex: Here is my Un" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.943, + "snippet": "n Francisco, CA\nAva: Let me look that up for you.\nAlex: Here is my Union Membership: None" + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.954, + "snippet": "at up for you.\nAlex: Here is my Union Membership: None" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0027.txt.redacted b/examples/chats/support_chats/redacted/chat_0027.txt.redacted new file mode 100644 index 00000000..de5d8cfe --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0027.txt.redacted @@ -0,0 +1,16 @@ +Support Chat Transcript #27 +================================== +<<>>: Can you assist me with resetting my password. +<<>>: Thanks, can you also provide your <<>>: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your <<>>: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Profile URL: <<>>? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0027.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0027.txt.redacted.redaction.json new file mode 100644 index 00000000..a85e7bd8 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0027.txt.redacted.redaction.json @@ -0,0 +1,117 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0027.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0027.txt.redacted", + "totalRedactions": 7, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.58, + "snippet": "Transcript #27\n==================================\nTaylor: Can you assist me with resetting my password.\nSo" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.541, + "snippet": "or: Can you assist me with resetting my password.\nSofia: Thanks, can you also provide your Declared Inter" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.711, + "snippet": "assword.\nSofia: Thanks, can you also provide your Declared Interests: hiking, cooking?\nTaylor: Can you check on that?\n" + }, + { + "type": "Declared Interests", + "value": "hiking, cooking", + "confidence": 0.775, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\nTaylor: Can you check on that?\nSofia: Let me loo" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.555, + "snippet": "heck on that?\nSofia: Let me look that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that u" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.549, + "snippet": "heck on that?\nSofia: Let me look that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that u" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.573, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.513, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nSofia: Thanks, can you also provide your Religion: None" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.867, + "snippet": "on that?\nSofia: Thanks, can you also provide your Religion: None?\nTaylor: Can you check on that?\nSofia: Than" + }, + { + "type": "Religion", + "value": "None", + "confidence": 0.837, + "snippet": "ofia: Thanks, can you also provide your Religion: None?\nTaylor: Can you check on that?\nSofia: Thanks, ca" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.502, + "snippet": "ur Religion: None?\nTaylor: Can you check on that?\nSofia: Thanks, can you also provide your Profile URL: h" + }, + { + "type": "Profile URL", + "value": "https://social.example.com/jordan", + "confidence": 0.986, + "snippet": "a: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0028.txt.redacted b/examples/chats/support_chats/redacted/chat_0028.txt.redacted new file mode 100644 index 00000000..2444ec65 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0028.txt.redacted @@ -0,0 +1,19 @@ +Support Chat Transcript #28 +================================== +Alex: Hi, I need help with logging into my account. +Sofia: Thanks, can you also provide your User Preferences: <<>>? +Alex: Can you check on that? +Sofia: Thanks, can you also provide your Phone: <<>>? +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Thanks, can you also provide your Race/Ethnicity: <<>>? +Alex: Here is my Survey Data: Q1=Yes, Q2=No +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Thanks, can you also provide your Biometric Identifier: <<>>? +Alex: Here is my Approximate Geolocation: San Francisco, CA +Sofia: Thanks, can you also provide your Social Security Number: <<>>? +Alex: Here is my Username: <<>> \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0028.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0028.txt.redacted.redaction.json new file mode 100644 index 00000000..f499a687 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0028.txt.redacted.redaction.json @@ -0,0 +1,93 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0028.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0028.txt.redacted", + "totalRedactions": 6, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Preferences", + "value": "marketing_emails=false", + "confidence": 0.829, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\nAlex: Can you check on that?\nSofia: Thanks, can " + }, + { + "type": "Biometric Identifier", + "value": "marketing_emails=false", + "confidence": 0.601, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\nAlex: Can you check on that?\nSofia: Thanks, can " + }, + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.999, + "snippet": "?\nSofia: Thanks, can you also provide your Phone: +1-202-555-0172?\nAlex: Can you check on that?\nSofia: Let me look " + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.992, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\nAlex: Here is my Survey Data: Q1=Yes, Q2=No\nSofi" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.902, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\nAlex: Here is my Approximate Geolocation: San Fr" + }, + { + "type": "User Preferences", + "value": "FaceID registered", + "confidence": 0.897, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\nAlex: Here is my Approximate Geolocation: San Fr" + }, + { + "type": "Social Security Number", + "value": "123-45-6789", + "confidence": 0.999, + "snippet": "can you also provide your Social Security Number: 123-45-6789?\nAlex: Here is my Username: jordan_92" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.966, + "snippet": "y Number: 123-45-6789?\nAlex: Here is my Username: jordan_92" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0029.txt.redacted b/examples/chats/support_chats/redacted/chat_0029.txt.redacted new file mode 100644 index 00000000..3e6d3233 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0029.txt.redacted @@ -0,0 +1,20 @@ +Support Chat Transcript #29 +================================== +<<>>: Hello—I'm having trouble with a double charge on my card. +<<>>: Thanks, can you also provide your Serial Number: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Approximate Geolocation: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Approximate Geolocation: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your <<>>: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0029.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0029.txt.redacted.redaction.json new file mode 100644 index 00000000..f1073c47 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0029.txt.redacted.redaction.json @@ -0,0 +1,177 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0029.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0029.txt.redacted", + "totalRedactions": 6, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.892, + "snippet": "Transcript #29\n==================================\nJordan: Hello—I'm having trouble with a double charge on" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.829, + "snippet": "m having trouble with a double charge on my card.\nAva: Thanks, can you also provide your Serial Number:" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.996, + "snippet": " Thanks, can you also provide your Serial Number: SN-1234567890?\nJordan: Can you check on that?\nAva: Let me look " + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.81, + "snippet": "u also provide your Serial Number: SN-1234567890?\nJordan: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.765, + "snippet": "er: SN-1234567890?\nJordan: Can you check on that?\nAva: Let me look that up for you.\nJordan: Here is my " + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.826, + "snippet": " check on that?\nAva: Let me look that up for you.\nJordan: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.681, + "snippet": " is my Approximate Geolocation: San Francisco, CA\nAva: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.793, + "snippet": "n Francisco, CA\nAva: Let me look that up for you.\nJordan: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.718, + "snippet": "k that up for you.\nJordan: Can you check on that?\nAva: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.766, + "snippet": " check on that?\nAva: Let me look that up for you.\nJordan: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.725, + "snippet": "k that up for you.\nJordan: Can you check on that?\nAva: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.729, + "snippet": " check on that?\nAva: Let me look that up for you.\nJordan: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.747, + "snippet": "k that up for you.\nJordan: Can you check on that?\nAva: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.75, + "snippet": " check on that?\nAva: Let me look that up for you.\nJordan: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.778, + "snippet": "k that up for you.\nJordan: Can you check on that?\nAva: Thanks, can you also provide your Approximate Ge" + }, + { + "type": "Region", + "value": "San Francisco, CA", + "confidence": 0.594, + "snippet": "an you also provide your Approximate Geolocation: San Francisco, CA?\nJordan: Can you check on that?\nAva: Thanks, can " + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.804, + "snippet": " your Approximate Geolocation: San Francisco, CA?\nJordan: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.759, + "snippet": "San Francisco, CA?\nJordan: Can you check on that?\nAva: Thanks, can you also provide your Declared Inter" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.609, + "snippet": "k on that?\nAva: Thanks, can you also provide your Declared Interests: hiking, cooking?\nJordan: Can you check on that?\n" + }, + { + "type": "Declared Interests", + "value": "hiking, cooking", + "confidence": 0.954, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\nJordan: Can you check on that?\nAva: Let me look " + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.774, + "snippet": "provide your Declared Interests: hiking, cooking?\nJordan: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.774, + "snippet": ": hiking, cooking?\nJordan: Can you check on that?\nAva: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0030.txt.redacted b/examples/chats/support_chats/redacted/chat_0030.txt.redacted new file mode 100644 index 00000000..cc952507 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0030.txt.redacted @@ -0,0 +1,18 @@ +Support Chat Transcript #30 +================================== +<<>>: Hi, I need help with updating my subscription. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my MAC Address: <<>> +<<>>: Let me look that up for you. +<<>>: Here is my Biometric Identifier: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0030.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0030.txt.redacted.redaction.json new file mode 100644 index 00000000..a3225cbb --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0030.txt.redacted.redaction.json @@ -0,0 +1,153 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0030.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0030.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.816, + "snippet": "Transcript #30\n==================================\nJordan: Hi, I need help with updating my subscription.\nM" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.843, + "snippet": "n: Hi, I need help with updating my subscription.\nMaya: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.758, + "snippet": " subscription.\nMaya: Let me look that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.856, + "snippet": "k that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.762, + "snippet": "check on that?\nMaya: Let me look that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.761, + "snippet": "k that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up for you.\nJordan: Here is my " + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.879, + "snippet": "check on that?\nMaya: Let me look that up for you.\nJordan: Here is my MAC Address: 00:1B:44:11:3A:B7\nMaya: " + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": " that up for you.\nJordan: Here is my MAC Address: 00:1B:44:11:3A:B7\nMaya: Let me look that up for you.\nJordan: Here i" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.751, + "snippet": "Jordan: Here is my MAC Address: 00:1B:44:11:3A:B7\nMaya: Let me look that up for you.\nJordan: Here is my " + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.848, + "snippet": "1B:44:11:3A:B7\nMaya: Let me look that up for you.\nJordan: Here is my Biometric Identifier: FaceID register" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.934, + "snippet": "for you.\nJordan: Here is my Biometric Identifier: FaceID registered\nMaya: Let me look that up for you.\nJordan: Can yo" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.729, + "snippet": "ere is my Biometric Identifier: FaceID registered\nMaya: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.788, + "snippet": "eID registered\nMaya: Let me look that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.779, + "snippet": "k that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.781, + "snippet": "check on that?\nMaya: Let me look that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.793, + "snippet": "k that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.763, + "snippet": "check on that?\nMaya: Let me look that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.842, + "snippet": "k that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0031.txt.redacted b/examples/chats/support_chats/redacted/chat_0031.txt.redacted new file mode 100644 index 00000000..e64f9d38 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0031.txt.redacted @@ -0,0 +1,16 @@ +Support Chat Transcript #31 +================================== +Taylor: Hi, I need help with logging into my account. +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Username: <<>>? +Taylor: Here is my Gender: <<>> +Sofia: Thanks, can you also provide your User ID: <<>>? +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0031.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0031.txt.redacted.redaction.json new file mode 100644 index 00000000..5d79d39b --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0031.txt.redacted.redaction.json @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0031.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0031.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User ID", + "value": "jordan_92", + "confidence": 0.875, + "snippet": "ofia: Thanks, can you also provide your Username: jordan_92?\nTaylor: Here is my Gender: Non-binary\nSofia: Tha" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.826, + "snippet": "ofia: Thanks, can you also provide your Username: jordan_92?\nTaylor: Here is my Gender: Non-binary\nSofia: Tha" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.971, + "snippet": "r Username: jordan_92?\nTaylor: Here is my Gender: Non-binary\nSofia: Thanks, can you also provide your User ID:" + }, + { + "type": "Sexual Orientation", + "value": "Non-binary", + "confidence": 0.61, + "snippet": "r Username: jordan_92?\nTaylor: Here is my Gender: Non-binary\nSofia: Thanks, can you also provide your User ID:" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.967, + "snippet": "Sofia: Thanks, can you also provide your User ID: U123456?\nTaylor: Can you check on that?\nSofia: Let me loo" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0032.txt.redacted b/examples/chats/support_chats/redacted/chat_0032.txt.redacted new file mode 100644 index 00000000..975f4089 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0032.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #32 +================================== +<<>>: Can you assist me with updating my subscription. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Gender: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Biometric Identifier: <<>>? +<<>>: Here is my Race/Ethnicity: <<>> \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0032.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0032.txt.redacted.redaction.json new file mode 100644 index 00000000..fc37fb1e --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0032.txt.redacted.redaction.json @@ -0,0 +1,129 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0032.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0032.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.893, + "snippet": "Transcript #32\n==================================\nSam: Can you assist me with updating my subscription." + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.883, + "snippet": " Can you assist me with updating my subscription.\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.852, + "snippet": "y subscription.\nAva: Let me look that up for you.\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.864, + "snippet": "look that up for you.\nSam: Can you check on that?\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.833, + "snippet": " check on that?\nAva: Let me look that up for you.\nSam: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.872, + "snippet": "look that up for you.\nSam: Can you check on that?\nAva: Thanks, can you also provide your Gender: Non-bi" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.982, + "snippet": "t?\nAva: Thanks, can you also provide your Gender: Non-binary?\nSam: Can you check on that?\nAva: Let me look tha" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.825, + "snippet": "ks, can you also provide your Gender: Non-binary?\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.866, + "snippet": "r Gender: Non-binary?\nSam: Can you check on that?\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.882, + "snippet": " check on that?\nAva: Let me look that up for you.\nSam: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.889, + "snippet": "look that up for you.\nSam: Can you check on that?\nAva: Thanks, can you also provide your Biometric Iden" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.918, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\nSam: Here is my Race/Ethnicity: Hispanic" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.911, + "snippet": "ide your Biometric Identifier: FaceID registered?\nSam: Here is my Race/Ethnicity: Hispanic" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.989, + "snippet": "aceID registered?\nSam: Here is my Race/Ethnicity: Hispanic" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0033.txt.redacted b/examples/chats/support_chats/redacted/chat_0033.txt.redacted new file mode 100644 index 00000000..0112e900 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0033.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #33 +================================== +<<>>: Can you assist me with firmware update errors. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Profile URL: <<>>/<<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0033.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0033.txt.redacted.redaction.json new file mode 100644 index 00000000..27578160 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0033.txt.redacted.redaction.json @@ -0,0 +1,123 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0033.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0033.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.925, + "snippet": "Transcript #33\n==================================\nJordan: Can you assist me with firmware update errors.\nA" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.838, + "snippet": "n: Can you assist me with firmware update errors.\nAva: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.891, + "snippet": " update errors.\nAva: Let me look that up for you.\nJordan: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.871, + "snippet": "k that up for you.\nJordan: Can you check on that?\nAva: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.858, + "snippet": " check on that?\nAva: Let me look that up for you.\nJordan: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.899, + "snippet": "k that up for you.\nJordan: Can you check on that?\nAva: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.821, + "snippet": " check on that?\nAva: Let me look that up for you.\nJordan: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.887, + "snippet": "k that up for you.\nJordan: Can you check on that?\nAva: Thanks, can you also provide your Profile URL: h" + }, + { + "type": "Profile URL", + "value": "https://social.example.com", + "confidence": 0.66, + "snippet": "a: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\nJordan: Can you check on that?\nAva: Let m" + }, + { + "type": "User Agent", + "value": "jordan", + "confidence": 0.601, + "snippet": "vide your Profile URL: https://social.example.com/jordan?\nJordan: Can you check on that?\nAva: Let me look " + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.865, + "snippet": "r Profile URL: https://social.example.com/jordan?\nJordan: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.897, + "snippet": "xample.com/jordan?\nJordan: Can you check on that?\nAva: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.849, + "snippet": " check on that?\nAva: Let me look that up for you.\nJordan: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0034.txt.redacted b/examples/chats/support_chats/redacted/chat_0034.txt.redacted new file mode 100644 index 00000000..96ac6db2 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0034.txt.redacted @@ -0,0 +1,19 @@ +Support Chat Transcript #34 +================================== +Alex: Can you assist me with a late delivery. +Noah: Thanks, can you also provide your Date of Birth: <<>>? +Alex: Can you check on that? +Noah: Thanks, can you also provide your Serial Number: <<>>? +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Thanks, can you also provide your User ID: <<>>? +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Here is my Email: <<>> +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Here is my Serial Number: <<>> +Noah: Let me look that up for you. +Alex: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0034.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0034.txt.redacted.redaction.json new file mode 100644 index 00000000..1fbd1284 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0034.txt.redacted.redaction.json @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0034.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0034.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.99, + "snippet": " Thanks, can you also provide your Date of Birth: 1990-05-12?\nAlex: Can you check on that?\nNoah: Thanks, can y" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.996, + "snippet": " Thanks, can you also provide your Serial Number: SN-1234567890?\nAlex: Can you check on that?\nNoah: Let me look t" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.992, + "snippet": "\nNoah: Thanks, can you also provide your User ID: U123456?\nAlex: Can you check on that?\nNoah: Let me look t" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.998, + "snippet": " me look that up for you.\nAlex: Here is my Email: jordan@example.com\nNoah: Let me look that up for you.\nAlex: Can you " + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.995, + "snippet": " that up for you.\nAlex: Here is my Serial Number: SN-1234567890\nNoah: Let me look that up for you.\nAlex: Can you " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0035.txt.redacted b/examples/chats/support_chats/redacted/chat_0035.txt.redacted new file mode 100644 index 00000000..296cb451 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0035.txt.redacted @@ -0,0 +1,12 @@ +Support Chat Transcript #35 +================================== +Jordan: Can you assist me with a double charge on my card. +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Thanks, can you also provide your Phone: <<>>? +Jordan: Here is my Username: <<>> +Ava: Let me look that up for you. +Jordan: Can you check on that? +Ava: Thanks, can you also provide your Gender: <<>>? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0035.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0035.txt.redacted.redaction.json new file mode 100644 index 00000000..eeec1439 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0035.txt.redacted.redaction.json @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0035.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0035.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.998, + "snippet": "at?\nAva: Thanks, can you also provide your Phone: +1-202-555-0172?\nJordan: Here is my Username: jordan_92\nAva: Let " + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.985, + "snippet": "ne: +1-202-555-0172?\nJordan: Here is my Username: jordan_92\nAva: Let me look that up for you.\nJordan: Can you" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.964, + "snippet": "t?\nAva: Thanks, can you also provide your Gender: Non-binary?" + }, + { + "type": "Sexual Orientation", + "value": "Non-binary", + "confidence": 0.549, + "snippet": "t?\nAva: Thanks, can you also provide your Gender: Non-binary?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0036.txt.redacted b/examples/chats/support_chats/redacted/chat_0036.txt.redacted new file mode 100644 index 00000000..8f80ffe4 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0036.txt.redacted @@ -0,0 +1,16 @@ +Support Chat Transcript #36 +================================== +Casey: Can you assist me with a double charge on my card. +Noah: Let me look that up for you. +Casey: Can you check on that? +Noah: Thanks, can you also provide your Gender: <<>>? +Casey: Can you check on that? +Noah: Thanks, can you also provide your Sexual Orientation: <<>>? +Casey: Here is my Survey Data: Q1=Yes, Q2=No +Noah: Thanks, can you also provide your MAC Address: <<>>? +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Here is my User ID: <<>> +Noah: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0036.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0036.txt.redacted.redaction.json new file mode 100644 index 00000000..8abe1f21 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0036.txt.redacted.redaction.json @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0036.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0036.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.953, + "snippet": "?\nNoah: Thanks, can you also provide your Gender: Non-binary?\nCasey: Can you check on that?\nNoah: Thanks, can " + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.996, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\nCasey: Here is my Survey Data: Q1=Yes, Q2=No\nNoa" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.999, + "snippet": "h: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\nCasey: Can you check on that?\nNoah: Let me look " + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.984, + "snippet": " look that up for you.\nCasey: Here is my User ID: U123456\nNoah: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0037.txt.redacted b/examples/chats/support_chats/redacted/chat_0037.txt.redacted new file mode 100644 index 00000000..49f9e963 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0037.txt.redacted @@ -0,0 +1,19 @@ +Support Chat Transcript #37 +================================== +<<>>: Can you assist me with a double charge on my card. +<<>>: Thanks, can you also provide your Union Membership: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your <<>>: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your User Preferences: <<>>? +<<>>: Here is my Approximate Geolocation: <<>> +<<>>: Let me look that up for you. +<<>>: Here is my Approximate Geolocation: <<>> +<<>>: Thanks, can you also provide your Username: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your MAC Address: <<>>? +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0037.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0037.txt.redacted.redaction.json new file mode 100644 index 00000000..bb62eca1 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0037.txt.redacted.redaction.json @@ -0,0 +1,159 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0037.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0037.txt.redacted", + "totalRedactions": 9, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Username", + "value": "Ethan", + "confidence": 0.787, + "snippet": "an you assist me with a double charge on my card.\nEthan: Thanks, can you also provide your Union Membersh" + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.903, + "snippet": "anks, can you also provide your Union Membership: None?\nTaylor: Can you check on that?\nEthan: Let me loo" + }, + { + "type": "Username", + "value": "Taylor", + "confidence": 0.612, + "snippet": "can you also provide your Union Membership: None?\nTaylor: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.863, + "snippet": " Membership: None?\nTaylor: Can you check on that?\nEthan: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.906, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nEthan: Thanks, can you also provide your Political Affi" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.532, + "snippet": "on that?\nEthan: Thanks, can you also provide your Political Affiliation: Independent?\nTaylor: Can you check on that?\nEtha" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.97, + "snippet": " can you also provide your Political Affiliation: Independent?\nTaylor: Can you check on that?\nEthan: Let me loo" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.791, + "snippet": "tion: Independent?\nTaylor: Can you check on that?\nEthan: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.896, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nEthan: Thanks, can you also provide your User Preferenc" + }, + { + "type": "User Preferences", + "value": "marketing_emails=false", + "confidence": 0.967, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\nTaylor: Here is my Approximate Geolocation: San " + }, + { + "type": "Username", + "value": "Taylor", + "confidence": 0.632, + "snippet": "de your User Preferences: marketing_emails=false?\nTaylor: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "Username", + "value": "Taylor", + "confidence": 0.654, + "snippet": "Francisco, CA\nEthan: Let me look that up for you.\nTaylor: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "MAC Address", + "value": "San Francisco, CA", + "confidence": 0.626, + "snippet": " you.\nTaylor: Here is my Approximate Geolocation: San Francisco, CA\nEthan: Thanks, can you also provide your Username" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.732, + "snippet": " is my Approximate Geolocation: San Francisco, CA\nEthan: Thanks, can you also provide your Username: jord" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.956, + "snippet": "than: Thanks, can you also provide your Username: jordan_92?\nTaylor: Can you check on that?\nEthan: Thanks, ca" + }, + { + "type": "Username", + "value": "Taylor", + "confidence": 0.537, + "snippet": "s, can you also provide your Username: jordan_92?\nTaylor: Can you check on that?\nEthan: Thanks, can you al" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.922, + "snippet": "ername: jordan_92?\nTaylor: Can you check on that?\nEthan: Thanks, can you also provide your MAC Address: 0" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.995, + "snippet": "n: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\nTaylor: Can you check on that?" + }, + { + "type": "Username", + "value": "Taylor", + "confidence": 0.783, + "snippet": "also provide your MAC Address: 00:1B:44:11:3A:B7?\nTaylor: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0038.txt.redacted b/examples/chats/support_chats/redacted/chat_0038.txt.redacted new file mode 100644 index 00000000..cd2f158d --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0038.txt.redacted @@ -0,0 +1,12 @@ +Support Chat Transcript #38 +================================== +<<>>: Hello—I'm having trouble with a late delivery. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Date of Birth: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0038.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0038.txt.redacted.redaction.json new file mode 100644 index 00000000..85b50187 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0038.txt.redacted.redaction.json @@ -0,0 +1,93 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0038.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0038.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.514, + "snippet": "Transcript #38\n==================================\nJordan: Hello—I'm having trouble with a late delivery.\nE" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.711, + "snippet": "n: Hello—I'm having trouble with a late delivery.\nEthan: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.562, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nJordan: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.517, + "snippet": "k that up for you.\nJordan: Can you check on that?\nEthan: Let me look that up for you.\nJordan: Here is my " + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.524, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nJordan: Here is my Date of Birth: 1990-05-12\nEthan: Let " + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.992, + "snippet": "hat up for you.\nJordan: Here is my Date of Birth: 1990-05-12\nEthan: Let me look that up for you.\nJordan: Can y" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.505, + "snippet": "h: 1990-05-12\nEthan: Let me look that up for you.\nJordan: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.632, + "snippet": "k that up for you.\nJordan: Can you check on that?\nEthan: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0039.txt.redacted b/examples/chats/support_chats/redacted/chat_0039.txt.redacted new file mode 100644 index 00000000..c42da5ee --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0039.txt.redacted @@ -0,0 +1,19 @@ +Support Chat Transcript #39 +================================== +<<>>: Good morning, question about firmware update errors. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Serial Number: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Profile Picture URL: https://example.com/img/profile123.jpg? +<<>>: Here is my Approximate Geolocation: San Francisco, CA +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my <<>>: <<>> +<<>>: Thanks, can you also provide your Username: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Social Security Number: <<>>? +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0039.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0039.txt.redacted.redaction.json new file mode 100644 index 00000000..534e9e39 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0039.txt.redacted.redaction.json @@ -0,0 +1,141 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0039.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0039.txt.redacted", + "totalRedactions": 7, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Username", + "value": "Riley", + "confidence": 0.517, + "snippet": "update errors.\nLiam: Let me look that up for you.\nRiley: Can you check on that?\nLiam: Thanks, can you als" + }, + { + "type": "Username", + "value": "Liam", + "confidence": 0.725, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nLiam: Thanks, can you also provide your Serial Number:" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.982, + "snippet": " Thanks, can you also provide your Serial Number: SN-1234567890?\nRiley: Can you check on that?\nLiam: Thanks, can " + }, + { + "type": "Username", + "value": "Riley", + "confidence": 0.527, + "snippet": "u also provide your Serial Number: SN-1234567890?\nRiley: Can you check on that?\nLiam: Thanks, can you als" + }, + { + "type": "Username", + "value": "Liam", + "confidence": 0.781, + "snippet": "ber: SN-1234567890?\nRiley: Can you check on that?\nLiam: Thanks, can you also provide your Profile Pictur" + }, + { + "type": "Username", + "value": "Riley", + "confidence": 0.644, + "snippet": "ture URL: https://example.com/img/profile123.jpg?\nRiley: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "Username", + "value": "Liam", + "confidence": 0.717, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nLiam: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "Username", + "value": "Liam", + "confidence": 0.662, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nLiam: Let me look that up for you.\nRiley: Here is my R" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.862, + "snippet": "m: Let me look that up for you.\nRiley: Here is my Religion: None\nLiam: Thanks, can you also provide your Use" + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.711, + "snippet": "m: Let me look that up for you.\nRiley: Here is my Religion: None\nLiam: Thanks, can you also provide your Use" + }, + { + "type": "Religion", + "value": "None", + "confidence": 0.963, + "snippet": "look that up for you.\nRiley: Here is my Religion: None\nLiam: Thanks, can you also provide your Username:" + }, + { + "type": "Profile Picture Image", + "value": "None", + "confidence": 0.886, + "snippet": "look that up for you.\nRiley: Here is my Religion: None\nLiam: Thanks, can you also provide your Username:" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.975, + "snippet": "Liam: Thanks, can you also provide your Username: jordan_92?\nRiley: Can you check on that?\nLiam: Thanks, can " + }, + { + "type": "Username", + "value": "Liam", + "confidence": 0.708, + "snippet": "sername: jordan_92?\nRiley: Can you check on that?\nLiam: Thanks, can you also provide your Social Securit" + }, + { + "type": "Social Security Number", + "value": "123-45-6789", + "confidence": 0.997, + "snippet": "can you also provide your Social Security Number: 123-45-6789?\nRiley: Can you check on that?" + }, + { + "type": "Username", + "value": "Riley", + "confidence": 0.578, + "snippet": "provide your Social Security Number: 123-45-6789?\nRiley: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0040.txt.redacted b/examples/chats/support_chats/redacted/chat_0040.txt.redacted new file mode 100644 index 00000000..9d0df590 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0040.txt.redacted @@ -0,0 +1,15 @@ +Support Chat Transcript #40 +================================== +<<>>: Hello—I'm having trouble with firmware update errors. +<<>>: Thanks, can you also provide your Sexual Orientation: <<>>? +<<>>: Here is my Approximate Geolocation: <<>> +<<>>: Let me look that up for you. +<<>>: Here is my Survey Data: Q1=Yes, Q2=No +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Serial Number: <<>>? +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0040.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0040.txt.redacted.redaction.json new file mode 100644 index 00000000..a69a0677 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0040.txt.redacted.redaction.json @@ -0,0 +1,135 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0040.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0040.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.715, + "snippet": "Transcript #40\n==================================\nRiley: Hello—I'm having trouble with firmware update er" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.686, + "snippet": "o—I'm having trouble with firmware update errors.\nEthan: Thanks, can you also provide your Sexual Orienta" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.991, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\nRiley: Here is my Approximate Geolocation: San F" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.783, + "snippet": "so provide your Sexual Orientation: Heterosexual?\nRiley: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "Region", + "value": "San Francisco, CA", + "confidence": 0.561, + "snippet": "exual?\nRiley: Here is my Approximate Geolocation: San Francisco, CA\nEthan: Let me look that up for you.\nRiley: Here i" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.589, + "snippet": " is my Approximate Geolocation: San Francisco, CA\nEthan: Let me look that up for you.\nRiley: Here is my S" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.671, + "snippet": "Francisco, CA\nEthan: Let me look that up for you.\nRiley: Here is my Survey Data: Q1=Yes, Q2=No\nEthan: Let" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.723, + "snippet": "Q1=Yes, Q2=No\nEthan: Let me look that up for you.\nRiley: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.679, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nEthan: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.639, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nRiley: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.756, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nEthan: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.643, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nRiley: Can you check on that?\nEthan: Thanks, can you al" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.754, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nEthan: Thanks, can you also provide your Serial Number:" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.999, + "snippet": " Thanks, can you also provide your Serial Number: SN-1234567890?\nRiley: Can you check on that?" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.674, + "snippet": "u also provide your Serial Number: SN-1234567890?\nRiley: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0041.txt.redacted b/examples/chats/support_chats/redacted/chat_0041.txt.redacted new file mode 100644 index 00000000..05fc1498 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0041.txt.redacted @@ -0,0 +1,19 @@ +Support Chat Transcript #41 +================================== +Alex: Good morning, question about a late delivery. +Liam: Let me look that up for you. +Alex: Can you check on that? +Liam: Let me look that up for you. +Alex: Can you check on that? +Liam: Let me look that up for you. +Alex: Can you check on that? +Liam: Let me look that up for you. +Alex: Can you check on that? +Liam: Let me look that up for you. +Alex: Can you check on that? +Liam: Let me look that up for you. +Alex: Can you check on that? +Liam: Thanks, can you also provide your <<>>: <<>>? +Alex: Can you check on that? +Liam: Thanks, can you also provide your <<>>: <<>>? +Alex: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0041.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0041.txt.redacted.redaction.json new file mode 100644 index 00000000..004eaa91 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0041.txt.redacted.redaction.json @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0041.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0041.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.878, + "snippet": " on that?\nLiam: Thanks, can you also provide your Union Membership: None?\nAlex: Can you check on that?\nLiam: Thanks," + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.976, + "snippet": "anks, can you also provide your Union Membership: None?\nAlex: Can you check on that?\nLiam: Thanks, can y" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.79, + "snippet": " on that?\nLiam: Thanks, can you also provide your Declared Interests: hiking, cooking?\nAlex: Can you check on that?" + }, + { + "type": "Declared Interests", + "value": "hiking, cooking", + "confidence": 0.875, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\nAlex: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0042.txt.redacted b/examples/chats/support_chats/redacted/chat_0042.txt.redacted new file mode 100644 index 00000000..1764ee74 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0042.txt.redacted @@ -0,0 +1,19 @@ +Support Chat Transcript #42 +================================== +Sam: Can you assist me with a double charge on my card. +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Here is my Username: <<>> +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? +Maya: Let me look that up for you. +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0042.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0042.txt.redacted.redaction.json new file mode 100644 index 00000000..58af41b4 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0042.txt.redacted.redaction.json @@ -0,0 +1,51 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0042.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0042.txt.redacted", + "totalRedactions": 1, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.987, + "snippet": "e look that up for you.\nSam: Here is my Username: jordan_92\nMaya: Let me look that up for you.\nSam: Can you c" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0043.txt.redacted b/examples/chats/support_chats/redacted/chat_0043.txt.redacted new file mode 100644 index 00000000..991010e9 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0043.txt.redacted @@ -0,0 +1,14 @@ +Support Chat Transcript #43 +================================== +<<>>: Hi, I need help with updating my subscription. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Email: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0043.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0043.txt.redacted.redaction.json new file mode 100644 index 00000000..bc493252 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0043.txt.redacted.redaction.json @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0043.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0043.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.582, + "snippet": "Transcript #43\n==================================\nSam: Hi, I need help with updating my subscription.\nM" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.605, + "snippet": "check on that?\nMaya: Let me look that up for you.\nSam: Here is my Email: jordan@example.com\nMaya: Let m" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.998, + "snippet": "t me look that up for you.\nSam: Here is my Email: jordan@example.com\nMaya: Let me look that up for you.\nSam: Can you c" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.517, + "snippet": "check on that?\nMaya: Let me look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.542, + "snippet": "look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0044.txt.redacted b/examples/chats/support_chats/redacted/chat_0044.txt.redacted new file mode 100644 index 00000000..7b72bccf --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0044.txt.redacted @@ -0,0 +1,15 @@ +Support Chat Transcript #44 +================================== +Sam: Good morning, question about a double charge on my card. +Liam: Let me look that up for you. +Sam: Here is my Gender: <<>> +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Thanks, can you also provide your <<>>: <<>>? +Sam: Can you check on that? +Liam: Thanks, can you also provide your <<>>: <<>>? +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0044.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0044.txt.redacted.redaction.json new file mode 100644 index 00000000..745f0653 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0044.txt.redacted.redaction.json @@ -0,0 +1,81 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0044.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0044.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.957, + "snippet": " me look that up for you.\nSam: Here is my Gender: Non-binary\nLiam: Let me look that up for you.\nSam: Can you c" + }, + { + "type": "Sexual Orientation", + "value": "Non-binary", + "confidence": 0.811, + "snippet": " me look that up for you.\nSam: Here is my Gender: Non-binary\nLiam: Let me look that up for you.\nSam: Can you c" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.751, + "snippet": " on that?\nLiam: Thanks, can you also provide your Political Affiliation: Independent?\nSam: Can you check on that?\nLiam: T" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.979, + "snippet": " can you also provide your Political Affiliation: Independent?\nSam: Can you check on that?\nLiam: Thanks, can yo" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.917, + "snippet": " on that?\nLiam: Thanks, can you also provide your Political Affiliation: Independent?\nSam: Can you check on that?\nLiam: L" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.985, + "snippet": " can you also provide your Political Affiliation: Independent?\nSam: Can you check on that?\nLiam: Let me look th" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0045.txt.redacted b/examples/chats/support_chats/redacted/chat_0045.txt.redacted new file mode 100644 index 00000000..9b0d1704 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0045.txt.redacted @@ -0,0 +1,17 @@ +Support Chat Transcript #45 +================================== +Alex: Good morning, question about firmware update errors. +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Here is my Email: <<>> +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Thanks, can you also provide your Email: <<>>? +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Here is my Survey Data: Q1=Yes, Q2=No +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0045.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0045.txt.redacted.redaction.json new file mode 100644 index 00000000..9346ba99 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0045.txt.redacted.redaction.json @@ -0,0 +1,57 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0045.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0045.txt.redacted", + "totalRedactions": 1, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.992, + "snippet": " me look that up for you.\nAlex: Here is my Email: jordan@example.com\nSofia: Let me look that up for you.\nAlex: Can you" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.989, + "snippet": "?\nSofia: Thanks, can you also provide your Email: jordan@example.com?\nAlex: Can you check on that?\nSofia: Let me look " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0046.txt.redacted b/examples/chats/support_chats/redacted/chat_0046.txt.redacted new file mode 100644 index 00000000..0e2aa51d --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0046.txt.redacted @@ -0,0 +1,18 @@ +Support Chat Transcript #46 +================================== +<<>>: Can you assist me with updating my subscription. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Social Security Number: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your MAC Address: <<>>? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0046.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0046.txt.redacted.redaction.json new file mode 100644 index 00000000..42ab7a01 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0046.txt.redacted.redaction.json @@ -0,0 +1,153 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0046.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0046.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.945, + "snippet": "Transcript #46\n==================================\nAlex: Can you assist me with updating my subscription." + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.924, + "snippet": " Can you assist me with updating my subscription.\nNoah: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.93, + "snippet": " subscription.\nNoah: Let me look that up for you.\nAlex: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.956, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nNoah: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.906, + "snippet": "check on that?\nNoah: Let me look that up for you.\nAlex: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.955, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nNoah: Let me look that up for you.\nAlex: Here is my So" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.938, + "snippet": "check on that?\nNoah: Let me look that up for you.\nAlex: Here is my Social Security Number: 123-45-6789\nN" + }, + { + "type": "Social Security Number", + "value": "123-45-6789", + "confidence": 0.998, + "snippet": "for you.\nAlex: Here is my Social Security Number: 123-45-6789\nNoah: Let me look that up for you.\nAlex: Can you " + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.895, + "snippet": "x: Here is my Social Security Number: 123-45-6789\nNoah: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.92, + "snippet": "r: 123-45-6789\nNoah: Let me look that up for you.\nAlex: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.936, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nNoah: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.918, + "snippet": "check on that?\nNoah: Let me look that up for you.\nAlex: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.94, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nNoah: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.902, + "snippet": "check on that?\nNoah: Let me look that up for you.\nAlex: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.963, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nNoah: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.918, + "snippet": "check on that?\nNoah: Let me look that up for you.\nAlex: Can you check on that?\nNoah: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.951, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nNoah: Thanks, can you also provide your MAC Address: 0" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": "h: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0047.txt.redacted b/examples/chats/support_chats/redacted/chat_0047.txt.redacted new file mode 100644 index 00000000..21248227 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0047.txt.redacted @@ -0,0 +1,15 @@ +Support Chat Transcript #47 +================================== +<<>>: Hi, I need help with a double charge on my card. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your <<>>: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Approximate Geolocation: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0047.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0047.txt.redacted.redaction.json new file mode 100644 index 00000000..60ec03d5 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0047.txt.redacted.redaction.json @@ -0,0 +1,147 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0047.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0047.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.732, + "snippet": "Transcript #47\n==================================\nRiley: Hi, I need help with a double charge on my card." + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.736, + "snippet": " Hi, I need help with a double charge on my card.\nMaya: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.766, + "snippet": "ge on my card.\nMaya: Let me look that up for you.\nRiley: Can you check on that?\nMaya: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.765, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nMaya: Thanks, can you also provide your Union Membersh" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.799, + "snippet": " on that?\nMaya: Thanks, can you also provide your Union Membership: None?\nRiley: Can you check on that?\nMaya: Let me" + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.969, + "snippet": "anks, can you also provide your Union Membership: None?\nRiley: Can you check on that?\nMaya: Let me look " + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.763, + "snippet": "can you also provide your Union Membership: None?\nRiley: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.713, + "snippet": "n Membership: None?\nRiley: Can you check on that?\nMaya: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.759, + "snippet": "check on that?\nMaya: Let me look that up for you.\nRiley: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.676, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nMaya: Let me look that up for you.\nRiley: Here is my A" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.85, + "snippet": "check on that?\nMaya: Let me look that up for you.\nRiley: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "Region", + "value": "San Francisco, CA", + "confidence": 0.608, + "snippet": "r you.\nRiley: Here is my Approximate Geolocation: San Francisco, CA\nMaya: Let me look that up for you.\nRiley: Can you" + }, + { + "type": "IP Address", + "value": "San Francisco, CA", + "confidence": 0.551, + "snippet": "r you.\nRiley: Here is my Approximate Geolocation: San Francisco, CA\nMaya: Let me look that up for you.\nRiley: Can you" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.655, + "snippet": " is my Approximate Geolocation: San Francisco, CA\nMaya: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.767, + "snippet": " Francisco, CA\nMaya: Let me look that up for you.\nRiley: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.755, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nMaya: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.748, + "snippet": "check on that?\nMaya: Let me look that up for you.\nRiley: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0048.txt.redacted b/examples/chats/support_chats/redacted/chat_0048.txt.redacted new file mode 100644 index 00000000..0605639e --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0048.txt.redacted @@ -0,0 +1,16 @@ +Support Chat Transcript #48 +================================== +<<>>: Can you assist me with a double charge on my card. +<<>>: Thanks, can you also provide your <<>>: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY----- +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0048.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0048.txt.redacted.redaction.json new file mode 100644 index 00000000..6e10f6ca --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0048.txt.redacted.redaction.json @@ -0,0 +1,141 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0048.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0048.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.96, + "snippet": "Transcript #48\n==================================\nAlex: Can you assist me with a double charge on my car" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.96, + "snippet": "an you assist me with a double charge on my card.\nAva: Thanks, can you also provide your Political Affi" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.505, + "snippet": "n my card.\nAva: Thanks, can you also provide your Political Affiliation: Independent?\nAlex: Can you check on that?\nAva: L" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.971, + "snippet": " can you also provide your Political Affiliation: Independent?\nAlex: Can you check on that?\nAva: Let me look th" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.955, + "snippet": " provide your Political Affiliation: Independent?\nAlex: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.949, + "snippet": "iation: Independent?\nAlex: Can you check on that?\nAva: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.939, + "snippet": " check on that?\nAva: Let me look that up for you.\nAlex: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.949, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nAva: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.937, + "snippet": " check on that?\nAva: Let me look that up for you.\nAlex: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.943, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nAva: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.946, + "snippet": " check on that?\nAva: Let me look that up for you.\nAlex: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.935, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nAva: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.952, + "snippet": " check on that?\nAva: Let me look that up for you.\nAlex: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.953, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nAva: Let me look that up for you.\nAlex: Here is my Pr" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.965, + "snippet": " check on that?\nAva: Let me look that up for you.\nAlex: Here is my Private Key: -----BEGIN PRIVATE KEY--" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.96, + "snippet": "ATE KEY----- [redacted] -----END PRIVATE KEY-----\nAva: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0049.txt.redacted b/examples/chats/support_chats/redacted/chat_0049.txt.redacted new file mode 100644 index 00000000..6a2e8064 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0049.txt.redacted @@ -0,0 +1,20 @@ +Support Chat Transcript #49 +================================== +Alex: Can you assist me with resetting my password. +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Thanks, can you also provide your Race/Ethnicity: <<>>? +Alex: Can you check on that? +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Thanks, can you also provide your Biometric Identifier: <<>>? +Alex: Here is my Social Security Number: <<>> +Noah: Let me look that up for you. +Alex: Can you check on that? +Noah: Thanks, can you also provide your <<>>: <<>>? +Alex: Can you check on that? +Noah: Thanks, can you also provide your <<>>: <<>>? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0049.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0049.txt.redacted.redaction.json new file mode 100644 index 00000000..3ad28767 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0049.txt.redacted.redaction.json @@ -0,0 +1,87 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0049.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0049.txt.redacted", + "totalRedactions": 7, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.995, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\nAlex: Can you check on that?\nNoah: Let me look t" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.861, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\nAlex: Here is my Social Security Number: 123-45-" + }, + { + "type": "Social Security Number", + "value": "123-45-6789", + "confidence": 0.999, + "snippet": "istered?\nAlex: Here is my Social Security Number: 123-45-6789\nNoah: Let me look that up for you.\nAlex: Can you " + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.611, + "snippet": " on that?\nNoah: Thanks, can you also provide your Religion: None?\nAlex: Can you check on that?\nNoah: Thanks," + }, + { + "type": "Religion", + "value": "None", + "confidence": 0.933, + "snippet": "Noah: Thanks, can you also provide your Religion: None?\nAlex: Can you check on that?\nNoah: Thanks, can y" + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.539, + "snippet": " on that?\nNoah: Thanks, can you also provide your Gender: Non-binary?" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.978, + "snippet": "?\nNoah: Thanks, can you also provide your Gender: Non-binary?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0050.txt.redacted b/examples/chats/support_chats/redacted/chat_0050.txt.redacted new file mode 100644 index 00000000..19f84394 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0050.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #50 +================================== +<<>>: Good morning, question about firmware update errors. +<<>>: Let me look that up for you. +<<>>: Here is my Cookies: sessionid=abcd1234 +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0050.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0050.txt.redacted.redaction.json new file mode 100644 index 00000000..3e2a61c4 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0050.txt.redacted.redaction.json @@ -0,0 +1,111 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0050.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0050.txt.redacted", + "totalRedactions": 2, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.885, + "snippet": "Transcript #50\n==================================\nJordan: Good morning, question about firmware update err" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.774, + "snippet": "d morning, question about firmware update errors.\nMaya: Let me look that up for you.\nJordan: Here is my " + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.883, + "snippet": "update errors.\nMaya: Let me look that up for you.\nJordan: Here is my Cookies: sessionid=abcd1234\nMaya: Let" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.76, + "snippet": "u.\nJordan: Here is my Cookies: sessionid=abcd1234\nMaya: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.836, + "snippet": "ionid=abcd1234\nMaya: Let me look that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.775, + "snippet": "k that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.839, + "snippet": "check on that?\nMaya: Let me look that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.814, + "snippet": "k that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.789, + "snippet": "check on that?\nMaya: Let me look that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.852, + "snippet": "k that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.826, + "snippet": "check on that?\nMaya: Let me look that up for you.\nJordan: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0051.txt.redacted b/examples/chats/support_chats/redacted/chat_0051.txt.redacted new file mode 100644 index 00000000..a67937c8 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0051.txt.redacted @@ -0,0 +1,15 @@ +Support Chat Transcript #51 +================================== +<<>>: Good morning, question about a late delivery. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your User ID: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0051.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0051.txt.redacted.redaction.json new file mode 100644 index 00000000..a720f30f --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0051.txt.redacted.redaction.json @@ -0,0 +1,129 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0051.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0051.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.71, + "snippet": "Transcript #51\n==================================\nAlex: Good morning, question about a late delivery.\nMa" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.598, + "snippet": "ex: Good morning, question about a late delivery.\nMaya: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.665, + "snippet": "late delivery.\nMaya: Let me look that up for you.\nAlex: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.546, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nMaya: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.57, + "snippet": "check on that?\nMaya: Let me look that up for you.\nAlex: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.652, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nMaya: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.534, + "snippet": "check on that?\nMaya: Let me look that up for you.\nAlex: Can you check on that?\nMaya: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.614, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nMaya: Thanks, can you also provide your User ID: U1234" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.963, + "snippet": "\nMaya: Thanks, can you also provide your User ID: U123456?\nAlex: Can you check on that?\nMaya: Let me look t" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.628, + "snippet": "anks, can you also provide your User ID: U123456?\nAlex: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.535, + "snippet": "ur User ID: U123456?\nAlex: Can you check on that?\nMaya: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.553, + "snippet": "check on that?\nMaya: Let me look that up for you.\nAlex: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.637, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nMaya: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.539, + "snippet": "check on that?\nMaya: Let me look that up for you.\nAlex: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0052.txt.redacted b/examples/chats/support_chats/redacted/chat_0052.txt.redacted new file mode 100644 index 00000000..69133b99 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0052.txt.redacted @@ -0,0 +1,15 @@ +Support Chat Transcript #52 +================================== +<<>>: Hi, I need help with updating my subscription. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Sexual Orientation: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Cookies: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your User ID: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Email: <<>>? +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0052.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0052.txt.redacted.redaction.json new file mode 100644 index 00000000..9dee6506 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0052.txt.redacted.redaction.json @@ -0,0 +1,147 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0052.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0052.txt.redacted", + "totalRedactions": 6, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.605, + "snippet": "Transcript #52\n==================================\nTaylor: Hi, I need help with updating my subscription.\nA" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.642, + "snippet": "r: Hi, I need help with updating my subscription.\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.545, + "snippet": "y subscription.\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.585, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.546, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.592, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Thanks, can you also provide your Sexual Orienta" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.991, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\nTaylor: Can you check on that?\nAva: Thanks, can " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.503, + "snippet": "so provide your Sexual Orientation: Heterosexual?\nTaylor: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.573, + "snippet": "ion: Heterosexual?\nTaylor: Can you check on that?\nAva: Thanks, can you also provide your Cookies: sessi" + }, + { + "type": "User Preferences", + "value": "sessionid=abcd1234", + "confidence": 0.508, + "snippet": "?\nAva: Thanks, can you also provide your Cookies: sessionid=abcd1234?\nTaylor: Can you check on that?\nAva: Thanks, can " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.539, + "snippet": "ou also provide your Cookies: sessionid=abcd1234?\nTaylor: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.577, + "snippet": "essionid=abcd1234?\nTaylor: Can you check on that?\nAva: Thanks, can you also provide your User ID: U1234" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.985, + "snippet": "?\nAva: Thanks, can you also provide your User ID: U123456?\nTaylor: Can you check on that?\nAva: Thanks, can " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.528, + "snippet": "anks, can you also provide your User ID: U123456?\nTaylor: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.575, + "snippet": " User ID: U123456?\nTaylor: Can you check on that?\nAva: Thanks, can you also provide your Email: jordan@" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.998, + "snippet": "at?\nAva: Thanks, can you also provide your Email: jordan@example.com?\nTaylor: Can you check on that?" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.552, + "snippet": " you also provide your Email: jordan@example.com?\nTaylor: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0053.txt.redacted b/examples/chats/support_chats/redacted/chat_0053.txt.redacted new file mode 100644 index 00000000..adcfc7d6 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0053.txt.redacted @@ -0,0 +1,12 @@ +Support Chat Transcript #53 +================================== +Jordan: Good morning, question about resetting my password. +Noah: Thanks, can you also provide your <<>>: <<>>? +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Thanks, can you also provide your Biometric Identifier: <<>>? +Jordan: Can you check on that? +Noah: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0053.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0053.txt.redacted.redaction.json new file mode 100644 index 00000000..91538935 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0053.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0053.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0053.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.791, + "snippet": "password.\nNoah: Thanks, can you also provide your Declared Interests: hiking, cooking?\nJordan: Can you check on that?\n" + }, + { + "type": "Declared Interests", + "value": "hiking, cooking", + "confidence": 0.909, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\nJordan: Can you check on that?\nNoah: Let me look" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.88, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\nJordan: Can you check on that?\nNoah: Let me look" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0054.txt.redacted b/examples/chats/support_chats/redacted/chat_0054.txt.redacted new file mode 100644 index 00000000..12d676c6 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0054.txt.redacted @@ -0,0 +1,16 @@ +Support Chat Transcript #54 +================================== +Riley: Hi, I need help with logging into my account. +Sofia: Let me look that up for you. +Riley: Can you check on that? +Sofia: Thanks, can you also provide your <<>>: <<>>? +Riley: Can you check on that? +Sofia: Let me look that up for you. +Riley: Can you check on that? +Sofia: Thanks, can you also provide your User Agent: <<>> (<<>>)? +Riley: Can you check on that? +Sofia: Let me look that up for you. +Riley: Can you check on that? +Sofia: Let me look that up for you. +Riley: Here is my Phone: <<>> +Sofia: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0054.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0054.txt.redacted.redaction.json new file mode 100644 index 00000000..283d1fe5 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0054.txt.redacted.redaction.json @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0054.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0054.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.545, + "snippet": "on that?\nSofia: Thanks, can you also provide your User Preferences: marketing_emails=false?\nRiley: Can you check on " + }, + { + "type": "User Preferences", + "value": "marketing_emails=false", + "confidence": 0.938, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\nRiley: Can you check on that?\nSofia: Let me look" + }, + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.97, + "snippet": "ia: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nRiley: Can you check" + }, + { + "type": "MAC Address", + "value": "Macintosh; Intel Mac OS X", + "confidence": 0.523, + "snippet": "an you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nRiley: Can you check on that?\nSofia: Let me loo" + }, + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.995, + "snippet": "me look that up for you.\nRiley: Here is my Phone: +1-202-555-0172\nSofia: Thanks, can you also provide your Private " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0055.txt.redacted b/examples/chats/support_chats/redacted/chat_0055.txt.redacted new file mode 100644 index 00000000..8278bd68 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0055.txt.redacted @@ -0,0 +1,17 @@ +Support Chat Transcript #55 +================================== +Sam: Hi, I need help with logging into my account. +Ethan: Thanks, can you also provide your Biometric Identifier: <<>> registered? +Sam: Can you check on that? +Ethan: Let me look that up for you. +Sam: Can you check on that? +Ethan: Let me look that up for you. +Sam: Can you check on that? +Ethan: Let me look that up for you. +Sam: Can you check on that? +Ethan: Let me look that up for you. +Sam: Here is my User ID: <<>> +Ethan: Let me look that up for you. +Sam: Can you check on that? +Ethan: Let me look that up for you. +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0055.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0055.txt.redacted.redaction.json new file mode 100644 index 00000000..153bf4b1 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0055.txt.redacted.redaction.json @@ -0,0 +1,57 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0055.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0055.txt.redacted", + "totalRedactions": 2, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Biometric Identifier", + "value": "FaceID", + "confidence": 0.947, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\nSam: Can you check on that?\nEthan: Le" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.987, + "snippet": "me look that up for you.\nSam: Here is my User ID: U123456\nEthan: Let me look that up for you.\nSam: Can you " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0056.txt.redacted b/examples/chats/support_chats/redacted/chat_0056.txt.redacted new file mode 100644 index 00000000..9809a218 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0056.txt.redacted @@ -0,0 +1,15 @@ +Support Chat Transcript #56 +================================== +<<>>: Hello—I'm having trouble with a late delivery. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Race/Ethnicity: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0056.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0056.txt.redacted.redaction.json new file mode 100644 index 00000000..b4abc2cf --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0056.txt.redacted.redaction.json @@ -0,0 +1,129 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0056.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0056.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.584, + "snippet": "Transcript #56\n==================================\nRiley: Hello—I'm having trouble with a late delivery.\nL" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.626, + "snippet": "y: Hello—I'm having trouble with a late delivery.\nLiam: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.56, + "snippet": "late delivery.\nLiam: Let me look that up for you.\nRiley: Can you check on that?\nLiam: Let me look that up" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.684, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nLiam: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.53, + "snippet": "check on that?\nLiam: Let me look that up for you.\nRiley: Can you check on that?\nLiam: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.635, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nLiam: Thanks, can you also provide your Race/Ethnicity" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.992, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\nRiley: Can you check on that?\nLiam: Let me look " + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.575, + "snippet": "n you also provide your Race/Ethnicity: Hispanic?\nRiley: Can you check on that?\nLiam: Let me look that up" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.54, + "snippet": "thnicity: Hispanic?\nRiley: Can you check on that?\nLiam: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.686, + "snippet": "check on that?\nLiam: Let me look that up for you.\nRiley: Can you check on that?\nLiam: Let me look that up" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.57, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nLiam: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.544, + "snippet": "check on that?\nLiam: Let me look that up for you.\nRiley: Can you check on that?\nLiam: Let me look that up" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.682, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nLiam: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.529, + "snippet": "check on that?\nLiam: Let me look that up for you.\nRiley: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0057.txt.redacted b/examples/chats/support_chats/redacted/chat_0057.txt.redacted new file mode 100644 index 00000000..4a6d3c53 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0057.txt.redacted @@ -0,0 +1,19 @@ +Support Chat Transcript #57 +================================== +Riley: Good morning, question about updating my subscription. +Liam: Thanks, can you also provide your Political Affiliation: <<>>? +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Thanks, can you also provide your Sexual Orientation: <<>>? +Riley: Can you check on that? +Liam: Thanks, can you also provide your Race/Ethnicity: <<>>? +Riley: Here is my Sexual Orientation: <<>> +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Here is my Profile Picture URL: https://example.com/img/profile123.jpg +Liam: Let me look that up for you. +Riley: Can you check on that? +Liam: Let me look that up for you. +Riley: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0057.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0057.txt.redacted.redaction.json new file mode 100644 index 00000000..42ae48f7 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0057.txt.redacted.redaction.json @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0057.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0057.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.982, + "snippet": " can you also provide your Political Affiliation: Independent?\nRiley: Can you check on that?\nLiam: Let me look " + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.975, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\nRiley: Can you check on that?\nLiam: Thanks, can " + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.978, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\nRiley: Here is my Sexual Orientation: Heterosexu" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.975, + "snippet": ": Hispanic?\nRiley: Here is my Sexual Orientation: Heterosexual\nLiam: Let me look that up for you.\nRiley: Can you" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0058.txt.redacted b/examples/chats/support_chats/redacted/chat_0058.txt.redacted new file mode 100644 index 00000000..6b4ef984 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0058.txt.redacted @@ -0,0 +1,11 @@ +Support Chat Transcript #58 +================================== +<<>>: Hello—I'm having trouble with firmware update errors. +<<>>: Thanks, can you also provide your Profile URL: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Race/Ethnicity: <<>> +<<>>: Thanks, can you also provide your MAC Address: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0058.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0058.txt.redacted.redaction.json new file mode 100644 index 00000000..c82cf369 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0058.txt.redacted.redaction.json @@ -0,0 +1,117 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0058.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0058.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.693, + "snippet": "Transcript #58\n==================================\nRiley: Hello—I'm having trouble with firmware update er" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.712, + "snippet": "o—I'm having trouble with firmware update errors.\nEthan: Thanks, can you also provide your Profile URL: h" + }, + { + "type": "Profile URL", + "value": "https://social.example.com/jordan", + "confidence": 0.98, + "snippet": "n: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\nRiley: Can you check on that?\nEthan: Let me look" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.664, + "snippet": "r Profile URL: https://social.example.com/jordan?\nRiley: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.693, + "snippet": "example.com/jordan?\nRiley: Can you check on that?\nEthan: Let me look that up for you.\nRiley: Here is my R" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.663, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nRiley: Here is my Race/Ethnicity: Hispanic\nEthan: Thank" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.997, + "snippet": "hat up for you.\nRiley: Here is my Race/Ethnicity: Hispanic\nEthan: Thanks, can you also provide your MAC Addr" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.669, + "snippet": "r you.\nRiley: Here is my Race/Ethnicity: Hispanic\nEthan: Thanks, can you also provide your MAC Address: 0" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.997, + "snippet": "n: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\nRiley: Can you check on that?\nEthan: Let me look" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.716, + "snippet": "also provide your MAC Address: 00:1B:44:11:3A:B7?\nRiley: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.703, + "snippet": " 00:1B:44:11:3A:B7?\nRiley: Can you check on that?\nEthan: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.648, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nRiley: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0059.txt.redacted b/examples/chats/support_chats/redacted/chat_0059.txt.redacted new file mode 100644 index 00000000..6a262cf2 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0059.txt.redacted @@ -0,0 +1,18 @@ +Support Chat Transcript #59 +================================== +<<>>: Hi, I need help with updating my subscription. +<<>>: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my User Preferences: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my <<>>: <<>> +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0059.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0059.txt.redacted.redaction.json new file mode 100644 index 00000000..f41d1a16 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0059.txt.redacted.redaction.json @@ -0,0 +1,159 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0059.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0059.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.929, + "snippet": "Transcript #59\n==================================\nTaylor: Hi, I need help with updating my subscription.\nA" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.911, + "snippet": "r: Hi, I need help with updating my subscription.\nAva: Thanks, can you also provide your Approximate Ge" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.905, + "snippet": " your Approximate Geolocation: San Francisco, CA?\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.882, + "snippet": "San Francisco, CA?\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.912, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.863, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Here is my " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.919, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Here is my User Preferences: marketing_emails=fa" + }, + { + "type": "User Preferences", + "value": "marketing_emails=false", + "confidence": 0.941, + "snippet": " up for you.\nTaylor: Here is my User Preferences: marketing_emails=false\nAva: Let me look that up for you.\nTaylor: Can you" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.793, + "snippet": "re is my User Preferences: marketing_emails=false\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.865, + "snippet": "ng_emails=false\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.885, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.863, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.855, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.887, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.855, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Here is my " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.928, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Here is my MAC Address: 00:1B:44:11:3A:B7\nAva: L" + }, + { + "type": "Region", + "value": "MAC Address", + "confidence": 0.512, + "snippet": ": Let me look that up for you.\nTaylor: Here is my MAC Address: 00:1B:44:11:3A:B7\nAva: Let me look that up for y" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.999, + "snippet": " that up for you.\nTaylor: Here is my MAC Address: 00:1B:44:11:3A:B7\nAva: Let me look that up for you." + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.877, + "snippet": "Taylor: Here is my MAC Address: 00:1B:44:11:3A:B7\nAva: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0060.txt.redacted b/examples/chats/support_chats/redacted/chat_0060.txt.redacted new file mode 100644 index 00000000..911eedf3 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0060.txt.redacted @@ -0,0 +1,12 @@ +Support Chat Transcript #60 +================================== +Jordan: Good morning, question about resetting my password. +Ethan: Let me look that up for you. +Jordan: Can you check on that? +Ethan: Thanks, can you also provide your Serial Number: <<>>? +Jordan: Here is my Phone: <<>> +Ethan: Let me look that up for you. +Jordan: Can you check on that? +Ethan: Thanks, can you also provide your <<>>: <<>>? +Jordan: Can you check on that? +Ethan: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0060.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0060.txt.redacted.redaction.json new file mode 100644 index 00000000..90bb3ab5 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0060.txt.redacted.redaction.json @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0060.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0060.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.998, + "snippet": " Thanks, can you also provide your Serial Number: SN-1234567890?\nJordan: Here is my Phone: +1-202-555-0172\nEthan:" + }, + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.993, + "snippet": " Number: SN-1234567890?\nJordan: Here is my Phone: +1-202-555-0172\nEthan: Let me look that up for you.\nJordan: Can y" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.527, + "snippet": "on that?\nEthan: Thanks, can you also provide your Declared Interests: hiking, cooking?\nJordan: Can you check on that?\n" + }, + { + "type": "Declared Interests", + "value": "hiking, cooking", + "confidence": 0.94, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\nJordan: Can you check on that?\nEthan: Let me loo" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0061.txt.redacted b/examples/chats/support_chats/redacted/chat_0061.txt.redacted new file mode 100644 index 00000000..7aad6423 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0061.txt.redacted @@ -0,0 +1,18 @@ +Support Chat Transcript #61 +================================== +<<>>: Hi, I need help with logging into my account. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Date of Birth: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your <<>>: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0061.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0061.txt.redacted.redaction.json new file mode 100644 index 00000000..3b679f3e --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0061.txt.redacted.redaction.json @@ -0,0 +1,159 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0061.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0061.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.722, + "snippet": "Transcript #61\n==================================\nTaylor: Hi, I need help with logging into my account.\nSo" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.632, + "snippet": "or: Hi, I need help with logging into my account.\nSofia: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.689, + "snippet": "o my account.\nSofia: Let me look that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that u" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.622, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that up for you.\nTaylor: Here is my " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.629, + "snippet": "heck on that?\nSofia: Let me look that up for you.\nTaylor: Here is my Date of Birth: 1990-05-12\nSofia: Let " + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.997, + "snippet": "hat up for you.\nTaylor: Here is my Date of Birth: 1990-05-12\nSofia: Let me look that up for you.\nTaylor: Can y" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.594, + "snippet": "you.\nTaylor: Here is my Date of Birth: 1990-05-12\nSofia: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.58, + "snippet": "h: 1990-05-12\nSofia: Let me look that up for you.\nTaylor: Can you check on that?\nSofia: Thanks, can you al" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.672, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nSofia: Thanks, can you also provide your Religion: None" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.986, + "snippet": "on that?\nSofia: Thanks, can you also provide your Religion: None?\nTaylor: Can you check on that?\nSofia: Let " + }, + { + "type": "Religion", + "value": "None", + "confidence": 0.979, + "snippet": "ofia: Thanks, can you also provide your Religion: None?\nTaylor: Can you check on that?\nSofia: Let me loo" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.575, + "snippet": "Thanks, can you also provide your Religion: None?\nTaylor: Can you check on that?\nSofia: Let me look that u" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.581, + "snippet": "ur Religion: None?\nTaylor: Can you check on that?\nSofia: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.629, + "snippet": "heck on that?\nSofia: Let me look that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that u" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.538, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.63, + "snippet": "heck on that?\nSofia: Let me look that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that u" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.582, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.601, + "snippet": "heck on that?\nSofia: Let me look that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that u" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.615, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nSofia: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0062.txt.redacted b/examples/chats/support_chats/redacted/chat_0062.txt.redacted new file mode 100644 index 00000000..a98ac274 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0062.txt.redacted @@ -0,0 +1,14 @@ +Support Chat Transcript #62 +================================== +<<>>: Can you assist me with a late delivery. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0062.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0062.txt.redacted.redaction.json new file mode 100644 index 00000000..a723bfc5 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0062.txt.redacted.redaction.json @@ -0,0 +1,117 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0062.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0062.txt.redacted", + "totalRedactions": 2, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.952, + "snippet": "Transcript #62\n==================================\nRiley: Can you assist me with a late delivery.\nEthan: L" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.916, + "snippet": "==\nRiley: Can you assist me with a late delivery.\nEthan: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.949, + "snippet": "ate delivery.\nEthan: Let me look that up for you.\nRiley: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.952, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nEthan: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.921, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nRiley: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.949, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nEthan: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.939, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nRiley: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.919, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nEthan: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.952, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nRiley: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.929, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nEthan: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.94, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nRiley: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.954, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nEthan: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0063.txt.redacted b/examples/chats/support_chats/redacted/chat_0063.txt.redacted new file mode 100644 index 00000000..43da6da7 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0063.txt.redacted @@ -0,0 +1,15 @@ +Support Chat Transcript #63 +================================== +Casey: Can you assist me with a double charge on my card. +Noah: Thanks, can you also provide your Political Affiliation: <<>>? +Casey: Can you check on that? +Noah: Thanks, can you also provide your Union Membership: <<>>? +Casey: Here is my Date of Birth: <<>> +Noah: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----? +Casey: Can you check on that? +Noah: Thanks, can you also provide your Serial Number: <<>>? +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Can you check on that? +Noah: Thanks, can you also provide your Profile Picture URL: https://example.com/img/profile123.jpg? +Casey: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0063.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0063.txt.redacted.redaction.json new file mode 100644 index 00000000..1a8fd1a3 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0063.txt.redacted.redaction.json @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0063.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0063.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.989, + "snippet": " can you also provide your Political Affiliation: Independent?\nCasey: Can you check on that?\nNoah: Thanks, can " + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.98, + "snippet": "anks, can you also provide your Union Membership: None?\nCasey: Here is my Date of Birth: 1990-05-12\nNoah" + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.999, + "snippet": "embership: None?\nCasey: Here is my Date of Birth: 1990-05-12\nNoah: Thanks, can you also provide your Private K" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.99, + "snippet": " Thanks, can you also provide your Serial Number: SN-1234567890?\nCasey: Can you check on that?\nNoah: Let me look " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0064.txt.redacted b/examples/chats/support_chats/redacted/chat_0064.txt.redacted new file mode 100644 index 00000000..d58202fa --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0064.txt.redacted @@ -0,0 +1,16 @@ +Support Chat Transcript #64 +================================== +<<>>: Hello—I'm having trouble with a double charge on my card. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Social Security Number: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Sexual Orientation: <<>>? +<<>>: Here is my MAC Address: <<>> +<<>>: Thanks, can you also provide your Profile URL: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0064.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0064.txt.redacted.redaction.json new file mode 100644 index 00000000..9b3377b0 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0064.txt.redacted.redaction.json @@ -0,0 +1,153 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0064.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0064.txt.redacted", + "totalRedactions": 6, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.785, + "snippet": "Transcript #64\n==================================\nCasey: Hello—I'm having trouble with a double charge on" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.769, + "snippet": "m having trouble with a double charge on my card.\nAva: Let me look that up for you.\nCasey: Can you chec" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.79, + "snippet": "rge on my card.\nAva: Let me look that up for you.\nCasey: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.778, + "snippet": "ok that up for you.\nCasey: Can you check on that?\nAva: Thanks, can you also provide your Social Securit" + }, + { + "type": "Social Security Number", + "value": "123-45-6789", + "confidence": 0.998, + "snippet": "can you also provide your Social Security Number: 123-45-6789?\nCasey: Can you check on that?\nAva: Thanks, can y" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.792, + "snippet": "provide your Social Security Number: 123-45-6789?\nCasey: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.758, + "snippet": "umber: 123-45-6789?\nCasey: Can you check on that?\nAva: Thanks, can you also provide your Sexual Orienta" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.99, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\nCasey: Here is my MAC Address: 00:1B:44:11:3A:B7" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.842, + "snippet": "so provide your Sexual Orientation: Heterosexual?\nCasey: Here is my MAC Address: 00:1B:44:11:3A:B7\nAva: T" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.994, + "snippet": "ion: Heterosexual?\nCasey: Here is my MAC Address: 00:1B:44:11:3A:B7\nAva: Thanks, can you also provide your Profile UR" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.744, + "snippet": "\nCasey: Here is my MAC Address: 00:1B:44:11:3A:B7\nAva: Thanks, can you also provide your Profile URL: h" + }, + { + "type": "Profile URL", + "value": "https://social.example.com/jordan", + "confidence": 0.99, + "snippet": "a: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\nCasey: Can you check on that?\nAva: Let me look t" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.808, + "snippet": "r Profile URL: https://social.example.com/jordan?\nCasey: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.714, + "snippet": "example.com/jordan?\nCasey: Can you check on that?\nAva: Let me look that up for you.\nCasey: Can you chec" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.789, + "snippet": " check on that?\nAva: Let me look that up for you.\nCasey: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.766, + "snippet": "ok that up for you.\nCasey: Can you check on that?\nAva: Let me look that up for you.\nCasey: Can you chec" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.813, + "snippet": " check on that?\nAva: Let me look that up for you.\nCasey: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.776, + "snippet": "ok that up for you.\nCasey: Can you check on that?\nAva: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0065.txt.redacted b/examples/chats/support_chats/redacted/chat_0065.txt.redacted new file mode 100644 index 00000000..ac60f697 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0065.txt.redacted @@ -0,0 +1,10 @@ +Support Chat Transcript #65 +================================== +Casey: Good morning, question about a double charge on my card. +Sofia: Let me look that up for you. +Casey: Can you check on that? +Sofia: Thanks, can you also provide your Email: <<>>? +Casey: Here is my Survey Data: Q1=Yes, Q2=No +Sofia: Thanks, can you also provide your <<>>: <<>>? +Casey: Can you check on that? +Sofia: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0065.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0065.txt.redacted.redaction.json new file mode 100644 index 00000000..b2803f5a --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0065.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0065.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0065.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.994, + "snippet": "?\nSofia: Thanks, can you also provide your Email: jordan@example.com?\nCasey: Here is my Survey Data: Q1=Yes, Q2=No\nSof" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.988, + "snippet": "s, Q2=No\nSofia: Thanks, can you also provide your Religion: None?\nCasey: Can you check on that?\nSofia: Let m" + }, + { + "type": "Religion", + "value": "None", + "confidence": 0.916, + "snippet": "ofia: Thanks, can you also provide your Religion: None?\nCasey: Can you check on that?\nSofia: Let me look" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0066.txt.redacted b/examples/chats/support_chats/redacted/chat_0066.txt.redacted new file mode 100644 index 00000000..c82faa68 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0066.txt.redacted @@ -0,0 +1,11 @@ +Support Chat Transcript #66 +================================== +Sam: Good morning, question about a double charge on my card. +Noah: Thanks, can you also provide your IP Address: <<>>? +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Can you check on that? +Noah: Let me look that up for you. +Sam: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0066.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0066.txt.redacted.redaction.json new file mode 100644 index 00000000..a8eca20b --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0066.txt.redacted.redaction.json @@ -0,0 +1,51 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0066.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0066.txt.redacted", + "totalRedactions": 1, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "192.168.1.42", + "confidence": 0.997, + "snippet": "ah: Thanks, can you also provide your IP Address: 192.168.1.42?\nSam: Can you check on that?\nNoah: Let me look th" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0067.txt.redacted b/examples/chats/support_chats/redacted/chat_0067.txt.redacted new file mode 100644 index 00000000..f565102e --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0067.txt.redacted @@ -0,0 +1,15 @@ +Support Chat Transcript #67 +================================== +<<>>: Can you assist me with logging into my account. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Profile URL: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my <<>>: <<>>=<<>> \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0067.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0067.txt.redacted.redaction.json new file mode 100644 index 00000000..96aa5e38 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0067.txt.redacted.redaction.json @@ -0,0 +1,147 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0067.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0067.txt.redacted", + "totalRedactions": 6, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.951, + "snippet": "Transcript #67\n==================================\nTaylor: Can you assist me with logging into my account.\n" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.929, + "snippet": ": Can you assist me with logging into my account.\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.952, + "snippet": "nto my account.\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.925, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Here is my " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.946, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Here is my Profile URL: https://social.example.c" + }, + { + "type": "Profile URL", + "value": "https://social.example.com/jordan", + "confidence": 0.975, + "snippet": " that up for you.\nTaylor: Here is my Profile URL: https://social.example.com/jordan\nAva: Let me look that up for you.\nTaylor: Can you" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.925, + "snippet": "my Profile URL: https://social.example.com/jordan\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.925, + "snippet": "mple.com/jordan\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.93, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.929, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.927, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.948, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.92, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nAva: Let me look that up for you.\nTaylor: Here is my " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.941, + "snippet": " check on that?\nAva: Let me look that up for you.\nTaylor: Here is my User Preferences: marketing_emails=fa" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.579, + "snippet": ": Let me look that up for you.\nTaylor: Here is my User Preferences: marketing_emails=false" + }, + { + "type": "User Preferences", + "value": "marketing_emails", + "confidence": 0.568, + "snippet": " up for you.\nTaylor: Here is my User Preferences: marketing_emails=false" + }, + { + "type": "User Preferences", + "value": "false", + "confidence": 0.685, + "snippet": "or: Here is my User Preferences: marketing_emails=false" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0068.txt.redacted b/examples/chats/support_chats/redacted/chat_0068.txt.redacted new file mode 100644 index 00000000..21fd8976 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0068.txt.redacted @@ -0,0 +1,20 @@ +Support Chat Transcript #68 +================================== +<<>>: Hello—I'm having trouble with updating my subscription. +<<>>: Thanks, can you also provide your Survey Data: Q1=Yes, Q2=No? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your MAC Address: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Serial Number: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Cookies: sessionid=abcd1234 +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0068.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0068.txt.redacted.redaction.json new file mode 100644 index 00000000..856f7a90 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0068.txt.redacted.redaction.json @@ -0,0 +1,165 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0068.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0068.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.973, + "snippet": "Transcript #68\n==================================\nSam: Hello—I'm having trouble with updating my subscr" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.952, + "snippet": "I'm having trouble with updating my subscription.\nAva: Thanks, can you also provide your Survey Data: Q" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.954, + "snippet": "you also provide your Survey Data: Q1=Yes, Q2=No?\nSam: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.961, + "snippet": " Data: Q1=Yes, Q2=No?\nSam: Can you check on that?\nAva: Thanks, can you also provide your MAC Address: 0" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": "a: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\nSam: Can you check on that?\nAva: Let me look tha" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.941, + "snippet": "also provide your MAC Address: 00:1B:44:11:3A:B7?\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.943, + "snippet": "s: 00:1B:44:11:3A:B7?\nSam: Can you check on that?\nAva: Let me look that up for you.\nSam: Here is my Ser" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.966, + "snippet": " check on that?\nAva: Let me look that up for you.\nSam: Here is my Serial Number: SN-1234567890\nAva: Let" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.998, + "snippet": "k that up for you.\nSam: Here is my Serial Number: SN-1234567890\nAva: Let me look that up for you.\nSam: Can you ch" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.844, + "snippet": "you.\nSam: Here is my Serial Number: SN-1234567890\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.936, + "snippet": ": SN-1234567890\nAva: Let me look that up for you.\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.927, + "snippet": "look that up for you.\nSam: Can you check on that?\nAva: Let me look that up for you.\nSam: Here is my Coo" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.957, + "snippet": " check on that?\nAva: Let me look that up for you.\nSam: Here is my Cookies: sessionid=abcd1234\nAva: Let " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.807, + "snippet": " you.\nSam: Here is my Cookies: sessionid=abcd1234\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.926, + "snippet": "sionid=abcd1234\nAva: Let me look that up for you.\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.924, + "snippet": "look that up for you.\nSam: Can you check on that?\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.922, + "snippet": " check on that?\nAva: Let me look that up for you.\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.925, + "snippet": "look that up for you.\nSam: Can you check on that?\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.954, + "snippet": " check on that?\nAva: Let me look that up for you.\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.95, + "snippet": "look that up for you.\nSam: Can you check on that?\nAva: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0069.txt.redacted b/examples/chats/support_chats/redacted/chat_0069.txt.redacted new file mode 100644 index 00000000..ac7aa19c --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0069.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #69 +================================== +<<>>: Hello—I'm having trouble with a late delivery. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Phone: <<>>? +<<>>: Here is my Date of Birth: <<>> \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0069.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0069.txt.redacted.redaction.json new file mode 100644 index 00000000..f32fe23c --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0069.txt.redacted.redaction.json @@ -0,0 +1,111 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0069.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0069.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.582, + "snippet": "Transcript #69\n==================================\nSam: Hello—I'm having trouble with a late delivery.\nM" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.645, + "snippet": "m: Hello—I'm having trouble with a late delivery.\nMaya: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.572, + "snippet": "late delivery.\nMaya: Let me look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.627, + "snippet": "check on that?\nMaya: Let me look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.51, + "snippet": "look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.51, + "snippet": "check on that?\nMaya: Let me look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.607, + "snippet": "look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.631, + "snippet": "look that up for you.\nSam: Can you check on that?\nMaya: Thanks, can you also provide your Phone: +1-202-" + }, + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.993, + "snippet": "t?\nMaya: Thanks, can you also provide your Phone: +1-202-555-0172?\nSam: Here is my Date of Birth: 1990-05-12" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.532, + "snippet": "can you also provide your Phone: +1-202-555-0172?\nSam: Here is my Date of Birth: 1990-05-12" + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.966, + "snippet": ": +1-202-555-0172?\nSam: Here is my Date of Birth: 1990-05-12" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0070.txt.redacted b/examples/chats/support_chats/redacted/chat_0070.txt.redacted new file mode 100644 index 00000000..a89135bf --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0070.txt.redacted @@ -0,0 +1,19 @@ +Support Chat Transcript #70 +================================== +<<>>: Can you assist me with a late delivery. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your <<>>: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0070.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0070.txt.redacted.redaction.json new file mode 100644 index 00000000..aa178ff7 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0070.txt.redacted.redaction.json @@ -0,0 +1,159 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0070.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0070.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.9, + "snippet": "Transcript #70\n==================================\nRiley: Can you assist me with a late delivery.\nNoah: Le" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.813, + "snippet": "==\nRiley: Can you assist me with a late delivery.\nNoah: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.894, + "snippet": "late delivery.\nNoah: Let me look that up for you.\nRiley: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.822, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nNoah: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.859, + "snippet": "check on that?\nNoah: Let me look that up for you.\nRiley: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.872, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nNoah: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.812, + "snippet": "check on that?\nNoah: Let me look that up for you.\nRiley: Can you check on that?\nNoah: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.873, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nNoah: Thanks, can you also provide your Religion: None" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.991, + "snippet": " on that?\nNoah: Thanks, can you also provide your Religion: None?\nRiley: Can you check on that?\nNoah: Let me" + }, + { + "type": "Religion", + "value": "None", + "confidence": 0.863, + "snippet": "Noah: Thanks, can you also provide your Religion: None?\nRiley: Can you check on that?\nNoah: Let me look " + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.786, + "snippet": "Thanks, can you also provide your Religion: None?\nRiley: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.83, + "snippet": "our Religion: None?\nRiley: Can you check on that?\nNoah: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.866, + "snippet": "check on that?\nNoah: Let me look that up for you.\nRiley: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.788, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nNoah: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.852, + "snippet": "check on that?\nNoah: Let me look that up for you.\nRiley: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.835, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nNoah: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.803, + "snippet": "check on that?\nNoah: Let me look that up for you.\nRiley: Can you check on that?\nNoah: Let me look that up" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.869, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nNoah: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.826, + "snippet": "check on that?\nNoah: Let me look that up for you.\nRiley: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0071.txt.redacted b/examples/chats/support_chats/redacted/chat_0071.txt.redacted new file mode 100644 index 00000000..733d65c0 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0071.txt.redacted @@ -0,0 +1,11 @@ +Support Chat Transcript #71 +================================== +<<>>: Good morning, question about updating my subscription. +<<>>: Thanks, can you also provide your <<>>: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Serial Number: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your MAC Address: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Cookies: sessionid=abcd1234? +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0071.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0071.txt.redacted.redaction.json new file mode 100644 index 00000000..898abc37 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0071.txt.redacted.redaction.json @@ -0,0 +1,123 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0071.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0071.txt.redacted", + "totalRedactions": 6, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.8, + "snippet": "Transcript #71\n==================================\nRiley: Good morning, question about updating my subscri" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.811, + "snippet": "morning, question about updating my subscription.\nNoah: Thanks, can you also provide your Declared Inter" + }, + { + "type": "User Preferences", + "value": "Declared Interests", + "confidence": 0.514, + "snippet": "cription.\nNoah: Thanks, can you also provide your Declared Interests: hiking, cooking?\nRiley: Can you check on that?\nN" + }, + { + "type": "Declared Interests", + "value": "hiking, cooking", + "confidence": 0.902, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\nRiley: Can you check on that?\nNoah: Thanks, can " + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.742, + "snippet": "provide your Declared Interests: hiking, cooking?\nRiley: Can you check on that?\nNoah: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.79, + "snippet": "s: hiking, cooking?\nRiley: Can you check on that?\nNoah: Thanks, can you also provide your Serial Number:" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.994, + "snippet": " Thanks, can you also provide your Serial Number: SN-1234567890?\nRiley: Can you check on that?\nNoah: Thanks, can " + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.731, + "snippet": "u also provide your Serial Number: SN-1234567890?\nRiley: Can you check on that?\nNoah: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.782, + "snippet": "ber: SN-1234567890?\nRiley: Can you check on that?\nNoah: Thanks, can you also provide your MAC Address: 0" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": "h: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\nRiley: Can you check on that?\nNoah: Thanks, can " + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.75, + "snippet": "also provide your MAC Address: 00:1B:44:11:3A:B7?\nRiley: Can you check on that?\nNoah: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.786, + "snippet": " 00:1B:44:11:3A:B7?\nRiley: Can you check on that?\nNoah: Thanks, can you also provide your Cookies: sessi" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.771, + "snippet": "ou also provide your Cookies: sessionid=abcd1234?\nRiley: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0072.txt.redacted b/examples/chats/support_chats/redacted/chat_0072.txt.redacted new file mode 100644 index 00000000..591a4423 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0072.txt.redacted @@ -0,0 +1,11 @@ +Support Chat Transcript #72 +================================== +Taylor: Can you assist me with a double charge on my card. +Ethan: Thanks, can you also provide your Profile Picture URL: https://example.com/img/profile123.jpg? +Taylor: Can you check on that? +Ethan: Let me look that up for you. +Taylor: Here is my Approximate Geolocation: <<>> +Ethan: Thanks, can you also provide your Gender: <<>>? +Taylor: Here is my Date of Birth: <<>> +Ethan: Let me look that up for you. +Taylor: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0072.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0072.txt.redacted.redaction.json new file mode 100644 index 00000000..4f8f451f --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0072.txt.redacted.redaction.json @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0072.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0072.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Region", + "value": "San Francisco, CA", + "confidence": 0.57, + "snippet": " you.\nTaylor: Here is my Approximate Geolocation: San Francisco, CA\nEthan: Thanks, can you also provide your Gender: " + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.983, + "snippet": "\nEthan: Thanks, can you also provide your Gender: Non-binary?\nTaylor: Here is my Date of Birth: 1990-05-12\nEth" + }, + { + "type": "Sexual Orientation", + "value": "Non-binary", + "confidence": 0.755, + "snippet": "\nEthan: Thanks, can you also provide your Gender: Non-binary?\nTaylor: Here is my Date of Birth: 1990-05-12\nEth" + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.996, + "snippet": "er: Non-binary?\nTaylor: Here is my Date of Birth: 1990-05-12\nEthan: Let me look that up for you.\nTaylor: Can y" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0073.txt.redacted b/examples/chats/support_chats/redacted/chat_0073.txt.redacted new file mode 100644 index 00000000..64666941 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0073.txt.redacted @@ -0,0 +1,12 @@ +Support Chat Transcript #73 +================================== +Alex: Hi, I need help with a late delivery. +Ethan: Thanks, can you also provide your User ID: <<>>? +Alex: Can you check on that? +Ethan: Thanks, can you also provide your Gender: <<>>? +Alex: Can you check on that? +Ethan: Let me look that up for you. +Alex: Can you check on that? +Ethan: Let me look that up for you. +Alex: Can you check on that? +Ethan: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0073.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0073.txt.redacted.redaction.json new file mode 100644 index 00000000..36452872 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0073.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0073.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0073.txt.redacted", + "totalRedactions": 2, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User ID", + "value": "U123456", + "confidence": 0.983, + "snippet": "Ethan: Thanks, can you also provide your User ID: U123456?\nAlex: Can you check on that?\nEthan: Thanks, can " + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.961, + "snippet": "\nEthan: Thanks, can you also provide your Gender: Non-binary?\nAlex: Can you check on that?\nEthan: Let me look " + }, + { + "type": "Sexual Orientation", + "value": "Non-binary", + "confidence": 0.658, + "snippet": "\nEthan: Thanks, can you also provide your Gender: Non-binary?\nAlex: Can you check on that?\nEthan: Let me look " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0074.txt.redacted b/examples/chats/support_chats/redacted/chat_0074.txt.redacted new file mode 100644 index 00000000..2481b0c9 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0074.txt.redacted @@ -0,0 +1,17 @@ +Support Chat Transcript #74 +================================== +Taylor: Hello—I'm having trouble with a late delivery. +Liam: Thanks, can you also provide your Profile URL: <<>>? +Taylor: Can you check on that? +Liam: Let me look that up for you. +Taylor: Here is my Approximate Geolocation: San Francisco, CA +Liam: Let me look that up for you. +Taylor: Can you check on that? +Liam: Let me look that up for you. +Taylor: Can you check on that? +Liam: Let me look that up for you. +Taylor: Can you check on that? +Liam: Thanks, can you also provide your Biometric Identifier: <<>>? +Taylor: Can you check on that? +Liam: Let me look that up for you. +Taylor: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0074.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0074.txt.redacted.redaction.json new file mode 100644 index 00000000..2255b714 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0074.txt.redacted.redaction.json @@ -0,0 +1,57 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0074.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0074.txt.redacted", + "totalRedactions": 2, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Profile URL", + "value": "https://social.example.com/jordan", + "confidence": 0.974, + "snippet": "m: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\nTaylor: Can you check on that?\nLiam: Let me look" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.951, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\nTaylor: Can you check on that?\nLiam: Let me look" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0075.txt.redacted b/examples/chats/support_chats/redacted/chat_0075.txt.redacted new file mode 100644 index 00000000..3b328cb0 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0075.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #75 +================================== +Riley: Hi, I need help with logging into my account. +Noah: Let me look that up for you. +Riley: Here is my Profile Picture URL: <<>>/img/profile123.jpg +Noah: Thanks, can you also provide your Email: <<>>? +Riley: Can you check on that? +Noah: Thanks, can you also provide your Profile URL: <<>>/<<>>? +Riley: Can you check on that? +Noah: Let me look that up for you. +Riley: Here is my Phone: <<>> +Noah: Let me look that up for you. +Riley: Here is my Gender: <<>> \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0075.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0075.txt.redacted.redaction.json new file mode 100644 index 00000000..ecd02f92 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0075.txt.redacted.redaction.json @@ -0,0 +1,87 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0075.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0075.txt.redacted", + "totalRedactions": 6, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Profile URL", + "value": "https://example.com", + "confidence": 0.538, + "snippet": "p for you.\nRiley: Here is my Profile Picture URL: https://example.com/img/profile123.jpg\nNoah: Thanks, can you also pro" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.982, + "snippet": "pg\nNoah: Thanks, can you also provide your Email: jordan@example.com?\nRiley: Can you check on that?\nNoah: Thanks, can " + }, + { + "type": "Profile URL", + "value": "https://social.example.com", + "confidence": 0.784, + "snippet": "h: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\nRiley: Can you check on that?\nNoah: Let m" + }, + { + "type": "Username", + "value": "jordan", + "confidence": 0.506, + "snippet": "vide your Profile URL: https://social.example.com/jordan?\nRiley: Can you check on that?\nNoah: Let me look " + }, + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.994, + "snippet": "me look that up for you.\nRiley: Here is my Phone: +1-202-555-0172\nNoah: Let me look that up for you.\nRiley: Here is" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.976, + "snippet": "e look that up for you.\nRiley: Here is my Gender: Non-binary" + }, + { + "type": "Sexual Orientation", + "value": "Non-binary", + "confidence": 0.706, + "snippet": "e look that up for you.\nRiley: Here is my Gender: Non-binary" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0076.txt.redacted b/examples/chats/support_chats/redacted/chat_0076.txt.redacted new file mode 100644 index 00000000..c7e4c6c0 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0076.txt.redacted @@ -0,0 +1,20 @@ +Support Chat Transcript #76 +================================== +Jordan: Good morning, question about updating my subscription. +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----? +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Let me look that up for you. +Jordan: Can you check on that? +Noah: Thanks, can you also provide your User Agent: <<>> (Macintosh; Intel Mac OS X)? +Jordan: Can you check on that? +Noah: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0076.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0076.txt.redacted.redaction.json new file mode 100644 index 00000000..a110041a --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0076.txt.redacted.redaction.json @@ -0,0 +1,51 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0076.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0076.txt.redacted", + "totalRedactions": 1, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.978, + "snippet": "ah: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nJordan: Can you chec" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0077.txt.redacted b/examples/chats/support_chats/redacted/chat_0077.txt.redacted new file mode 100644 index 00000000..5f61d469 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0077.txt.redacted @@ -0,0 +1,14 @@ +Support Chat Transcript #77 +================================== +<<>>: Hi, I need help with firmware update errors. +<<>>: Let me look that up for you. +<<>>: Here is my Serial Number: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Gender: <<>> +<<>>: Thanks, can you also provide your Declared Interests: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0077.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0077.txt.redacted.redaction.json new file mode 100644 index 00000000..f3fc8b17 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0077.txt.redacted.redaction.json @@ -0,0 +1,105 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0077.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0077.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.678, + "snippet": "Transcript #77\n==================================\nJordan: Hi, I need help with firmware update errors.\nAva" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.554, + "snippet": "dan: Hi, I need help with firmware update errors.\nAva: Let me look that up for you.\nJordan: Here is my " + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.673, + "snippet": " update errors.\nAva: Let me look that up for you.\nJordan: Here is my Serial Number: SN-1234567890\nAva: Let" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.999, + "snippet": "hat up for you.\nJordan: Here is my Serial Number: SN-1234567890\nAva: Let me look that up for you.\nJordan: Can you" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.506, + "snippet": ".\nJordan: Here is my Serial Number: SN-1234567890\nAva: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.522, + "snippet": "k that up for you.\nJordan: Can you check on that?\nAva: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.585, + "snippet": " check on that?\nAva: Let me look that up for you.\nJordan: Here is my Gender: Non-binary\nAva: Thanks, can y" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.977, + "snippet": " look that up for you.\nJordan: Here is my Gender: Non-binary\nAva: Thanks, can you also provide your Declared I" + }, + { + "type": "Declared Interests", + "value": "hiking, cooking", + "confidence": 0.914, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\nJordan: Can you check on that?\nAva: Let me look " + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.559, + "snippet": "provide your Declared Interests: hiking, cooking?\nJordan: Can you check on that?\nAva: Let me look that up " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0078.txt.redacted b/examples/chats/support_chats/redacted/chat_0078.txt.redacted new file mode 100644 index 00000000..0ac3952d --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0078.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #78 +================================== +Jordan: Good morning, question about logging into my account. +Ethan: Let me look that up for you. +Jordan: Here is my <<>>: <<>> +Ethan: Thanks, can you also provide your User Agent: <<>> (Macintosh; Intel Mac OS X)? +Jordan: Can you check on that? +Ethan: Let me look that up for you. +Jordan: Can you check on that? +Ethan: Thanks, can you also provide your MAC Address: <<>>? +Jordan: Can you check on that? +Ethan: Let me look that up for you. +Jordan: Here is my Declared Interests: <<>> \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0078.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0078.txt.redacted.redaction.json new file mode 100644 index 00000000..700343d0 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0078.txt.redacted.redaction.json @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0078.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0078.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Religion", + "value": "Religion", + "confidence": 0.856, + "snippet": ": Let me look that up for you.\nJordan: Here is my Religion: None\nEthan: Thanks, can you also provide your Us" + }, + { + "type": "Religion", + "value": "None", + "confidence": 0.888, + "snippet": "ook that up for you.\nJordan: Here is my Religion: None\nEthan: Thanks, can you also provide your User Age" + }, + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.964, + "snippet": "an: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nJordan: Can you chec" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.999, + "snippet": "n: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\nJordan: Can you check on that?\nEthan: Let me loo" + }, + { + "type": "Declared Interests", + "value": "hiking, cooking", + "confidence": 0.572, + "snippet": "p for you.\nJordan: Here is my Declared Interests: hiking, cooking" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0079.txt.redacted b/examples/chats/support_chats/redacted/chat_0079.txt.redacted new file mode 100644 index 00000000..de53d6e2 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0079.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #79 +================================== +Jordan: Hi, I need help with a late delivery. +Ethan: Thanks, can you also provide your <<>>: <<>>? +Jordan: Can you check on that? +Ethan: Thanks, can you also provide your User Preferences: <<>>=false? +Jordan: Here is my Phone: <<>> +Ethan: Thanks, can you also provide your User Preferences: <<>>=false? +Jordan: Can you check on that? +Ethan: Let me look that up for you. +Jordan: Can you check on that? +Ethan: Let me look that up for you. +Jordan: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0079.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0079.txt.redacted.redaction.json new file mode 100644 index 00000000..e9e0263a --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0079.txt.redacted.redaction.json @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0079.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0079.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.648, + "snippet": "elivery.\nEthan: Thanks, can you also provide your Political Affiliation: Independent?\nJordan: Can you check on that?\nEtha" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.977, + "snippet": " can you also provide your Political Affiliation: Independent?\nJordan: Can you check on that?\nEthan: Thanks, ca" + }, + { + "type": "User Preferences", + "value": "marketing_emails", + "confidence": 0.948, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\nJordan: Here is my Phone: +1-202-555-0172\n" + }, + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.995, + "snippet": "marketing_emails=false?\nJordan: Here is my Phone: +1-202-555-0172\nEthan: Thanks, can you also provide your User Pre" + }, + { + "type": "User Preferences", + "value": "marketing_emails=false", + "confidence": 0.95, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\nJordan: Can you check on that?\nEthan: Let me loo" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0080.txt.redacted b/examples/chats/support_chats/redacted/chat_0080.txt.redacted new file mode 100644 index 00000000..782cfdc0 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0080.txt.redacted @@ -0,0 +1,11 @@ +Support Chat Transcript #80 +================================== +<<>>: Good morning, question about logging into my account. +<<>>: Thanks, can you also provide your Gender: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Approximate Geolocation: San Francisco, CA? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0080.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0080.txt.redacted.redaction.json new file mode 100644 index 00000000..be56dfd5 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0080.txt.redacted.redaction.json @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0080.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0080.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.522, + "snippet": "Transcript #80\n==================================\nJordan: Good morning, question about logging into my acc" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.559, + "snippet": " morning, question about logging into my account.\nMaya: Thanks, can you also provide your Gender: Non-bi" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.919, + "snippet": ".\nMaya: Thanks, can you also provide your Gender: Non-binary?\nJordan: Can you check on that?\nMaya: Thanks, can" + }, + { + "type": "Sexual Orientation", + "value": "Non-binary", + "confidence": 0.711, + "snippet": ".\nMaya: Thanks, can you also provide your Gender: Non-binary?\nJordan: Can you check on that?\nMaya: Thanks, can" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0081.txt.redacted b/examples/chats/support_chats/redacted/chat_0081.txt.redacted new file mode 100644 index 00000000..a2160ed9 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0081.txt.redacted @@ -0,0 +1,16 @@ +Support Chat Transcript #81 +================================== +Riley: Hi, I need help with firmware update errors. +<<>>: Let me look that up for you. +Riley: Here is my Profile URL: <<>> +<<>>: Let me look that up for you. +Riley: Can you check on that? +<<>>: Let me look that up for you. +Riley: Can you check on that? +<<>>: Let me look that up for you. +Riley: Here is my User Agent: <<>> (Macintosh; Intel Mac OS X) +<<>>: Let me look that up for you. +Riley: Here is my Social Security Number: <<>> +<<>>: Let me look that up for you. +Riley: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0081.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0081.txt.redacted.redaction.json new file mode 100644 index 00000000..204f144a --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0081.txt.redacted.redaction.json @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0081.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0081.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Profile URL", + "value": "https://social.example.com/jordan", + "confidence": 0.883, + "snippet": "k that up for you.\nRiley: Here is my Profile URL: https://social.example.com/jordan\nNoah: Let me look that up for you.\nRiley: Can you" + }, + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.913, + "snippet": "ok that up for you.\nRiley: Here is my User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)\nNoah: Let me look tha" + }, + { + "type": "Social Security Number", + "value": "123-45-6789", + "confidence": 0.999, + "snippet": "or you.\nRiley: Here is my Social Security Number: 123-45-6789\nNoah: Let me look that up for you.\nRiley: Can you" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.531, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nNoah: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0082.txt.redacted b/examples/chats/support_chats/redacted/chat_0082.txt.redacted new file mode 100644 index 00000000..ef49f302 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0082.txt.redacted @@ -0,0 +1,10 @@ +Support Chat Transcript #82 +================================== +<<>>: Good morning, question about a late delivery. +Maya: Let me look that up for you. +<<>>: Can you check on that? +Maya: Thanks, can you also provide your Race/Ethnicity: <<>>? +<<>>: Here is my Username: <<>> +Maya: Let me look that up for you. +<<>>: Can you check on that? +Maya: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0082.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0082.txt.redacted.redaction.json new file mode 100644 index 00000000..1cd208e9 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0082.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0082.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0082.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Username", + "value": "Riley", + "confidence": 0.51, + "snippet": "late delivery.\nMaya: Let me look that up for you.\nRiley: Can you check on that?\nMaya: Thanks, can you als" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.991, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\nRiley: Here is my Username: jordan_92\nMaya: Let " + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.969, + "snippet": "/Ethnicity: Hispanic?\nRiley: Here is my Username: jordan_92\nMaya: Let me look that up for you.\nRiley: Can you" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0083.txt.redacted b/examples/chats/support_chats/redacted/chat_0083.txt.redacted new file mode 100644 index 00000000..1ddf5ed9 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0083.txt.redacted @@ -0,0 +1,20 @@ +Support Chat Transcript #83 +================================== +Sam: Hi, I need help with logging into my account. +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Thanks, can you also provide your User Agent: <<>> (Macintosh; Intel Mac OS X)? +Sam: Can you check on that? +Sofia: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----? +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Here is my Race/Ethnicity: <<>> +Sofia: Let me look that up for you. +Sam: Here is my Serial Number: <<>> +Sofia: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0083.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0083.txt.redacted.redaction.json new file mode 100644 index 00000000..ef88e67f --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0083.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0083.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0083.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.982, + "snippet": "ia: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nSam: Can you check o" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.993, + "snippet": " that up for you.\nSam: Here is my Race/Ethnicity: Hispanic\nSofia: Let me look that up for you.\nSam: Here is " + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.998, + "snippet": "k that up for you.\nSam: Here is my Serial Number: SN-1234567890\nSofia: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0084.txt.redacted b/examples/chats/support_chats/redacted/chat_0084.txt.redacted new file mode 100644 index 00000000..e9bc7ba6 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0084.txt.redacted @@ -0,0 +1,18 @@ +Support Chat Transcript #84 +================================== +Taylor: Hello—I'm having trouble with logging into my account. +Noah: Thanks, can you also provide your Sexual Orientation: <<>>? +Taylor: Can you check on that? +Noah: Let me look that up for you. +Taylor: Can you check on that? +Noah: Thanks, can you also provide your User Agent: <<>> (Macintosh; Intel Mac OS X)? +Taylor: Here is my Survey Data: Q1=Yes, Q2=No +Noah: Let me look that up for you. +Taylor: Can you check on that? +Noah: Let me look that up for you. +Taylor: Can you check on that? +Noah: Let me look that up for you. +Taylor: Can you check on that? +Noah: Let me look that up for you. +Taylor: Here is my Survey Data: Q1=Yes, Q2=No +Noah: Thanks, can you also provide your Gender: <<>>? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0084.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0084.txt.redacted.redaction.json new file mode 100644 index 00000000..1fd4b0c6 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0084.txt.redacted.redaction.json @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0084.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0084.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.988, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\nTaylor: Can you check on that?\nNoah: Let me look" + }, + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.967, + "snippet": "ah: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nTaylor: Here is my S" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.97, + "snippet": "o\nNoah: Thanks, can you also provide your Gender: Non-binary?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0085.txt.redacted b/examples/chats/support_chats/redacted/chat_0085.txt.redacted new file mode 100644 index 00000000..3c510306 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0085.txt.redacted @@ -0,0 +1,19 @@ +Support Chat Transcript #85 +================================== +<<>>: Hi, I need help with updating my subscription. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my IP Address: <<>> +<<>>: Thanks, can you also provide your Email: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your MAC Address: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0085.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0085.txt.redacted.redaction.json new file mode 100644 index 00000000..c787695c --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0085.txt.redacted.redaction.json @@ -0,0 +1,165 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0085.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0085.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.917, + "snippet": "Transcript #85\n==================================\nRiley: Hi, I need help with updating my subscription.\nM" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.834, + "snippet": "y: Hi, I need help with updating my subscription.\nMaya: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.933, + "snippet": " subscription.\nMaya: Let me look that up for you.\nRiley: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.857, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nMaya: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.931, + "snippet": "check on that?\nMaya: Let me look that up for you.\nRiley: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.861, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nMaya: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.895, + "snippet": "check on that?\nMaya: Let me look that up for you.\nRiley: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.87, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nMaya: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.892, + "snippet": "check on that?\nMaya: Let me look that up for you.\nRiley: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.852, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nMaya: Let me look that up for you.\nRiley: Here is my I" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.932, + "snippet": "check on that?\nMaya: Let me look that up for you.\nRiley: Here is my IP Address: 192.168.1.42\nMaya: Thanks" + }, + { + "type": "IP Address", + "value": "192.168.1.42", + "confidence": 0.994, + "snippet": "ok that up for you.\nRiley: Here is my IP Address: 192.168.1.42\nMaya: Thanks, can you also provide your Email: jo" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.766, + "snippet": "r you.\nRiley: Here is my IP Address: 192.168.1.42\nMaya: Thanks, can you also provide your Email: jordan@" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.997, + "snippet": "42\nMaya: Thanks, can you also provide your Email: jordan@example.com?\nRiley: Can you check on that?\nMaya: Thanks, can " + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.852, + "snippet": " you also provide your Email: jordan@example.com?\nRiley: Can you check on that?\nMaya: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.868, + "snippet": "jordan@example.com?\nRiley: Can you check on that?\nMaya: Thanks, can you also provide your MAC Address: 0" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.993, + "snippet": "a: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\nRiley: Can you check on that?\nMaya: Let me look " + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.856, + "snippet": "also provide your MAC Address: 00:1B:44:11:3A:B7?\nRiley: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.888, + "snippet": " 00:1B:44:11:3A:B7?\nRiley: Can you check on that?\nMaya: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.894, + "snippet": "check on that?\nMaya: Let me look that up for you.\nRiley: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0086.txt.redacted b/examples/chats/support_chats/redacted/chat_0086.txt.redacted new file mode 100644 index 00000000..db9cb697 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0086.txt.redacted @@ -0,0 +1,11 @@ +Support Chat Transcript #86 +================================== +Alex: Good morning, question about a double charge on my card. +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Thanks, can you also provide your Race/Ethnicity: <<>>? +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Can you check on that? +Sofia: Let me look that up for you. +Alex: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0086.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0086.txt.redacted.redaction.json new file mode 100644 index 00000000..5701b29a --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0086.txt.redacted.redaction.json @@ -0,0 +1,51 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0086.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0086.txt.redacted", + "totalRedactions": 1, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.989, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\nAlex: Can you check on that?\nSofia: Let me look " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0087.txt.redacted b/examples/chats/support_chats/redacted/chat_0087.txt.redacted new file mode 100644 index 00000000..98aad123 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0087.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #87 +================================== +<<>>: Hello—I'm having trouble with firmware update errors. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0087.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0087.txt.redacted.redaction.json new file mode 100644 index 00000000..d54579ce --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0087.txt.redacted.redaction.json @@ -0,0 +1,111 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0087.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0087.txt.redacted", + "totalRedactions": 2, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.783, + "snippet": "Transcript #87\n==================================\nJordan: Hello—I'm having trouble with firmware update er" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.849, + "snippet": "o—I'm having trouble with firmware update errors.\nLiam: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.739, + "snippet": "update errors.\nLiam: Let me look that up for you.\nJordan: Can you check on that?\nLiam: Let me look that up" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.843, + "snippet": "k that up for you.\nJordan: Can you check on that?\nLiam: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.779, + "snippet": "check on that?\nLiam: Let me look that up for you.\nJordan: Can you check on that?\nLiam: Let me look that up" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.786, + "snippet": "k that up for you.\nJordan: Can you check on that?\nLiam: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.799, + "snippet": "check on that?\nLiam: Let me look that up for you.\nJordan: Can you check on that?\nLiam: Let me look that up" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.833, + "snippet": "k that up for you.\nJordan: Can you check on that?\nLiam: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.709, + "snippet": "check on that?\nLiam: Let me look that up for you.\nJordan: Can you check on that?\nLiam: Let me look that up" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.88, + "snippet": "k that up for you.\nJordan: Can you check on that?\nLiam: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.767, + "snippet": "check on that?\nLiam: Let me look that up for you.\nJordan: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0088.txt.redacted b/examples/chats/support_chats/redacted/chat_0088.txt.redacted new file mode 100644 index 00000000..1c8216fa --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0088.txt.redacted @@ -0,0 +1,20 @@ +Support Chat Transcript #88 +================================== +Sam: Can you assist me with a double charge on my card. +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Can you check on that? +Sofia: Thanks, can you also provide your Private Key: -----BEGIN PRIVATE KEY----- [redacted] -----END PRIVATE KEY-----? +Sam: Can you check on that? +Sofia: Thanks, can you also provide your Biometric Identifier: <<>>? +Sam: Can you check on that? +Sofia: Let me look that up for you. +Sam: Here is my Date of Birth: <<>> +Sofia: Let me look that up for you. +Sam: Here is my IP Address: <<>> +Sofia: Thanks, can you also provide your Date of Birth: <<>>? +Sam: Can you check on that? +Sofia: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0088.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0088.txt.redacted.redaction.json new file mode 100644 index 00000000..44e40253 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0088.txt.redacted.redaction.json @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0088.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0088.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.892, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\nSam: Can you check on that?\nSofia: Let me look t" + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.971, + "snippet": "k that up for you.\nSam: Here is my Date of Birth: 1990-05-12\nSofia: Let me look that up for you.\nSam: Here is " + }, + { + "type": "IP Address", + "value": "192.168.1.42", + "confidence": 0.997, + "snippet": "look that up for you.\nSam: Here is my IP Address: 192.168.1.42\nSofia: Thanks, can you also provide your Date of " + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.977, + "snippet": " Thanks, can you also provide your Date of Birth: 1990-05-12?\nSam: Can you check on that?\nSofia: Let me look t" + }, + { + "type": "Credit Card Number", + "value": "1990-05-12", + "confidence": 0.59, + "snippet": " Thanks, can you also provide your Date of Birth: 1990-05-12?\nSam: Can you check on that?\nSofia: Let me look t" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0089.txt.redacted b/examples/chats/support_chats/redacted/chat_0089.txt.redacted new file mode 100644 index 00000000..e01318a0 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0089.txt.redacted @@ -0,0 +1,16 @@ +Support Chat Transcript #89 +================================== +<<>>: Can you assist me with resetting my password. +<<>>: Let me look that up for you. +<<>>: Here is my Declared Interests: <<>> +<<>>: Thanks, can you also provide your Race/Ethnicity: <<>>? +<<>>: Here is my Union Membership: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your MAC Address: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0089.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0089.txt.redacted.redaction.json new file mode 100644 index 00000000..5f22cab7 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0089.txt.redacted.redaction.json @@ -0,0 +1,153 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0089.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0089.txt.redacted", + "totalRedactions": 6, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.783, + "snippet": "Transcript #89\n==================================\nTaylor: Can you assist me with resetting my password.\nLi" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.741, + "snippet": "or: Can you assist me with resetting my password.\nLiam: Let me look that up for you.\nTaylor: Here is my " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.819, + "snippet": "g my password.\nLiam: Let me look that up for you.\nTaylor: Here is my Declared Interests: hiking, cooking\nL" + }, + { + "type": "Declared Interests", + "value": "hiking, cooking", + "confidence": 0.894, + "snippet": "p for you.\nTaylor: Here is my Declared Interests: hiking, cooking\nLiam: Thanks, can you also provide your Race/Ethn" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.714, + "snippet": "r: Here is my Declared Interests: hiking, cooking\nLiam: Thanks, can you also provide your Race/Ethnicity" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.984, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\nTaylor: Here is my Union Membership: None\nLiam: " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.768, + "snippet": "n you also provide your Race/Ethnicity: Hispanic?\nTaylor: Here is my Union Membership: None\nLiam: Let me l" + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.967, + "snippet": "y: Hispanic?\nTaylor: Here is my Union Membership: None\nLiam: Let me look that up for you.\nTaylor: Can yo" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.71, + "snippet": "spanic?\nTaylor: Here is my Union Membership: None\nLiam: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.687, + "snippet": "mbership: None\nLiam: Let me look that up for you.\nTaylor: Can you check on that?\nLiam: Let me look that up" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.712, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nLiam: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.698, + "snippet": "check on that?\nLiam: Let me look that up for you.\nTaylor: Can you check on that?\nLiam: Let me look that up" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.769, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nLiam: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.693, + "snippet": "check on that?\nLiam: Let me look that up for you.\nTaylor: Can you check on that?\nLiam: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.728, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nLiam: Thanks, can you also provide your MAC Address: 0" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": "m: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\nTaylor: Can you check on that?\nLiam: Let me look" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.755, + "snippet": "also provide your MAC Address: 00:1B:44:11:3A:B7?\nTaylor: Can you check on that?\nLiam: Let me look that up" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.676, + "snippet": "00:1B:44:11:3A:B7?\nTaylor: Can you check on that?\nLiam: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0090.txt.redacted b/examples/chats/support_chats/redacted/chat_0090.txt.redacted new file mode 100644 index 00000000..1f0d0eee --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0090.txt.redacted @@ -0,0 +1,11 @@ +Support Chat Transcript #90 +================================== +<<>>: Hi, I need help with firmware update errors. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Union Membership: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Email: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0090.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0090.txt.redacted.redaction.json new file mode 100644 index 00000000..f2743924 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0090.txt.redacted.redaction.json @@ -0,0 +1,111 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0090.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0090.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.807, + "snippet": "Transcript #90\n==================================\nSam: Hi, I need help with firmware update errors.\nAva" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.828, + "snippet": "Sam: Hi, I need help with firmware update errors.\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.773, + "snippet": " update errors.\nAva: Let me look that up for you.\nSam: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.818, + "snippet": "look that up for you.\nSam: Can you check on that?\nAva: Thanks, can you also provide your Union Membersh" + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.983, + "snippet": "anks, can you also provide your Union Membership: None?\nSam: Can you check on that?\nAva: Thanks, can you" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.781, + "snippet": "can you also provide your Union Membership: None?\nSam: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.809, + "snippet": "ion Membership: None?\nSam: Can you check on that?\nAva: Thanks, can you also provide your Email: jordan@" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.998, + "snippet": "at?\nAva: Thanks, can you also provide your Email: jordan@example.com?\nSam: Can you check on that?\nAva: Let me look tha" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.791, + "snippet": " you also provide your Email: jordan@example.com?\nSam: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.842, + "snippet": ": jordan@example.com?\nSam: Can you check on that?\nAva: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.796, + "snippet": " check on that?\nAva: Let me look that up for you.\nSam: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0091.txt.redacted b/examples/chats/support_chats/redacted/chat_0091.txt.redacted new file mode 100644 index 00000000..284e0e1a --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0091.txt.redacted @@ -0,0 +1,14 @@ +Support Chat Transcript #91 +================================== +<<>>: Good morning, question about logging into my account. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your <<>>: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Approximate Geolocation: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0091.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0091.txt.redacted.redaction.json new file mode 100644 index 00000000..2f8a1e13 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0091.txt.redacted.redaction.json @@ -0,0 +1,141 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0091.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0091.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.792, + "snippet": "Transcript #91\n==================================\nJordan: Good morning, question about logging into my acc" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.773, + "snippet": " morning, question about logging into my account.\nMaya: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.717, + "snippet": "to my account.\nMaya: Let me look that up for you.\nJordan: Can you check on that?\nMaya: Thanks, can you als" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.796, + "snippet": "k that up for you.\nJordan: Can you check on that?\nMaya: Thanks, can you also provide your Union Membersh" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.902, + "snippet": " on that?\nMaya: Thanks, can you also provide your Union Membership: None?\nJordan: Can you check on that?\nMaya: Let m" + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.967, + "snippet": "anks, can you also provide your Union Membership: None?\nJordan: Can you check on that?\nMaya: Let me look" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.725, + "snippet": "can you also provide your Union Membership: None?\nJordan: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.745, + "snippet": " Membership: None?\nJordan: Can you check on that?\nMaya: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.69, + "snippet": "check on that?\nMaya: Let me look that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.72, + "snippet": "k that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up for you.\nJordan: Here is my " + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.829, + "snippet": "check on that?\nMaya: Let me look that up for you.\nJordan: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "Region", + "value": "San Francisco, CA", + "confidence": 0.599, + "snippet": " you.\nJordan: Here is my Approximate Geolocation: San Francisco, CA\nMaya: Let me look that up for you.\nJordan: Can yo" + }, + { + "type": "IP Address", + "value": "San Francisco, CA", + "confidence": 0.581, + "snippet": " you.\nJordan: Here is my Approximate Geolocation: San Francisco, CA\nMaya: Let me look that up for you.\nJordan: Can yo" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.705, + "snippet": " is my Approximate Geolocation: San Francisco, CA\nMaya: Let me look that up for you.\nJordan: Can you che" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.716, + "snippet": " Francisco, CA\nMaya: Let me look that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.766, + "snippet": "k that up for you.\nJordan: Can you check on that?\nMaya: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0092.txt.redacted b/examples/chats/support_chats/redacted/chat_0092.txt.redacted new file mode 100644 index 00000000..0834e7c9 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0092.txt.redacted @@ -0,0 +1,11 @@ +Support Chat Transcript #92 +================================== +<<>>: Hi, I need help with firmware update errors. +<<>>: Thanks, can you also provide your <<>>: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Sexual Orientation: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your IP Address: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0092.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0092.txt.redacted.redaction.json new file mode 100644 index 00000000..9c618946 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0092.txt.redacted.redaction.json @@ -0,0 +1,129 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0092.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0092.txt.redacted", + "totalRedactions": 6, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.869, + "snippet": "Transcript #92\n==================================\nRiley: Hi, I need help with firmware update errors.\nEth" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.886, + "snippet": "ley: Hi, I need help with firmware update errors.\nEthan: Thanks, can you also provide your Religion: None" + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.679, + "snippet": " errors.\nEthan: Thanks, can you also provide your Religion: None?\nRiley: Can you check on that?\nEthan: Thank" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.596, + "snippet": " errors.\nEthan: Thanks, can you also provide your Religion: None?\nRiley: Can you check on that?\nEthan: Thank" + }, + { + "type": "Religion", + "value": "None", + "confidence": 0.863, + "snippet": "than: Thanks, can you also provide your Religion: None?\nRiley: Can you check on that?\nEthan: Thanks, can" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.877, + "snippet": "Thanks, can you also provide your Religion: None?\nRiley: Can you check on that?\nEthan: Thanks, can you al" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.898, + "snippet": "our Religion: None?\nRiley: Can you check on that?\nEthan: Thanks, can you also provide your Sexual Orienta" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.975, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\nRiley: Can you check on that?\nEthan: Thanks, can" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.859, + "snippet": "so provide your Sexual Orientation: Heterosexual?\nRiley: Can you check on that?\nEthan: Thanks, can you al" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.896, + "snippet": "tion: Heterosexual?\nRiley: Can you check on that?\nEthan: Thanks, can you also provide your IP Address: 19" + }, + { + "type": "IP Address", + "value": "192.168.1.42", + "confidence": 0.998, + "snippet": "an: Thanks, can you also provide your IP Address: 192.168.1.42?\nRiley: Can you check on that?\nEthan: Let me look" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.897, + "snippet": "n you also provide your IP Address: 192.168.1.42?\nRiley: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.909, + "snippet": "ress: 192.168.1.42?\nRiley: Can you check on that?\nEthan: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.877, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nRiley: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0093.txt.redacted b/examples/chats/support_chats/redacted/chat_0093.txt.redacted new file mode 100644 index 00000000..7355d13f --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0093.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #93 +================================== +<<>>: Hi, I need help with a late delivery. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Username: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Political Affiliation: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0093.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0093.txt.redacted.redaction.json new file mode 100644 index 00000000..de1590d8 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0093.txt.redacted.redaction.json @@ -0,0 +1,87 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0093.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0093.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Username", + "value": "Taylor", + "confidence": 0.63, + "snippet": "late delivery.\nMaya: Let me look that up for you.\nTaylor: Can you check on that?\nMaya: Thanks, can you als" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.972, + "snippet": "Maya: Thanks, can you also provide your Username: jordan_92?\nTaylor: Can you check on that?\nMaya: Let me look" + }, + { + "type": "Username", + "value": "Taylor", + "confidence": 0.572, + "snippet": "check on that?\nMaya: Let me look that up for you.\nTaylor: Here is my Political Affiliation: Independent\nMa" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.978, + "snippet": "or you.\nTaylor: Here is my Political Affiliation: Independent\nMaya: Let me look that up for you.\nTaylor: Can yo" + }, + { + "type": "Username", + "value": "Maya", + "confidence": 0.513, + "snippet": "or: Here is my Political Affiliation: Independent\nMaya: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "Username", + "value": "Maya", + "confidence": 0.51, + "snippet": "k that up for you.\nTaylor: Can you check on that?\nMaya: Let me look that up for you.\nTaylor: Can you che" + }, + { + "type": "Username", + "value": "Taylor", + "confidence": 0.504, + "snippet": "check on that?\nMaya: Let me look that up for you.\nTaylor: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0094.txt.redacted b/examples/chats/support_chats/redacted/chat_0094.txt.redacted new file mode 100644 index 00000000..2cb56e0a --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0094.txt.redacted @@ -0,0 +1,20 @@ +Support Chat Transcript #94 +================================== +<<>>: Hi, I need help with firmware update errors. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Here is my Biometric Identifier: <<>> +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Username: <<>>? +<<>>: Here is my Union Membership: <<>> +<<>>: Thanks, can you also provide your MAC Address: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your User Preferences: <<>>? +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your User Agent: <<>> (Macintosh; Intel Mac OS X)? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0094.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0094.txt.redacted.redaction.json new file mode 100644 index 00000000..8f078f88 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0094.txt.redacted.redaction.json @@ -0,0 +1,177 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0094.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0094.txt.redacted", + "totalRedactions": 8, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Username", + "value": "Alex", + "confidence": 0.534, + "snippet": "Transcript #94\n==================================\nAlex: Hi, I need help with firmware update errors.\nEth" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.589, + "snippet": "lex: Hi, I need help with firmware update errors.\nEthan: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "Username", + "value": "Alex", + "confidence": 0.518, + "snippet": "pdate errors.\nEthan: Let me look that up for you.\nAlex: Can you check on that?\nEthan: Let me look that u" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.858, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nEthan: Let me look that up for you.\nAlex: Here is my Bi" + }, + { + "type": "Username", + "value": "Alex", + "confidence": 0.538, + "snippet": "heck on that?\nEthan: Let me look that up for you.\nAlex: Here is my Biometric Identifier: FaceID register" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.945, + "snippet": "p for you.\nAlex: Here is my Biometric Identifier: FaceID registered\nEthan: Let me look that up for you.\nAlex: Can you" + }, + { + "type": "User Preferences", + "value": "FaceID registered", + "confidence": 0.759, + "snippet": "p for you.\nAlex: Here is my Biometric Identifier: FaceID registered\nEthan: Let me look that up for you.\nAlex: Can you" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.872, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nEthan: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.871, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nEthan: Thanks, can you also provide your Username: jord" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.945, + "snippet": "than: Thanks, can you also provide your Username: jordan_92?\nAlex: Here is my Union Membership: None\nEthan: T" + }, + { + "type": "Username", + "value": "Alex", + "confidence": 0.529, + "snippet": "s, can you also provide your Username: jordan_92?\nAlex: Here is my Union Membership: None\nEthan: Thanks," + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.93, + "snippet": "me: jordan_92?\nAlex: Here is my Union Membership: None\nEthan: Thanks, can you also provide your MAC Addr" + }, + { + "type": "User Agent", + "value": "None", + "confidence": 0.756, + "snippet": "me: jordan_92?\nAlex: Here is my Union Membership: None\nEthan: Thanks, can you also provide your MAC Addr" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.503, + "snippet": "ordan_92?\nAlex: Here is my Union Membership: None\nEthan: Thanks, can you also provide your MAC Address: 0" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.999, + "snippet": "n: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\nAlex: Can you check on that?\nEthan: Let me look " + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.815, + "snippet": ": 00:1B:44:11:3A:B7?\nAlex: Can you check on that?\nEthan: Let me look that up for you.\nAlex: Can you check" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.808, + "snippet": "ook that up for you.\nAlex: Can you check on that?\nEthan: Thanks, can you also provide your User Preferenc" + }, + { + "type": "User Preferences", + "value": "marketing_emails=false", + "confidence": 0.902, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\nAlex: Can you check on that?\nEthan: Thanks, can " + }, + { + "type": "Biometric Identifier", + "value": "marketing_emails=false", + "confidence": 0.565, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\nAlex: Can you check on that?\nEthan: Thanks, can " + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.86, + "snippet": "keting_emails=false?\nAlex: Can you check on that?\nEthan: Thanks, can you also provide your User Agent: Mo" + }, + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.825, + "snippet": "an: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?" + }, + { + "type": "Union Membership", + "value": "Mozilla/5.0", + "confidence": 0.536, + "snippet": "an: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0095.txt.redacted b/examples/chats/support_chats/redacted/chat_0095.txt.redacted new file mode 100644 index 00000000..243fa687 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0095.txt.redacted @@ -0,0 +1,19 @@ +Support Chat Transcript #95 +================================== +Sam: Can you assist me with logging into my account. +Liam: Thanks, can you also provide your Date of Birth: <<>>? +Sam: Here is my Race/Ethnicity: <<>> +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Thanks, can you also provide your Union Membership: <<>>? +Sam: Can you check on that? +Liam: Thanks, can you also provide your Phone: <<>>? +Sam: Here is my User Agent: <<>> (Macintosh; Intel Mac OS X) +Liam: Let me look that up for you. +Sam: Can you check on that? +Liam: Let me look that up for you. +Sam: Here is my Survey Data: Q1=Yes, Q2=No \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0095.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0095.txt.redacted.redaction.json new file mode 100644 index 00000000..c1d349e6 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0095.txt.redacted.redaction.json @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0095.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0095.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.995, + "snippet": " Thanks, can you also provide your Date of Birth: 1990-05-12?\nSam: Here is my Race/Ethnicity: Hispanic\nLiam: L" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.994, + "snippet": "irth: 1990-05-12?\nSam: Here is my Race/Ethnicity: Hispanic\nLiam: Let me look that up for you.\nSam: Can you c" + }, + { + "type": "Union Membership", + "value": "None", + "confidence": 0.956, + "snippet": "anks, can you also provide your Union Membership: None?\nSam: Can you check on that?\nLiam: Thanks, can yo" + }, + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.999, + "snippet": "t?\nLiam: Thanks, can you also provide your Phone: +1-202-555-0172?\nSam: Here is my User Agent: Mozilla/5.0 (Macinto" + }, + { + "type": "User Agent", + "value": "Mozilla/5.0", + "confidence": 0.893, + "snippet": "one: +1-202-555-0172?\nSam: Here is my User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)\nLiam: Let me look tha" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0096.txt.redacted b/examples/chats/support_chats/redacted/chat_0096.txt.redacted new file mode 100644 index 00000000..e2a54ee3 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0096.txt.redacted @@ -0,0 +1,14 @@ +Support Chat Transcript #96 +================================== +<<>>: Good morning, question about logging into my account. +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Thanks, can you also provide your Sexual Orientation: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0096.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0096.txt.redacted.redaction.json new file mode 100644 index 00000000..efd8e028 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0096.txt.redacted.redaction.json @@ -0,0 +1,123 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0096.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0096.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.976, + "snippet": "Transcript #96\n==================================\nRiley: Good morning, question about logging into my acc" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.957, + "snippet": " morning, question about logging into my account.\nAva: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.973, + "snippet": "nto my account.\nAva: Let me look that up for you.\nRiley: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.958, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nAva: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.973, + "snippet": " check on that?\nAva: Let me look that up for you.\nRiley: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.952, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nAva: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.973, + "snippet": " check on that?\nAva: Let me look that up for you.\nRiley: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.954, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nAva: Let me look that up for you.\nRiley: Can you chec" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.96, + "snippet": " check on that?\nAva: Let me look that up for you.\nRiley: Can you check on that?\nAva: Thanks, can you also" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.96, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nAva: Thanks, can you also provide your Sexual Orienta" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.994, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\nRiley: Can you check on that?\nAva: Let me look t" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.969, + "snippet": "so provide your Sexual Orientation: Heterosexual?\nRiley: Can you check on that?\nAva: Let me look that up " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.96, + "snippet": "tion: Heterosexual?\nRiley: Can you check on that?\nAva: Let me look that up for you." + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0097.txt.redacted b/examples/chats/support_chats/redacted/chat_0097.txt.redacted new file mode 100644 index 00000000..8eec4abf --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0097.txt.redacted @@ -0,0 +1,13 @@ +Support Chat Transcript #97 +================================== +<<>>: Hi, I need help with firmware update errors. +<<>>: Thanks, can you also provide your Serial Number: <<>>? +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? +<<>>: Let me look that up for you. +<<>>: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0097.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0097.txt.redacted.redaction.json new file mode 100644 index 00000000..59329a69 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0097.txt.redacted.redaction.json @@ -0,0 +1,117 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0097.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0097.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.75, + "snippet": "Transcript #97\n==================================\nSam: Hi, I need help with firmware update errors.\nMay" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.753, + "snippet": "Sam: Hi, I need help with firmware update errors.\nMaya: Thanks, can you also provide your Serial Number:" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.998, + "snippet": " Thanks, can you also provide your Serial Number: SN-1234567890?\nSam: Can you check on that?\nMaya: Let me look th" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.681, + "snippet": "u also provide your Serial Number: SN-1234567890?\nSam: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.657, + "snippet": "umber: SN-1234567890?\nSam: Can you check on that?\nMaya: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.687, + "snippet": "check on that?\nMaya: Let me look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.585, + "snippet": "look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.725, + "snippet": "check on that?\nMaya: Let me look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.655, + "snippet": "look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.633, + "snippet": "check on that?\nMaya: Let me look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.744, + "snippet": "look that up for you.\nSam: Can you check on that?\nMaya: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.697, + "snippet": "check on that?\nMaya: Let me look that up for you.\nSam: Can you check on that?" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0098.txt.redacted b/examples/chats/support_chats/redacted/chat_0098.txt.redacted new file mode 100644 index 00000000..2531081a --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0098.txt.redacted @@ -0,0 +1,19 @@ +Support Chat Transcript #98 +================================== +Taylor: Hi, I need help with resetting my password. +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Let me look that up for you. +Taylor: Here is my Sexual Orientation: <<>> +Sofia: Let me look that up for you. +Taylor: Can you check on that? +Sofia: Thanks, can you also provide your Political Affiliation: <<>>? +Taylor: Here is my Profile URL: <<>> +Sofia: Let me look that up for you. +Taylor: Here is my Race/Ethnicity: <<>> \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0098.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0098.txt.redacted.redaction.json new file mode 100644 index 00000000..9b4bfe75 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0098.txt.redacted.redaction.json @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0098.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0098.txt.redacted", + "totalRedactions": 4, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.988, + "snippet": "p for you.\nTaylor: Here is my Sexual Orientation: Heterosexual\nSofia: Let me look that up for you.\nTaylor: Can y" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.994, + "snippet": " can you also provide your Political Affiliation: Independent?\nTaylor: Here is my Profile URL: https://social.e" + }, + { + "type": "Profile URL", + "value": "https://social.example.com/jordan", + "confidence": 0.983, + "snippet": "ion: Independent?\nTaylor: Here is my Profile URL: https://social.example.com/jordan\nSofia: Let me look that up for you.\nTaylor: Here " + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.963, + "snippet": "at up for you.\nTaylor: Here is my Race/Ethnicity: Hispanic" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0099.txt.redacted b/examples/chats/support_chats/redacted/chat_0099.txt.redacted new file mode 100644 index 00000000..3c47a619 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0099.txt.redacted @@ -0,0 +1,15 @@ +Support Chat Transcript #99 +================================== +Taylor: Good morning, question about a late delivery. +Maya: Thanks, can you also provide your User Preferences: <<>>? +Taylor: Can you check on that? +Maya: Let me look that up for you. +Taylor: Can you check on that? +Maya: Thanks, can you also provide your Email: <<>>? +Taylor: Can you check on that? +Maya: Let me look that up for you. +Taylor: Here is my <<>>: <<>> +Maya: Let me look that up for you. +Taylor: Here is my <<>>: <<>> +Maya: Let me look that up for you. +Taylor: Here is my Gender: <<>> \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0099.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0099.txt.redacted.redaction.json new file mode 100644 index 00000000..65a835f5 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0099.txt.redacted.redaction.json @@ -0,0 +1,81 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0099.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0099.txt.redacted", + "totalRedactions": 5, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User Preferences", + "value": "marketing_emails=false", + "confidence": 0.915, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\nTaylor: Can you check on that?\nMaya: Let me look" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.999, + "snippet": "t?\nMaya: Thanks, can you also provide your Email: jordan@example.com?\nTaylor: Can you check on that?\nMaya: Let me look" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.58, + "snippet": ": Let me look that up for you.\nTaylor: Here is my User ID: U123456\nMaya: Let me look that up for you.\nTaylo" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.99, + "snippet": "look that up for you.\nTaylor: Here is my User ID: U123456\nMaya: Let me look that up for you.\nTaylor: Here i" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.991, + "snippet": "look that up for you.\nTaylor: Here is my User ID: U123456\nMaya: Let me look that up for you.\nTaylor: Here i" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.945, + "snippet": " look that up for you.\nTaylor: Here is my Gender: Non-binary" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/chat_0100.txt.redacted b/examples/chats/support_chats/redacted/chat_0100.txt.redacted new file mode 100644 index 00000000..7167bf5a --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0100.txt.redacted @@ -0,0 +1,11 @@ +Support Chat Transcript #100 +================================== +Casey: Can you assist me with resetting my password. +Noah: Let me look that up for you. +Casey: Here is my User ID: <<>> +Noah: Thanks, can you also provide your <<>>: <<>>? +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Can you check on that? +Noah: Let me look that up for you. +Casey: Can you check on that? \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/chat_0100.txt.redacted.redaction.json b/examples/chats/support_chats/redacted/chat_0100.txt.redacted.redaction.json new file mode 100644 index 00000000..b3b90579 --- /dev/null +++ b/examples/chats/support_chats/redacted/chat_0100.txt.redacted.redaction.json @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0100.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0100.txt.redacted", + "totalRedactions": 3, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "User ID", + "value": "U123456", + "confidence": 0.987, + "snippet": " look that up for you.\nCasey: Here is my User ID: U123456\nNoah: Thanks, can you also provide your Declared " + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.608, + "snippet": ": U123456\nNoah: Thanks, can you also provide your Declared Interests: hiking, cooking?\nCasey: Can you check on that?\nN" + }, + { + "type": "User Preferences", + "value": "Declared Interests", + "confidence": 0.579, + "snippet": ": U123456\nNoah: Thanks, can you also provide your Declared Interests: hiking, cooking?\nCasey: Can you check on that?\nN" + }, + { + "type": "Declared Interests", + "value": "hiking, cooking", + "confidence": 0.885, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\nCasey: Can you check on that?\nNoah: Let me look " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0001.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0001.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..598047df --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0001.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0001.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0001.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.949, + "snippet": "a: Thanks, can you also provide your <<>>: <<>>?\nTaylor: Can you check on that?\nSofia: Let me loo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.78, + "snippet": "on that?\nSofia: Thanks, can you also provide your <<>>: None?\nTaylor: Can you check on that?\nSofia: Let " + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.979, + "snippet": "anks, can you also provide your <<>>: None?\nTaylor: Can you check on that?\nSofia: Let me loo" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0001.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0001.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..27edb1b1 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0001.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0001.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0001.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.841, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.799, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.938, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.917, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.932, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.924, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.541, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.844, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.784, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.543, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.77, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.941, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.881, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.853, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.63, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.913, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Union Membership", + "value": "Industry", + "confidence": 0.845, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.857, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.821, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.689, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.841, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.895, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.636, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.779, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.953, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.902, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.984, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.865, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.723, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.969, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.52, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.528, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.601, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.588, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.562, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.533, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.697, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.669, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.603, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.715, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.684, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.915, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.952, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.947, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.656, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.849, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.939, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.963, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.667, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.897, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.784, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.776, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.937, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.911, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.568, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.836, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "IP Address", + "value": "https://social.example.com/jordan", + "confidence": 0.534, + "snippet": "\n \"type\": \"Profile URL\",\n \"value\": \"https://social.example.com/jordan\",\n \"confidence\": 0.949,\n \"snippet\":" + }, + { + "type": "IP Address", + "value": "https://social.example.com/jordan", + "confidence": 0.69, + "snippet": "a: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\\nTaylor: Can you check on that?\\nSofia: Let me l" + }, + { + "type": "Account Number", + "value": "https://social.example.com/jordan", + "confidence": 0.556, + "snippet": "a: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\\nTaylor: Can you check on that?\\nSofia: Let me l" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.874, + "snippet": "ia: Let me loo\"\n },\n {\n \"type\": \"Union Membership\",\n \"value\": \"Union Membership\",\n \"c" + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.648, + "snippet": "ia: Let me loo\"\n },\n {\n \"type\": \"Union Membership\",\n \"value\": \"Union Membership\",\n \"c" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.875, + "snippet": " \"type\": \"Union Membership\",\n \"value\": \"Union Membership\",\n \"confidence\": 0.78,\n \"snippet\": " + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.543, + "snippet": " \"type\": \"Union Membership\",\n \"value\": \"Union Membership\",\n \"confidence\": 0.78,\n \"snippet\": " + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.89, + "snippet": "n that?\\nSofia: Thanks, can you also provide your Union Membership: None?\\nTaylor: Can you check on that?\\nSofia: Le" + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.503, + "snippet": "n that?\\nSofia: Thanks, can you also provide your Union Membership: None?\\nTaylor: Can you check on that?\\nSofia: Le" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0002.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0002.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..390ecb00 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0002.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0002.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0002.txt.redacted", + "totalRedactions": 2, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.997, + "snippet": "a: Thanks, can you also provide your <<>>: <<>>?\nCasey: Can you check on that?\nSofia: Let me look" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.849, + "snippet": "k that up for you.\nCasey: Here is my <<>>: <<>>" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.523, + "snippet": "k that up for you.\nCasey: Here is my <<>>: <<>>" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0002.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0002.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..664cd105 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0002.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,417 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0002.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0002.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.775, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.753, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.945, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.875, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.932, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.926, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.986, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.802, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.699, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.691, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.764, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.936, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.557, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.886, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.509, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.839, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.574, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.959, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.839, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.72, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.713, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.808, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.541, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.784, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.954, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.912, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.984, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.855, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.623, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.969, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.952, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.737, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.684, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.751, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture URL", + "confidence": 0.584, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.583, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.551, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.622, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Industry", + "value": "Profile URL", + "confidence": 0.596, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.751, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.726, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Profile Picture Image", + "value": "Race/Ethnicity", + "confidence": 0.512, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.642, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.667, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.83, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.919, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.941, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.612, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.83, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.944, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.852, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.942, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.748, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.695, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.929, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.912, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.79, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.672, + "snippet": "\n \"type\": \"MAC Address\",\n \"value\": \"00:1B:44:11:3A:B7\",\n \"confidence\": 0.997,\n \"snippet\":" + }, + { + "type": "Passport Number", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.646, + "snippet": "\n \"type\": \"MAC Address\",\n \"value\": \"00:1B:44:11:3A:B7\",\n \"confidence\": 0.997,\n \"snippet\":" + }, + { + "type": "Passport Number", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.71, + "snippet": "a: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\\nCasey: Can you check on that?\\nSofia: Let me lo" + }, + { + "type": "MAC Address", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.683, + "snippet": "a: Thanks, can you also provide your MAC Address: 00:1B:44:11:3A:B7?\\nCasey: Can you check on that?\\nSofia: Let me lo" + }, + { + "type": "Account Number", + "value": "https://social.example.com/jordan", + "confidence": 0.568, + "snippet": "\n \"type\": \"Profile URL\",\n \"value\": \"https://social.example.com/jordan\",\n \"confidence\": 0.849,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0003.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0003.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..77188760 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0003.txt.redacted.redaction.json.redacted @@ -0,0 +1,57 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0003.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0003.txt.redacted", + "totalRedactions": 1, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.564, + "snippet": " morning, question about logging into my account.\n<<>>: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.53, + "snippet": "look that up for you.\nSam: Can you check on that?\n<<>>: Let me look that up for you.\nSam: Can you check " + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0003.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0003.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..4668c5bd --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0003.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,429 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0003.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0003.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.836, + "snippet": "cted\",\n \"totalRedactions\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.79, + "snippet": "cted\",\n \"totalRedactions\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.935, + "snippet": "ons\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.905, + "snippet": "ons\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.948, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.943, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.819, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.81, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.708, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.823, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.949, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.896, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.869, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.594, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.591, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.956, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.876, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.847, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.663, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.859, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.901, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.629, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.812, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.631, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.96, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.922, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.982, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.873, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.749, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.971, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.57, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.955, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.517, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.52, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.618, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.6, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.561, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.661, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.682, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.651, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.601, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.662, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.662, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.932, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.953, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.939, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.656, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.854, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Social Security Number", + "confidence": 0.516, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.948, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.804, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.556, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.975, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.9, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.852, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.929, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.929, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.51, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.857, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.869, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": " + }, + { + "type": "User ID", + "value": "Liam", + "confidence": 0.745, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": 0.564,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.92, + "snippet": "Can you check \"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": " + }, + { + "type": "User ID", + "value": "Liam", + "confidence": 0.762, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": 0.53,\n \"snippet\": " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0004.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0004.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..87afa7d6 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0004.txt.redacted.redaction.json.redacted @@ -0,0 +1,99 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0004.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0004.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.985, + "snippet": " Transcript #4\n==================================\n<<>>: Good morning, question about updating my subscri" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.98, + "snippet": "morning, question about updating my subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.98, + "snippet": "y subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.979, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my IP " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.985, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: 192.168.1.42\n<<>>: Let me " + }, + { + "type": "<<>>", + "value": "192.168.1.42", + "confidence": 0.997, + "snippet": "look that up for you.\n<<>>: Here is my <<>>: 192.168.1.42\n<<>>: Let me look that up for you.\n<<>>: Can you ch" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.969, + "snippet": "for you.\n<<>>: Here is my <<>>: 192.168.1.42\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.983, + "snippet": "s: 192.168.1.42\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.984, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0004.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0004.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..9fd8fe5d --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0004.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0004.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0004.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.872, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.813, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.945, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.92, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.951, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.945, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.649, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.519, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.79, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.758, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.719, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.771, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.945, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.887, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.875, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.635, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.558, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.927, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.502, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.862, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.807, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.641, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.888, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.903, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.519, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.786, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.965, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.93, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.783, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.779, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.973, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.951, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.582, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.574, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.541, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.532, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.522, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.724, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.53, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.681, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.65, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.593, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.591, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.658, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.938, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.944, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.527, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.914, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.587, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.821, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.961, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.799, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.968, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.835, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.789, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.947, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.94, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.505, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.849, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.913, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Sam", + "confidence": 0.771, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.985,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.686, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.985,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.868, + "snippet": "ing my subscri\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.797, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.98,\n \"snippet\": " + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.658, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.98,\n \"snippet\": " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0005.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0005.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..61a817a2 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0005.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0005.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0005.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "Liam", + "confidence": 0.538, + "snippet": "===\nTaylor: Hi, I need help with a late delivery.\nLiam: Let me look that up for you.\nTaylor: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.981, + "snippet": "m: Thanks, can you also provide your <<>>: <<>>?\nTaylor: Can you check on that?\nLiam: Let me look" + }, + { + "type": "<<>>", + "value": "Hispanic", + "confidence": 0.995, + "snippet": "Thanks, can you also provide your <<>>: Hispanic?\nTaylor: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0005.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0005.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..259fbf09 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0005.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0005.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0005.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.851, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.814, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.939, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.915, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.952, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.927, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.633, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.502, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.833, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.776, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.684, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.769, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.931, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.897, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.878, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.562, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.534, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.944, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.863, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.817, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.692, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.858, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.903, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.701, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.818, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.563, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.952, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.907, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.983, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.915, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.713, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.966, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.951, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.674, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.652, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.722, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.569, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.568, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture URL", + "confidence": 0.553, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.525, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.625, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.577, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.707, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.694, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.504, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.682, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.655, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.919, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.95, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.947, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.576, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.855, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.944, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.87, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.535, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.979, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.891, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.831, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.933, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.928, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.859, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.557, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.943, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": " + }, + { + "type": "Account Number", + "value": "https://social.example.com/jordan", + "confidence": 0.588, + "snippet": "\n \"type\": \"Profile URL\",\n \"value\": \"https://social.example.com/jordan\",\n \"confidence\": 0.981,\n \"snippet\":" + }, + { + "type": "Account Number", + "value": "https://social.example.com/jordan", + "confidence": 0.602, + "snippet": "m: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\\nTaylor: Can you check on that?\\nLiam: Let me lo" + }, + { + "type": "Union Membership", + "value": "https://social.example.com/jordan", + "confidence": 0.509, + "snippet": "m: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\\nTaylor: Can you check on that?\\nLiam: Let me lo" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0006.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0006.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..5bd6caa4 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0006.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0006.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0006.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Profile Picture Image", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.981, + "snippet": "\nLiam: Thanks, can you also provide your <<>>: <<>>?\nRiley: Can you check on that?\nLiam: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.992, + "snippet": "ks, can you also provide your <<>>: <<>>?\nRiley: Can you check on that?\nLiam: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.997, + "snippet": "hat up for you.\nRiley: Here is my <<>>: <<>>\nLiam: Let me look that up for you.\nRiley: Can you" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0006.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0006.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..355f8100 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0006.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,477 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0006.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0006.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.825, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.787, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.929, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.888, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.96, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.956, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.71, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.786, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.643, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.836, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.958, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.879, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.742, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Biometric Identifier", + "value": "Gender", + "confidence": 0.512, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.867, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.727, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.858, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Industry", + "confidence": 0.506, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.88, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.858, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.913, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.927, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.595, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.819, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.599, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.965, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.945, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.501, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.98, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.805, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.773, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.98, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.504, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.953, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.736, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.692, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.522, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.703, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.754, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.699, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Industry", + "value": "Race/Ethnicity", + "confidence": 0.534, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.619, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.666, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.949, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.942, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.599, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.864, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.832, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.841, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.855, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.88, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.672, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.938, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.871, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.878, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.664, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.882, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence\": 0.981,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "U123456", + "confidence": 0.517, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence\": 0.981,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.512, + "snippet": "ppet\": \"\\nLiam: Thanks, can you also provide your User ID: U123456?\\nRiley: Can you check on that?\\nLiam: T" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.892, + "snippet": "nLiam: Thanks, can you also provide your User ID: U123456?\\nRiley: Can you check on that?\\nLiam: Thanks, ca" + }, + { + "type": "Race/Ethnicity", + "value": "U123456", + "confidence": 0.655, + "snippet": "nLiam: Thanks, can you also provide your User ID: U123456?\\nRiley: Can you check on that?\\nLiam: Thanks, ca" + }, + { + "type": "User Agent", + "value": "U123456", + "confidence": 0.531, + "snippet": "nLiam: Thanks, can you also provide your User ID: U123456?\\nRiley: Can you check on that?\\nLiam: Thanks, ca" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.603, + "snippet": " \"type\": \"Sexual Orientation\",\n \"value\": \"Heterosexual\",\n \"confidence\": 0.992,\n \"snippet\":" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.79, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\\nRiley: Can you check on that?\\nLiam: Let me loo" + }, + { + "type": "Religion", + "value": "Heterosexual", + "confidence": 0.571, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\\nRiley: Can you check on that?\\nLiam: Let me loo" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.642, + "snippet": ": Let me look \"\n },\n {\n \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidenc" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.608, + "snippet": ": Let me look \"\n },\n {\n \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidenc" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.512, + "snippet": ": Let me look \"\n },\n {\n \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidenc" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.687, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.997,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.636, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.997,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0007.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0007.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..463a6083 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0007.txt.redacted.redaction.json.redacted @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0007.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0007.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.525, + "snippet": "d1234?\nRiley: Here is my Approximate Geolocation: <<>>\nSofia: Let me look that up for you.\nRiley: Here i" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.997, + "snippet": "hat up for you.\nRiley: Here is my <<>>: <<>>\nSofia: Let me look that up for you.\nRiley: Can yo" + }, + { + "type": "<<>>", + "value": "SN-1234567890", + "confidence": 0.997, + "snippet": " Thanks, can you also provide your <<>>: SN-1234567890?\nRiley: Here is my <<>>: marketing_em" + }, + { + "type": "<<>>", + "value": "false", + "confidence": 0.681, + "snippet": "ey: Here is my <<>>: marketing_emails=false" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0007.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0007.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..12bb2ff5 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0007.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,495 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0007.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0007.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.865, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.615, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.922, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.91, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.949, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.941, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.584, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.799, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.786, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.54, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.715, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.952, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.879, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.541, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.865, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.672, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.563, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.819, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Industry", + "confidence": 0.537, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.851, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.791, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.583, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.868, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.892, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.674, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.517, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.781, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.959, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.93, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.983, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.874, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.82, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.961, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.559, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.581, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Religion", + "value": "Profile Picture Image", + "confidence": 0.57, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.621, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.605, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.687, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.527, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.697, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.873, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.706, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.966, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.843, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.624, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.919, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.572, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.841, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.955, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Serial Number", + "value": "Tax Information", + "confidence": 0.573, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.852, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.863, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.677, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.553, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.922, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.909, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.537, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.772, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.898, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Region\",\n \"value\": \"San Francisco, CA\",\n \"" + }, + { + "type": "MAC Address", + "value": "San Francisco, CA", + "confidence": 0.68, + "snippet": " {\n \"type\": \"Region\",\n \"value\": \"San Francisco, CA\",\n \"confidence\": 0.525,\n \"snippet\":" + }, + { + "type": "IP Address", + "value": "San Francisco, CA", + "confidence": 0.646, + "snippet": " {\n \"type\": \"Region\",\n \"value\": \"San Francisco, CA\",\n \"confidence\": 0.525,\n \"snippet\":" + }, + { + "type": "Passport Number", + "value": "San Francisco, CA", + "confidence": 0.53, + "snippet": " {\n \"type\": \"Region\",\n \"value\": \"San Francisco, CA\",\n \"confidence\": 0.525,\n \"snippet\":" + }, + { + "type": "MAC Address", + "value": "San Francisco, CA", + "confidence": 0.647, + "snippet": "1234?\\nRiley: Here is my Approximate Geolocation: San Francisco, CA\\nSofia: Let me look that up for you.\\nRiley: Here" + }, + { + "type": "IP Address", + "value": "San Francisco, CA", + "confidence": 0.628, + "snippet": "1234?\\nRiley: Here is my Approximate Geolocation: San Francisco, CA\\nSofia: Let me look that up for you.\\nRiley: Here" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.881, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.997,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Hispanic", + "confidence": 0.655, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.997,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.571, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.997,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.894, + "snippet": "at up for you.\\nRiley: Here is my Race/Ethnicity: Hispanic\\nSofia: Let me look that up for you.\\nRiley: Can " + }, + { + "type": "User Agent", + "value": "Hispanic", + "confidence": 0.605, + "snippet": "at up for you.\\nRiley: Here is my Race/Ethnicity: Hispanic\\nSofia: Let me look that up for you.\\nRiley: Can " + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.592, + "snippet": "at up for you.\\nRiley: Here is my Race/Ethnicity: Hispanic\\nSofia: Let me look that up for you.\\nRiley: Can " + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.574, + "snippet": "nRiley: Can yo\"\n },\n {\n \"type\": \"Serial Number\",\n \"value\": \"SN-1234567890\",\n \"conf" + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.507, + "snippet": "nRiley: Can yo\"\n },\n {\n \"type\": \"Serial Number\",\n \"value\": \"SN-1234567890\",\n \"conf" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0008.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0008.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..02e31361 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0008.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0008.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0008.txt.redacted", + "totalRedactions": 2, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.98, + "snippet": "n: Thanks, can you also provide your <<>>: <<>>?\nCasey: Can you check on that?\nEthan: Let me look" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.524, + "snippet": "n: Thanks, can you also provide your <<>>: <<>>?\nCasey: Can you check on that?\nEthan: Let me look" + }, + { + "type": "<<>>", + "value": "+1-202-555-0172", + "confidence": 0.999, + "snippet": "?\nEthan: Thanks, can you also provide your <<>>: +1-202-555-0172?\nCasey: Can you check on that?\nEthan: Let me look" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0008.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0008.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..1a2d8cc2 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0008.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,435 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0008.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0008.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.826, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.768, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.933, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.911, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.934, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.915, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.587, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.511, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.833, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.741, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.662, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.737, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.952, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.507, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.889, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.649, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.869, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.569, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.931, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.852, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.784, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.674, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.85, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.912, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.71, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.782, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.508, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.951, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.882, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.985, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.903, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.772, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.965, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.531, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.953, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.773, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.73, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.818, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.623, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.54, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture URL", + "confidence": 0.534, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.556, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Industry", + "value": "Profile URL", + "confidence": 0.538, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.816, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.77, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Profile Picture Image", + "value": "Race/Ethnicity", + "confidence": 0.511, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.638, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.665, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.918, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.943, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.946, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.547, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.87, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.952, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.881, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.509, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.923, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.729, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.568, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.937, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.924, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.783, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Union Membership", + "value": "https://social.example.com/jordan", + "confidence": 0.52, + "snippet": "n: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\\nCasey: Can you check on that?\\nEthan: Let me lo" + }, + { + "type": "Account Number", + "value": "https://social.example.com/jordan", + "confidence": 0.546, + "snippet": " \"type\": \"Profile Picture URL\",\n \"value\": \"https://social.example.com/jordan\",\n \"confidence\": 0.524,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0009.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0009.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..974f2b9d --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0009.txt.redacted.redaction.json.redacted @@ -0,0 +1,147 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0009.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0009.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.688, + "snippet": " Transcript #9\n==================================\n<<>>: Good morning, question about firmware update err" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.736, + "snippet": "d morning, question about firmware update errors.\n<<>>: Thanks, can you also provide your Private Key: -" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.71, + "snippet": "TE KEY----- [redacted] -----END PRIVATE KEY-----?\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.682, + "snippet": " PRIVATE KEY-----?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.605, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.66, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.659, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you al" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.616, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>:" + }, + { + "type": "<<>>", + "value": "SN-1234567890", + "confidence": 0.998, + "snippet": " Thanks, can you also provide your <<>>: SN-1234567890?\n<<>>: Can you check on that?\n<<>>: Let me loo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.613, + "snippet": "u also provide your <<>>: SN-1234567890?\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.504, + "snippet": "er: SN-1234567890?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.717, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.585, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.773, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>:" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": " that up for you.\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: Let me look that up for you.\n<<>>: Can y" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.631, + "snippet": "<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.7, + "snippet": "B:44:11:3A:B7\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0009.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0009.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..ffae9ba3 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0009.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,459 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0009.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0009.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.876, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.837, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.953, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.851, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.96, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.923, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.564, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.828, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.781, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.729, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.809, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.946, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.893, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.876, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.672, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.946, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.507, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.869, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.817, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.773, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.895, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.895, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.581, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.794, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.965, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.933, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.86, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.784, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.969, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.678, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.633, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.525, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.55, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.527, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.516, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.727, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.677, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.646, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.533, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.615, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.698, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.566, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.501, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.936, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.947, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.607, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.92, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.688, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.812, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.965, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Serial Number", + "value": "Tax Information", + "confidence": 0.508, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.822, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.59, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.966, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.802, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.754, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.956, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.933, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.516, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.846, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.883, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.694, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.688,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Taylor", + "confidence": 0.644, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.688,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.817, + "snippet": "are update err\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Sofia\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Sofia", + "confidence": 0.734, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sofia\",\n \"confidence\": 0.736,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.728, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sofia\",\n \"confidence\": 0.736,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0010.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0010.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..4d3068c3 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0010.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0010.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0010.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.651, + "snippet": " Can you assist me with updating my subscription.\n<<>>: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.838, + "snippet": "on that?\n<<>>: Thanks, can you also provide your <<>>: <<>>?" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.898, + "snippet": "ks, can you also provide your <<>>: <<>>?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0010.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0010.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..67abd635 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0010.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,453 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0010.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0010.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.804, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.753, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.956, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.872, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.955, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.94, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.879, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.817, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Sexual Orientation", + "value": "Driver's License", + "confidence": 0.633, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.955, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.918, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.875, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.711, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.563, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.974, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.91, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Income", + "value": "Interaction Events", + "confidence": 0.543, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.867, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.912, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.824, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.96, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.951, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.637, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.982, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.83, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.77, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.973, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.624, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.961, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.642, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.582, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.52, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.724, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.713, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.586, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.708, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.544, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.828, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.773, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.605, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.695, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.759, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.928, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.957, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.805, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.785, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.634, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.871, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.952, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.833, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.567, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.96, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.843, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.796, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.94, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.789, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.77, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.828, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.907, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Ethan", + "confidence": 0.755, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\": 0.651,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.743, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\": 0.651,\n \"snippet\":" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.675, + "snippet": " \"type\": \"Declared Interests\",\n \"value\": \"Declared Interests\",\n \"confidence\": 0.838,\n \"snippet\":" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.532, + "snippet": "n that?\\nEthan: Thanks, can you also provide your Declared Interests: hiking, cooking?\"\n },\n {\n \"type" + }, + { + "type": "Industry", + "value": "hiking, cooking", + "confidence": 0.937, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\"\n },\n {\n \"type\": \"Declared Inte" + }, + { + "type": "User Agent", + "value": "hiking, cooking", + "confidence": 0.509, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\"\n },\n {\n \"type\": \"Declared Inte" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.748, + "snippet": "king, cooking?\"\n },\n {\n \"type\": \"Declared Interests\",\n \"value\": \"hiking, cooking\",\n \"co" + }, + { + "type": "Industry", + "value": "hiking, cooking", + "confidence": 0.903, + "snippet": " \"type\": \"Declared Interests\",\n \"value\": \"hiking, cooking\",\n \"confidence\": 0.898,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0011.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0011.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..4465e311 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0011.txt.redacted.redaction.json.redacted @@ -0,0 +1,93 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0011.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0011.txt.redacted", + "totalRedactions": 7, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.959, + "snippet": "e errors.\nMaya: Thanks, can you also provide your <<>>: None?\nSam: Can you check on that?\nMaya: Let me l" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.947, + "snippet": "Maya: Thanks, can you also provide your <<>>: None?\nSam: Can you check on that?\nMaya: Let me look th" + }, + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.965, + "snippet": "ya: Thanks, can you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nSam: Here is my Appr" + }, + { + "type": "<<>>", + "value": "San Francisco, CA", + "confidence": 0.564, + "snippet": "c OS X)?\nSam: Here is my Approximate Geolocation: San Francisco, CA\nMaya: Let me look that up for you.\nSam: Can you c" + }, + { + "type": "<<>>", + "value": "1990-05-12", + "confidence": 0.991, + "snippet": "k that up for you.\nSam: Here is my <<>>: 1990-05-12\nMaya: Thanks, can you also provide your <<>>: " + }, + { + "type": "<<>>", + "value": "U123456", + "confidence": 0.992, + "snippet": "\nMaya: Thanks, can you also provide your <<>>: U123456?\nSam: Can you check on that?\nMaya: Let me look th" + }, + { + "type": "<<>>", + "value": "jordan@example.com", + "confidence": 0.999, + "snippet": "t me look that up for you.\nSam: Here is my <<>>: jordan@example.com\nMaya: Let me look that up for you.\nSam: Can you c" + }, + { + "type": "<<>>", + "value": "jordan@example.com", + "confidence": 0.997, + "snippet": "t?\nMaya: Thanks, can you also provide your <<>>: jordan@example.com?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0011.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0011.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..b35d1ec7 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0011.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,519 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0011.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0011.txt.redacted.redaction.json.redacted", + "totalRedactions": 34, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.808, + "snippet": "cted\",\n \"totalRedactions\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.752, + "snippet": "cted\",\n \"totalRedactions\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.942, + "snippet": "ons\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.929, + "snippet": "ons\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.943, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.943, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.545, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.992, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.849, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.773, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.748, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.958, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.505, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.9, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.865, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.704, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.649, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.83, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Religion", + "value": "Industry", + "confidence": 0.669, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.592, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.88, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.859, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.615, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.875, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.906, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.642, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.824, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.958, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.923, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.506, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.982, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.891, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.65, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.975, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.656, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.956, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.512, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.594, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.548, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Religion", + "value": "Profile Picture Image", + "confidence": 0.525, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.622, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.607, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.546, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.694, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.548, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.748, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.663, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.635, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.697, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.914, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.759, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Religion", + "confidence": 0.637, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.904, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.95, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.917, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.586, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.861, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.941, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.816, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.958, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.757, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.749, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.936, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.922, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.564, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.826, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.73, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Religion\",\n \"value\": \"Religion\",\n \"confidenc" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.687, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Religion\",\n \"value\": \"Religion\",\n \"confidenc" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.845, + "snippet": " {\n \"type\": \"Religion\",\n \"value\": \"Religion\",\n \"confidence\": 0.959,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.722, + "snippet": " {\n \"type\": \"Religion\",\n \"value\": \"Religion\",\n \"confidence\": 0.959,\n \"snippet\":" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.804, + "snippet": " errors.\\nMaya: Thanks, can you also provide your Religion: None?\\nSam: Can you check on that?\\nMaya: Let me" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.65, + "snippet": " errors.\\nMaya: Thanks, can you also provide your Religion: None?\\nSam: Can you check on that?\\nMaya: Let me" + }, + { + "type": "Race/Ethnicity", + "value": "Religion", + "confidence": 0.6, + "snippet": " errors.\\nMaya: Thanks, can you also provide your Religion: None?\\nSam: Can you check on that?\\nMaya: Let me" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.657, + "snippet": "Maya: Let me l\"\n },\n {\n \"type\": \"Religion\",\n \"value\": \"None\",\n \"confidence\": " + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.603, + "snippet": "Maya: Let me l\"\n },\n {\n \"type\": \"Religion\",\n \"value\": \"None\",\n \"confidence\": " + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.822, + "snippet": "nippet\": \"Maya: Thanks, can you also provide your Religion: None?\\nSam: Can you check on that?\\nMaya: Let me" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.692, + "snippet": "nippet\": \"Maya: Thanks, can you also provide your Religion: None?\\nSam: Can you check on that?\\nMaya: Let me" + }, + { + "type": "Race/Ethnicity", + "value": "Religion", + "confidence": 0.613, + "snippet": "nippet\": \"Maya: Thanks, can you also provide your Religion: None?\\nSam: Can you check on that?\\nMaya: Let me" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.948, + "snippet": "Let me look th\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Mozilla/5.0\",\n \"confid" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0012.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0012.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..cc782873 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0012.txt.redacted.redaction.json.redacted @@ -0,0 +1,117 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0012.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0012.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.803, + "snippet": "Transcript #12\n==================================\n<<>>: Good morning, question about a double charge on " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.758, + "snippet": "rning, question about a double charge on my card.\n<<>>: Thanks, can you also provide your <<>>: jordan@" + }, + { + "type": "<<>>", + "value": "jordan@example.com", + "confidence": 0.995, + "snippet": "rd.\n<<>>: Thanks, can you also provide your <<>>: jordan@example.com?\n<<>>: Can you check on that?\n<<>>: Thanks, can you" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.707, + "snippet": " you also provide your <<>>: jordan@example.com?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.727, + "snippet": ": jordan@example.com?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Political Affi" + }, + { + "type": "<<>>", + "value": "Independent", + "confidence": 0.984, + "snippet": " can you also provide your <<>>: Independent?\n<<>>: Can you check on that?\n<<>>: Let me look tha" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.705, + "snippet": " provide your <<>>: Independent?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.658, + "snippet": "liation: Independent?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.728, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.631, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.808, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.721, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0012.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0012.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..7f8bd063 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0012.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0012.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0012.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.875, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.783, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.946, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.908, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.961, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.929, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.596, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.779, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.766, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.704, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.773, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.94, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.885, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.863, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.585, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.555, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.934, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.854, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.808, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.666, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.903, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.907, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.539, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.514, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.79, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.958, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.932, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.978, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.81, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.791, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.979, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.505, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.618, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.587, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.573, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.551, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.536, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.732, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.73, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.688, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.608, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.576, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.681, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.935, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.948, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Page views", + "value": "Serial Number", + "confidence": 0.553, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.905, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.633, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.826, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.817, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.562, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.966, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.82, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.791, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.948, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.933, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.838, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.502, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.893, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Sam", + "confidence": 0.703, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.803,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.542, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.803,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.87, + "snippet": "ble charge on \"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.703, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.758,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.591, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.758,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0013.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0013.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..873387d9 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0013.txt.redacted.redaction.json.redacted @@ -0,0 +1,99 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0013.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0013.txt.redacted", + "totalRedactions": 7, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.984, + "snippet": "h: Thanks, can you also provide your <<>>: <<>>?\nSam: Can you check on that?\nNoah: Thanks, can yo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.78, + "snippet": " on that?\nNoah: Thanks, can you also provide your <<>>: None?\nSam: Can you check on that?\nNoah: Let me l" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.964, + "snippet": "anks, can you also provide your <<>>: None?\nSam: Can you check on that?\nNoah: Let me look th" + }, + { + "type": "<<>>", + "value": "SN-1234567890", + "confidence": 0.995, + "snippet": "k that up for you.\nSam: Here is my <<>>: SN-1234567890\nNoah: Thanks, can you also provide your Union Mem" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.795, + "snippet": "234567890\nNoah: Thanks, can you also provide your <<>>: None?\nSam: Can you check on that?\nNoah: Let me l" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.955, + "snippet": "anks, can you also provide your <<>>: None?\nSam: Can you check on that?\nNoah: Let me look th" + }, + { + "type": "<<>>", + "value": "123-45-6789", + "confidence": 0.998, + "snippet": " for you.\nSam: Here is my <<>>: 123-45-6789\nNoah: Thanks, can you also provide your IP Addres" + }, + { + "type": "<<>>", + "value": "192.168.1.42", + "confidence": 0.998, + "snippet": "ah: Thanks, can you also provide your <<>>: 192.168.1.42?\nSam: Can you check on that?\nNoah: Let me look th" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.972, + "snippet": "ook that up for you.\nSam: Here is my <<>>: 00:1B:44:11:3A:B7" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0013.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0013.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..6c867611 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0013.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,435 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0013.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0013.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.844, + "snippet": "cted\",\n \"totalRedactions\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.81, + "snippet": "cted\",\n \"totalRedactions\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.936, + "snippet": "ons\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.916, + "snippet": "ons\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.932, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.921, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.509, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.851, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.777, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.555, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.76, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.946, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.882, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.851, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.619, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.913, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Union Membership", + "value": "Industry", + "confidence": 0.836, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.853, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.815, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.686, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.839, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.893, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.637, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.777, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.952, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.9, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.985, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.856, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.749, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.968, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.565, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.953, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.526, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.6, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.579, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.556, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.544, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.692, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.662, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.561, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.721, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.679, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.915, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.95, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.948, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.631, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.85, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.938, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.963, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.626, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.895, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.782, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.773, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.938, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.912, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.553, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.835, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "IP Address", + "value": "https://social.example.com/jordan", + "confidence": 0.524, + "snippet": "\n \"type\": \"Profile URL\",\n \"value\": \"https://social.example.com/jordan\",\n \"confidence\": 0.984,\n \"snippet\":" + }, + { + "type": "IP Address", + "value": "https://social.example.com/jordan", + "confidence": 0.681, + "snippet": "h: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\\nSam: Can you check on that?\\nNoah: Thanks, can " + }, + { + "type": "Account Number", + "value": "https://social.example.com/jordan", + "confidence": 0.562, + "snippet": "h: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\\nSam: Can you check on that?\\nNoah: Thanks, can " + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.874, + "snippet": "Thanks, can yo\"\n },\n {\n \"type\": \"Union Membership\",\n \"value\": \"Union Membership\",\n \"c" + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.603, + "snippet": "Thanks, can yo\"\n },\n {\n \"type\": \"Union Membership\",\n \"value\": \"Union Membership\",\n \"c" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.88, + "snippet": " \"type\": \"Union Membership\",\n \"value\": \"Union Membership\",\n \"confidence\": 0.78,\n \"snippet\": " + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.9, + "snippet": "on that?\\nNoah: Thanks, can you also provide your Union Membership: None?\\nSam: Can you check on that?\\nNoah: Let me" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0014.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0014.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..a77491d0 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0014.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0014.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0014.txt.redacted", + "totalRedactions": 2, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.969, + "snippet": "Thanks, can you also provide your <<>>: <<>>?\nCasey: Can you check on that?\nAva: Let me look t" + }, + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.944, + "snippet": "va: Thanks, can you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nCasey: Here is my Pr" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.979, + "snippet": "hat up for you.\nCasey: Here is my <<>>: <<>>\nAva: Let me look that up for you.\nCasey: Can you " + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0014.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0014.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..6042c8dd --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0014.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,459 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0014.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0014.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.897, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.873, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.929, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.91, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.954, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.882, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.57, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.989, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.837, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.793, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.519, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.759, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.95, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.944, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.898, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.788, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.57, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.794, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Industry", + "confidence": 0.727, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.871, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.854, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.799, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.852, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.93, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.719, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.823, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.534, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.958, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.949, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.519, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.978, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.872, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.672, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.976, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.94, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Religion", + "value": "Profile Picture Image", + "confidence": 0.664, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.559, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.639, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.604, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.547, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.505, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.681, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.653, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.602, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.788, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Religion", + "confidence": 0.678, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.535, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.938, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.948, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.911, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.611, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.523, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.834, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.957, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.884, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.935, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.766, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.738, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.934, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.931, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.871, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.503, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidenc" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.828, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.969,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.649, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.969,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.768, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\\nCasey: Can you check on that?\\nAva: Let me look" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.621, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\\nCasey: Can you check on that?\\nAva: Let me look" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.877, + "snippet": " Let me look t\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Mozilla/5.0\",\n \"confid" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.935, + "snippet": "\"snippet\": \"va: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\\nCasey:" + }, + { + "type": "User ID", + "value": "User Agent", + "confidence": 0.633, + "snippet": "\"snippet\": \"va: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\\nCasey:" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0015.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0015.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..b206db74 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0015.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0015.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0015.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "+1-202-555-0172", + "confidence": 0.999, + "snippet": "?\nSofia: Thanks, can you also provide your <<>>: +1-202-555-0172?\nTaylor: Can you check on that?\nSofia: Thanks, ca" + }, + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.976, + "snippet": "ia: Thanks, can you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?" + }, + { + "type": "<<>>", + "value": "Intel Mac OS X", + "confidence": 0.559, + "snippet": " provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0015.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0015.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..4b2a0921 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0015.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,435 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0015.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0015.txt.redacted.redaction.json.redacted", + "totalRedactions": 34, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.832, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.795, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.948, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.854, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.943, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.942, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.992, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.854, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.682, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.635, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.731, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.917, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.628, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.899, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.856, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.723, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.921, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.806, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.774, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.74, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.86, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.792, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.627, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.8, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.521, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.948, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.945, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.547, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.975, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.89, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.632, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.876, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.519, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.948, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.706, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.772, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.597, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.574, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.691, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Industry", + "value": "Profile URL", + "confidence": 0.587, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.576, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.704, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.579, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.593, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.693, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.552, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.541, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.917, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.941, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.942, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.692, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.818, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.954, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.875, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.964, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.881, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.774, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.949, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.918, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.836, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.76, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Phone\",\n \"value\": \"+1-202-555-0172\",\n \"co" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.815, + "snippet": "ia: Thanks, ca\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Mozilla/5.0\",\n \"confid" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.939, + "snippet": "\"snippet\": \"ia: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\"\n " + }, + { + "type": "User ID", + "value": "User Agent", + "confidence": 0.555, + "snippet": "\"snippet\": \"ia: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\"\n " + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.534, + "snippet": "tel Mac OS X)?\"\n },\n {\n \"type\": \"MAC Address\",\n \"value\": \"Intel Mac OS X\",\n \"con" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0016.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0016.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..7a14bd07 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0016.txt.redacted.redaction.json.redacted @@ -0,0 +1,57 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0016.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0016.txt.redacted", + "totalRedactions": 2, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.951, + "snippet": "an: Thanks, can you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nRiley: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.999, + "snippet": "can you also provide your <<>>: <<>>?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0016.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0016.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..c889e047 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0016.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,465 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0016.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0016.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.85, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.816, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.93, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.889, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.944, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.931, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.532, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.914, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.744, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.734, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.941, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.898, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.866, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.673, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.604, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.942, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.852, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.836, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.783, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.835, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.889, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.629, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.767, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.665, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.947, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.94, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.549, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.981, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.905, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.641, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.964, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.942, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.761, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.644, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.744, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.573, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.563, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.51, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.5, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.633, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.542, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.736, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.65, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.624, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.723, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.528, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.931, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.947, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.968, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.723, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.529, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.526, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.952, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.887, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.978, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.921, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.791, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.921, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.908, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.829, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.886, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Mozilla/5.0\",\n \"confid" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.943, + "snippet": "\"snippet\": \"an: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\\nRiley:" + }, + { + "type": "User ID", + "value": "User Agent", + "confidence": 0.543, + "snippet": "\"snippet\": \"an: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\\nRiley:" + }, + { + "type": "Political Affiliation", + "value": "123-45-6789", + "confidence": 0.748, + "snippet": "ype\": \"Social Security Number\",\n \"value\": \"123-45-6789\",\n \"confidence\": 0.999,\n \"snippet\":" + }, + { + "type": "Social Security Number", + "value": "123-45-6789", + "confidence": 0.648, + "snippet": "ype\": \"Social Security Number\",\n \"value\": \"123-45-6789\",\n \"confidence\": 0.999,\n \"snippet\":" + }, + { + "type": "Passport Number", + "value": "123-45-6789", + "confidence": 0.535, + "snippet": "ype\": \"Social Security Number\",\n \"value\": \"123-45-6789\",\n \"confidence\": 0.999,\n \"snippet\":" + }, + { + "type": "Political Affiliation", + "value": "123-45-6789", + "confidence": 0.804, + "snippet": "can you also provide your Social Security Number: 123-45-6789?\"\n }\n ]\n ]\n}" + }, + { + "type": "Social Security Number", + "value": "123-45-6789", + "confidence": 0.77, + "snippet": "can you also provide your Social Security Number: 123-45-6789?\"\n }\n ]\n ]\n}" + }, + { + "type": "Passport Number", + "value": "123-45-6789", + "confidence": 0.536, + "snippet": "can you also provide your Social Security Number: 123-45-6789?\"\n }\n ]\n ]\n}" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0017.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0017.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..78cd123d --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0017.txt.redacted.redaction.json.redacted @@ -0,0 +1,135 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0017.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0017.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.861, + "snippet": "Transcript #17\n==================================\n<<>>: Hello—I'm having trouble with updating my subscr" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.878, + "snippet": "I'm having trouble with updating my subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.857, + "snippet": " subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.906, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.887, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Private Key: -----BEGIN PRIVATE KEY--" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.902, + "snippet": "ATE KEY----- [redacted] -----END PRIVATE KEY-----\n<<>>: Thanks, can you also provide your Political Affi" + }, + { + "type": "<<>>", + "value": "Independent", + "confidence": 0.965, + "snippet": " can you also provide your <<>>: Independent?\n<<>>: Can you check on that?\n<<>>: Let me look" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.83, + "snippet": " provide your <<>>: Independent?\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.853, + "snippet": "tion: Independent?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.784, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.809, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.826, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.807, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.824, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.865, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0017.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0017.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..82c12424 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0017.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,441 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0017.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0017.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.859, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.809, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.947, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.914, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.945, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.937, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.663, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.803, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.779, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.741, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.783, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.945, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.901, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.877, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.653, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.535, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.935, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.543, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.868, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.827, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.655, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.879, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.899, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.55, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.792, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.567, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.961, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.924, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.98, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.826, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.774, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.971, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.584, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.582, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.563, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.538, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.526, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.715, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.563, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.712, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.667, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.665, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.621, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.666, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.94, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.946, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.536, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.924, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.615, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.813, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.799, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.964, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.866, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.821, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.949, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.943, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.533, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.861, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.533, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.882, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.703, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.861,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.86, + "snippet": "ting my subscr\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.66, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.878,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0018.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0018.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..14826086 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0018.txt.redacted.redaction.json.redacted @@ -0,0 +1,129 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0018.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0018.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.955, + "snippet": "Transcript #18\n==================================\n<<>>: Hi, I need help with resetting my password.\n<<>>:" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.949, + "snippet": "<<>>: Hi, I need help with resetting my password.\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.951, + "snippet": "ng my password.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.953, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Se" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.957, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: Heterosexual\n<<>>:" + }, + { + "type": "<<>>", + "value": "Heterosexual", + "confidence": 0.991, + "snippet": " up for you.\n<<>>: Here is my <<>>: Heterosexual\n<<>>: Let me look that up for you.\n<<>>: Can you c" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.909, + "snippet": "<<>>: Here is my <<>>: Heterosexual\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.947, + "snippet": "n: Heterosexual\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.934, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.941, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.938, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.949, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.958, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.959, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0018.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0018.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..8adde73d --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0018.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,435 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0018.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0018.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.873, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.799, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.943, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.916, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.951, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.939, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.585, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.811, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.789, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.701, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.777, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.947, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.897, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.875, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.596, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.558, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.944, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.858, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.81, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.686, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.883, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.901, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.506, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.804, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.507, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.96, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.918, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.975, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.817, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.81, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.518, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.599, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.584, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.578, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.552, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.712, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.692, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.673, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.581, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.613, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.686, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.939, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.95, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.536, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.924, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.672, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.82, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.805, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.964, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.86, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.801, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.945, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.937, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.864, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.889, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": " + }, + { + "type": "User ID", + "value": "Alex", + "confidence": 0.711, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": 0.955,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.636, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": 0.955,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.861, + "snippet": "assword.\\nAva:\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.601, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.949,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Alex", + "confidence": 0.7, + "snippet": "\n \"confidence\": 0.949,\n \"snippet\": \"Alex: Hi, I need help with resetting my password.\\nAva" + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.625, + "snippet": "\n \"confidence\": 0.949,\n \"snippet\": \"Alex: Hi, I need help with resetting my password.\\nAva" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0019.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0019.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..0eedd1d0 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0019.txt.redacted.redaction.json.redacted @@ -0,0 +1,141 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0019.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0019.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.671, + "snippet": "Transcript #19\n==================================\n<<>>: Can you assist me with firmware update errors.\nN" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.59, + "snippet": "y: Can you assist me with firmware update errors.\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.657, + "snippet": "update errors.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.608, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.547, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.672, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.584, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.642, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Sexual Orienta" + }, + { + "type": "<<>>", + "value": "Heterosexual", + "confidence": 0.995, + "snippet": "ks, can you also provide your <<>>: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.555, + "snippet": "so provide your <<>>: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.595, + "snippet": "tion: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: jordan@" + }, + { + "type": "<<>>", + "value": "jordan@example.com", + "confidence": 0.996, + "snippet": "t?\n<<>>: Thanks, can you also provide your <<>>: jordan@example.com?\n<<>>: Can you check on that?\n<<>>: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.576, + "snippet": " you also provide your <<>>: jordan@example.com?\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.511, + "snippet": "jordan@example.com?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.671, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.574, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0019.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0019.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..c2f6ffaa --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0019.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0019.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0019.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.868, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.825, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.953, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.851, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.96, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.931, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.992, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.509, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.837, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.768, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.731, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.79, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.953, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.897, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.877, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.607, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.507, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.943, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.54, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.863, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.834, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.762, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.887, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.894, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.583, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.796, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.51, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.964, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.926, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.981, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.86, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.812, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.968, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.579, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.955, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.675, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.577, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.512, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.566, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.554, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.713, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.706, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.693, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.575, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.636, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.703, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.557, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.937, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.944, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.547, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.922, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.67, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.81, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.809, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.519, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.973, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.843, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.772, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.952, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.93, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.528, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.856, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.82, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Casey\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Casey", + "confidence": 0.627, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Casey\",\n \"confidence\": 0.671,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.716, + "snippet": "ate errors.\\nN\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Noah\",\n \"confidence\": " + }, + { + "type": "User ID", + "value": "Noah", + "confidence": 0.699, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Noah\",\n \"confidence\": 0.59,\n \"snippet\": " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0020.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0020.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..587ce249 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0020.txt.redacted.redaction.json.redacted @@ -0,0 +1,105 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0020.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0020.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Profile Picture Image", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.674, + "snippet": "Transcript #20\n==================================\n<<>>: Can you assist me with updating my subscription." + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.716, + "snippet": " Can you assist me with updating my subscription.\n<<>>: Thanks, can you also provide your <<>>" + }, + { + "type": "<<>>", + "value": "Heterosexual", + "confidence": 0.988, + "snippet": "ks, can you also provide your <<>>: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.742, + "snippet": "so provide your <<>>: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.665, + "snippet": "tion: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.705, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.651, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: Non-bi" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.975, + "snippet": "?\n<<>>: Thanks, can you also provide your <<>>: Non-binary?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.687, + "snippet": "ks, can you also provide your <<>>: Non-binary?\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.661, + "snippet": "<<>>: Non-binary?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Approximate Ge" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0020.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0020.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..a826b24f --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0020.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,453 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0020.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0020.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.848, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.813, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.941, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.899, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.953, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.93, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.674, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.992, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.542, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.797, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.749, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.712, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.953, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.893, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.884, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.635, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.58, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.906, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.537, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.865, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.815, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.888, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.904, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.555, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.793, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.563, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.962, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.925, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.86, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.812, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.961, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.578, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.625, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.614, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.533, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.718, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.801, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.696, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.681, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.613, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.582, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.553, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.943, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.944, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.647, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.956, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.51, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.823, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.966, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Serial Number", + "value": "Tax Information", + "confidence": 0.609, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.816, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.965, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.876, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.826, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.946, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.944, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.55, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.855, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.515, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.882, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Casey\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Casey", + "confidence": 0.686, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Casey\",\n \"confidence\": 0.674,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.564, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Casey\",\n \"confidence\": 0.674,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.807, + "snippet": " subscription.\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": " + }, + { + "type": "User ID", + "value": "Maya", + "confidence": 0.841, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.716,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.712, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.716,\n \"snippet\":" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orienta", + "confidence": 0.689, + "snippet": "ription.\\nMaya: Thanks, can you also provide your Sexual Orienta\"\n },\n {\n \"type\": \"Sexual Orienta" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0021.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0021.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..36970099 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0021.txt.redacted.redaction.json.redacted @@ -0,0 +1,99 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0021.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0021.txt.redacted", + "totalRedactions": 6, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Region", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.711, + "snippet": "Transcript #21\n==================================\n<<>>: Can you assist me with firmware update errors.\nS" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.559, + "snippet": "m: Can you assist me with firmware update errors.\n<<>>: Thanks, can you also provide your <<>>" + }, + { + "type": "<<>>", + "value": "Independent", + "confidence": 0.974, + "snippet": " can you also provide your <<>>: Independent?\n<<>>: Here is my <<>>: SN-1234567890\nSof" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.734, + "snippet": " provide your <<>>: Independent?\n<<>>: Here is my <<>>: SN-1234567890\n<<>>: L" + }, + { + "type": "<<>>", + "value": "SN-1234567890", + "confidence": 0.996, + "snippet": "tion: Independent?\n<<>>: Here is my <<>>: SN-1234567890\n<<>>: Let me look that up for you.\n<<>>: Here is " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.571, + "snippet": "SN-1234567890\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: jordan_92\n<<>>: Let me loo" + }, + { + "type": "<<>>", + "value": "jordan_92", + "confidence": 0.976, + "snippet": "e look that up for you.\n<<>>: Here is my <<>>: jordan_92\n<<>>: Let me look that up for you.\n<<>>: Can you " + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.98, + "snippet": "hat up for you.\n<<>>: Here is my <<>>: None\n<<>>: Let me look that up for you.\n<<>>: Can you " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.549, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0021.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0021.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..fc64008c --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0021.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,423 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0021.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0021.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.896, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.734, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.933, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.908, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Declared Interests", + "value": "Biometric Identifier", + "confidence": 0.542, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.93, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.843, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.552, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.905, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.85, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.688, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.765, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.954, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.889, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.892, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.639, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.607, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.936, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.917, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.881, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.8, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.824, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.919, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.828, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.854, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "User ID", + "value": "Name", + "confidence": 0.502, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.968, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.943, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.986, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.885, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.977, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.516, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.803, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.514, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.518, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.505, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.75, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.504, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.74, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Username", + "value": "Race/Ethnicity", + "confidence": 0.593, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.729, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.539, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.941, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.956, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.642, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.915, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.61, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.851, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.97, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.861, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.618, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "Name", + "value": "User ID", + "confidence": 0.61, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.591, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.909, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.906, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.588, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.852, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.824, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Username\",\n \"value\": \"Sam\",\n \"confidence\": 0" + }, + { + "type": "Username", + "value": "Sam", + "confidence": 0.905, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Sam\",\n \"confidence\": 0.711,\n \"snippet\":" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.786, + "snippet": "ate errors.\\nS\"\n },\n {\n \"type\": \"Username\",\n \"value\": \"Sofia\",\n \"confidence\":" + }, + { + "type": "Username", + "value": "Sofia", + "confidence": 0.893, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Sofia\",\n \"confidence\": 0.559,\n \"snippet\":" + }, + { + "type": "Political Affiliation", + "value": "Political Affi", + "confidence": 0.603, + "snippet": "errors.\\nSofia: Thanks, can you also provide your Political Affi\"\n },\n {\n \"type\": \"Political Affi" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0022.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0022.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..576021f8 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0022.txt.redacted.redaction.json.redacted @@ -0,0 +1,87 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0022.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0022.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.762, + "snippet": "m having trouble with a double charge on my card.\n<<>>: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.892, + "snippet": "look that up for you.\nSam: Can you check on that?\n<<>>: Let me look that up for you.\nSam: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.872, + "snippet": "look that up for you.\nSam: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: 0" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.999, + "snippet": "n: Thanks, can you also provide your <<>>: 00:1B:44:11:3A:B7?\nSam: Can you check on that?\n<<>>: Thanks, can y" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.891, + "snippet": "s: 00:1B:44:11:3A:B7?\nSam: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: jord" + }, + { + "type": "<<>>", + "value": "jordan_92", + "confidence": 0.976, + "snippet": "than: Thanks, can you also provide your <<>>: jordan_92?\nSam: Can you check on that?" + }, + { + "type": "<<>>", + "value": "Sam", + "confidence": 0.589, + "snippet": "s, can you also provide your <<>>: jordan_92?\nSam: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0022.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0022.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..b0e61761 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0022.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,465 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0022.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0022.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.912, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.765, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.905, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.892, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.949, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.937, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.594, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.799, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.765, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.686, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.817, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Religion", + "value": "Driver's License", + "confidence": 0.509, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.97, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.898, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.851, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.572, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.557, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.956, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.871, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.847, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.668, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.896, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.927, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.625, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.504, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.823, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.968, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.924, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.982, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.799, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.794, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.979, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.521, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.952, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.553, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.566, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.549, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.545, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.52, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.741, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.542, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.693, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.562, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.669, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.945, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.958, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.516, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.886, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.573, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.839, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.965, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.825, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.669, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.593, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Name", + "value": "User ID", + "confidence": 0.54, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.926, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.88, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.887, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.917, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\":" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.796, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\": 0.762,\n \"snippet\":" + }, + { + "type": "User Preferences", + "value": "Ethan", + "confidence": 0.612, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\": 0.762,\n \"snippet\":" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.894, + "snippet": "Can you check \"\n },\n {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\":" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.771, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\": 0.892,\n \"snippet\":" + }, + { + "type": "User Preferences", + "value": "Ethan", + "confidence": 0.666, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\": 0.892,\n \"snippet\":" + }, + { + "type": "Name", + "value": "Ethan", + "confidence": 0.57, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\": 0.892,\n \"snippet\":" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.9, + "snippet": "Can you check \"\n },\n {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\":" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.808, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\": 0.872,\n \"snippet\":" + }, + { + "type": "User Preferences", + "value": "Ethan", + "confidence": 0.671, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\": 0.872,\n \"snippet\":" + }, + { + "type": "Name", + "value": "Ethan", + "confidence": 0.568, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\": 0.872,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0023.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0023.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..0d17fd16 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0023.txt.redacted.redaction.json.redacted @@ -0,0 +1,123 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0023.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0023.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.875, + "snippet": "Transcript #23\n==================================\n<<>>: Hello—I'm having trouble with firmware update er" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.922, + "snippet": "o—I'm having trouble with firmware update errors.\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.881, + "snippet": "pdate errors.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you al" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.917, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: 19" + }, + { + "type": "<<>>", + "value": "192.168.1.42", + "confidence": 0.998, + "snippet": "an: Thanks, can you also provide your <<>>: 192.168.1.42?\n<<>>: Here is my <<>>: Hispanic\nEtha" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.909, + "snippet": "n you also provide your <<>>: 192.168.1.42?\n<<>>: Here is my <<>>: Hispanic\n<<>>: Let m" + }, + { + "type": "<<>>", + "value": "Hispanic", + "confidence": 0.997, + "snippet": " 192.168.1.42?\n<<>>: Here is my <<>>: Hispanic\n<<>>: Let me look that up for you.\n<<>>: Can y" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.833, + "snippet": "1.42?\n<<>>: Here is my <<>>: Hispanic\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.898, + "snippet": "ity: Hispanic\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.899, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.889, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.925, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.891, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0023.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0023.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..2600f3eb --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0023.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,459 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0023.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0023.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.872, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.82, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.955, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.856, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.959, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.922, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.993, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.521, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.834, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.778, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.746, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.814, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.949, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.516, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.898, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.876, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.658, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.944, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.551, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.868, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.837, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.768, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.89, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.896, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.609, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.796, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.963, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.929, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.847, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.836, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.966, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.58, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.636, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.649, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.559, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.546, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.525, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.515, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.719, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.691, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.616, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.552, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.651, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.709, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.553, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.517, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.935, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.949, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.923, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.691, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.81, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.961, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.816, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.522, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.971, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.84, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.772, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.956, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.936, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.502, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.856, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.853, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.77, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.875,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Taylor", + "confidence": 0.711, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.875,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.761, + "snippet": "ware update er\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Ethan", + "confidence": 0.734, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\": 0.922,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.732, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\": 0.922,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0024.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0024.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..d30b3149 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0024.txt.redacted.redaction.json.redacted @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0024.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0024.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.974, + "snippet": " me look that up for you.\nSam: Here is my <<>>: <<>>\nSofia: Thanks, can you also provide your Politica" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.55, + "snippet": " me look that up for you.\nSam: Here is my <<>>: <<>>\nSofia: Thanks, can you also provide your Politica" + }, + { + "type": "<<>>", + "value": "Independent", + "confidence": 0.992, + "snippet": " can you also provide your <<>>: Independent?\nSam: Can you check on that?\nSofia: Let me look t" + }, + { + "type": "<<>>", + "value": "U123456", + "confidence": 0.976, + "snippet": "me look that up for you.\nSam: Here is my <<>>: U123456\nSofia: Thanks, can you also provide your User Age" + }, + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.953, + "snippet": "ia: Thanks, can you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nSam: Can you check o" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0024.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0024.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..d1e262c9 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0024.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,495 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0024.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0024.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.793, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.691, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.942, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.837, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Gender", + "value": "Biometric Identifier", + "confidence": 0.737, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.945, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.894, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.514, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.993, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.532, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.517, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.754, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.623, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.594, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.746, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.966, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.847, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.888, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.755, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.555, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.757, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Industry", + "confidence": 0.634, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.918, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.851, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.813, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.945, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.853, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.883, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.974, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.924, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.989, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.94, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.982, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.696, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.625, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Date of Birth", + "value": "Phone", + "confidence": 0.546, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.938, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.548, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.504, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.703, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.645, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.523, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.722, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.607, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.584, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.859, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.821, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.506, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.607, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.954, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.962, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.959, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.879, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.959, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.86, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.923, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.69, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.538, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.933, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Gender", + "value": "User Preferences", + "confidence": 0.903, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.723, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.82, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.769, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confide" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.634, + "snippet": " {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.974,\n \"snippet\":" + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.75, + "snippet": "pet\": \" me look that up for you.\\nSam: Here is my Gender: Non-binary\\nSofia: Thanks, can you also provide " + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.835, + "snippet": "me look that up for you.\\nSam: Here is my Gender: Non-binary\\nSofia: Thanks, can you also provide your Politic" + }, + { + "type": "User ID", + "value": "Non-binary", + "confidence": 0.66, + "snippet": "me look that up for you.\\nSam: Here is my Gender: Non-binary\\nSofia: Thanks, can you also provide your Politic" + }, + { + "type": "Race/Ethnicity", + "value": "Non-binary", + "confidence": 0.56, + "snippet": "me look that up for you.\\nSam: Here is my Gender: Non-binary\\nSofia: Thanks, can you also provide your Politic" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.928, + "snippet": " your Politica\"\n },\n {\n \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confide" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.776, + "snippet": " \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.55,\n \"snippet\": " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.635, + "snippet": "pet\": \" me look that up for you.\\nSam: Here is my Gender: Non-binary\\nSofia: Thanks, can you also provide " + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.725, + "snippet": "me look that up for you.\\nSam: Here is my Gender: Non-binary\\nSofia: Thanks, can you also provide your Politic" + }, + { + "type": "User ID", + "value": "Non-binary", + "confidence": 0.715, + "snippet": "me look that up for you.\\nSam: Here is my Gender: Non-binary\\nSofia: Thanks, can you also provide your Politic" + }, + { + "type": "Race/Ethnicity", + "value": "Non-binary", + "confidence": 0.583, + "snippet": "me look that up for you.\\nSam: Here is my Gender: Non-binary\\nSofia: Thanks, can you also provide your Politic" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.941, + "snippet": " your Politica\"\n },\n {\n \"type\": \"Political Affiliation\",\n \"value\": \"Independent\",\n \"confid" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0025.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0025.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..ba4cdb1a --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0025.txt.redacted.redaction.json.redacted @@ -0,0 +1,147 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0025.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0025.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.943, + "snippet": "Transcript #25\n==================================\n<<>>: Can you assist me with logging into my account.\n" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.936, + "snippet": ": Can you assist me with logging into my account.\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.945, + "snippet": "nto my account.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.944, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.96, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.938, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.96, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: jordan@example.com\n<<>>: Let me" + }, + { + "type": "<<>>", + "value": "jordan@example.com", + "confidence": 0.998, + "snippet": "e look that up for you.\n<<>>: Here is my <<>>: jordan@example.com\n<<>>: Let me look that up for you.\n<<>>: Can you" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.915, + "snippet": "you.\n<<>>: Here is my <<>>: jordan@example.com\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.936, + "snippet": "dan@example.com\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.945, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your User Preferenc" + }, + { + "type": "<<>>", + "value": "marketing_emails=false", + "confidence": 0.96, + "snippet": "anks, can you also provide your <<>>: marketing_emails=false?\n<<>>: Can you check on that?\n<<>>: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.939, + "snippet": "de your <<>>: marketing_emails=false?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.933, + "snippet": "ting_emails=false?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.946, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.954, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.944, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0025.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0025.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..4d0c4bb7 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0025.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,423 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0025.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0025.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.823, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.806, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.943, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.893, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.942, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.938, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.619, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.989, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.518, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.787, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.781, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.718, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.753, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.951, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.898, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.872, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.582, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.565, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.941, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.855, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.81, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.664, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.882, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.904, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.589, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.799, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.585, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.958, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.925, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.981, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.847, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.745, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.955, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.583, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.594, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.592, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.563, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.712, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.744, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.678, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.637, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.624, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.668, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.937, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.947, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.92, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.603, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.817, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.805, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.967, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.87, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.813, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.948, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.936, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.862, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.899, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "User ID", + "value": "Taylor", + "confidence": 0.785, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.943,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.748, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.943,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.872, + "snippet": " my account.\\n\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.8, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.936,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.76, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.936,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0026.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0026.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..fb252370 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0026.txt.redacted.redaction.json.redacted @@ -0,0 +1,153 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0026.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0026.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.941, + "snippet": "Transcript #26\n==================================\n<<>>: Can you assist me with logging into my account.\n" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.91, + "snippet": ": Can you assist me with logging into my account.\n<<>>: Let me look that up for you.\n<<>>: Here is my Ge" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.937, + "snippet": "nto my account.\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: Non-binary\n<<>>: Let me look t" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.949, + "snippet": "me look that up for you.\n<<>>: Here is my <<>>: Non-binary\n<<>>: Let me look that up for you.\n<<>>: Can you c" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.862, + "snippet": "t up for you.\n<<>>: Here is my <<>>: Non-binary\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.92, + "snippet": "der: Non-binary\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.898, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my MA" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.931, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: L" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": "ok that up for you.\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: Let me look that up for you.\n<<>>: Can you c" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.85, + "snippet": ".\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.887, + "snippet": ":1B:44:11:3A:B7\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.896, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.879, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.904, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Ap" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.924, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.89, + "snippet": " is my Approximate Geolocation: San Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Here is my Un" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.943, + "snippet": "n Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: None" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.954, + "snippet": "at up for you.\n<<>>: Here is my <<>>: None" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0026.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0026.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..19446b49 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0026.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,429 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0026.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0026.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.823, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.802, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.941, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.897, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.947, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.938, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.607, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.517, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.786, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.782, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.714, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.764, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.949, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.895, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.872, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.589, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.563, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.942, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.852, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.804, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.673, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.882, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.906, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.587, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.799, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.56, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.957, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.926, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.981, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.848, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.762, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.955, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.598, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.588, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.588, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.557, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.715, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.751, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.675, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.619, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.612, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.666, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.938, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.948, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.92, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.597, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.819, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.802, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.964, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.866, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.814, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.946, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.935, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.509, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.863, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.868, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.781, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": 0.941,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Alex", + "confidence": 0.741, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": 0.941,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.84, + "snippet": " my account.\\n\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.813, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.91,\n \"snippet\": " + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.751, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.91,\n \"snippet\": " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0027.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0027.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..2ea7c570 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0027.txt.redacted.redaction.json.redacted @@ -0,0 +1,117 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0027.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0027.txt.redacted", + "totalRedactions": 7, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.58, + "snippet": "Transcript #27\n==================================\n<<>>: Can you assist me with resetting my password.\nSo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.541, + "snippet": "or: Can you assist me with resetting my password.\n<<>>: Thanks, can you also provide your Declared Inter" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.711, + "snippet": "assword.\n<<>>: Thanks, can you also provide your <<>>: hiking, cooking?\n<<>>: Can you check on that?\n" + }, + { + "type": "<<>>", + "value": "hiking, cooking", + "confidence": 0.775, + "snippet": "ks, can you also provide your <<>>: hiking, cooking?\n<<>>: Can you check on that?\n<<>>: Let me loo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.555, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.549, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.573, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.513, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: None" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.867, + "snippet": "on that?\n<<>>: Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Than" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.837, + "snippet": "ofia: Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Thanks, ca" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.502, + "snippet": "ur <<>>: None?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: h" + }, + { + "type": "<<>>", + "value": "https://social.example.com/jordan", + "confidence": 0.986, + "snippet": "a: Thanks, can you also provide your <<>>: https://social.example.com/jordan?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0027.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0027.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..a94e411a --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0027.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,435 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0027.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0027.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.871, + "snippet": "cted\",\n \"totalRedactions\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.795, + "snippet": "cted\",\n \"totalRedactions\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.932, + "snippet": "ons\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.923, + "snippet": "ons\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Declared Interests", + "value": "Biometric Identifier", + "confidence": 0.595, + "snippet": "ons\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.941, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.934, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.616, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.777, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.637, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.737, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.943, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.897, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.888, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.635, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.524, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.943, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.868, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.697, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.645, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.877, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.901, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.802, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.541, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.962, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.928, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.973, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.831, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.809, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.973, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.957, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.562, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.627, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.597, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.584, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.584, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.71, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.732, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.676, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.599, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.622, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.686, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.937, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.949, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.559, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.918, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.692, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.674, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.828, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.811, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.966, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.867, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.811, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.939, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.938, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.759, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.85, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.913, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "User ID", + "value": "Taylor", + "confidence": 0.661, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.58,\n \"snippet\": " + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.542, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.58,\n \"snippet\": " + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.9, + "snippet": " password.\\nSo\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Sofia\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Sofia", + "confidence": 0.727, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sofia\",\n \"confidence\": 0.541,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.603, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sofia\",\n \"confidence\": 0.541,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0028.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0028.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..0e888365 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0028.txt.redacted.redaction.json.redacted @@ -0,0 +1,93 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0028.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0028.txt.redacted", + "totalRedactions": 6, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.829, + "snippet": "anks, can you also provide your <<>>: <<>>?\nAlex: Can you check on that?\nSofia: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.601, + "snippet": "anks, can you also provide your <<>>: <<>>?\nAlex: Can you check on that?\nSofia: Thanks, can " + }, + { + "type": "<<>>", + "value": "+1-202-555-0172", + "confidence": 0.999, + "snippet": "?\nSofia: Thanks, can you also provide your <<>>: +1-202-555-0172?\nAlex: Can you check on that?\nSofia: Let me look " + }, + { + "type": "<<>>", + "value": "Hispanic", + "confidence": 0.992, + "snippet": "Thanks, can you also provide your <<>>: Hispanic?\nAlex: Here is my Survey Data: Q1=Yes, Q2=No\nSofi" + }, + { + "type": "<<>>", + "value": "FaceID registered", + "confidence": 0.902, + "snippet": ", can you also provide your <<>>: FaceID registered?\nAlex: Here is my Approximate Geolocation: San Fr" + }, + { + "type": "<<>>", + "value": "FaceID registered", + "confidence": 0.897, + "snippet": ", can you also provide your <<>>: FaceID registered?\nAlex: Here is my Approximate Geolocation: San Fr" + }, + { + "type": "<<>>", + "value": "123-45-6789", + "confidence": 0.999, + "snippet": "can you also provide your <<>>: 123-45-6789?\nAlex: Here is my <<>>: jordan_92" + }, + { + "type": "<<>>", + "value": "jordan_92", + "confidence": 0.966, + "snippet": "y Number: 123-45-6789?\nAlex: Here is my <<>>: jordan_92" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0028.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0028.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..719fa449 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0028.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,471 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0028.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0028.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.829, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.815, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Union Membership", + "value": "Account Number", + "confidence": 0.6, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.95, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.95, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.948, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.913, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.556, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.989, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.751, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.701, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.628, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.79, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Sexual Orientation", + "value": "Driver's License", + "confidence": 0.514, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.931, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.902, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.842, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.593, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.532, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.964, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.847, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.805, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.897, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.937, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.54, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.508, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.817, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.959, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.939, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.582, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.978, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.824, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.759, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.924, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.959, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.572, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.511, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.636, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.605, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.599, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.727, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.667, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.571, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.535, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.624, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.941, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.953, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.578, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.87, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.543, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.843, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.964, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.843, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.715, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.874, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.565, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.955, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.949, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.795, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.724, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Preferences\",\n \"value\": \"marketing_emails=false\",\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.584, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Preferences\",\n \"value\": \"marketing_emails=false\",\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.582, + "snippet": " \"snippet\": \"anks, can you also provide your User Preferences: marketing_emails=false?\\nAlex: Can you check on " + }, + { + "type": "User Preferences", + "value": "marketing_emails=false", + "confidence": 0.64, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\\nAlex: Can you check on that?\\nSofia: Thanks, ca" + }, + { + "type": "Biometric Identifier", + "value": "marketing_emails=false", + "confidence": 0.54, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\\nAlex: Can you check on that?\\nSofia: Thanks, ca" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.952, + "snippet": ": Thanks, can \"\n },\n {\n \"type\": \"Biometric Identifier\",\n \"value\": \"marketing_emails=false\",\n " + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.943, + "snippet": ": Thanks, can \"\n },\n {\n \"type\": \"Biometric Identifier\",\n \"value\": \"marketing_emails=false\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Biometric Identifier", + "confidence": 0.54, + "snippet": ": Thanks, can \"\n },\n {\n \"type\": \"Biometric Identifier\",\n \"value\": \"marketing_emails=false\",\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.716, + "snippet": " \"snippet\": \"anks, can you also provide your User Preferences: marketing_emails=false?\\nAlex: Can you check on " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.561, + "snippet": " \"snippet\": \"anks, can you also provide your User Preferences: marketing_emails=false?\\nAlex: Can you check on " + }, + { + "type": "User Preferences", + "value": "marketing_emails=false", + "confidence": 0.597, + "snippet": "anks, can you also provide your User Preferences: marketing_emails=false?\\nAlex: Can you check on that?\\nSofia: Thanks, ca" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.833, + "snippet": ": Thanks, can \"\n },\n {\n \"type\": \"Phone\",\n \"value\": \"+1-202-555-0172\",\n \"co" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0029.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0029.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..4225c162 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0029.txt.redacted.redaction.json.redacted @@ -0,0 +1,177 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0029.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0029.txt.redacted", + "totalRedactions": 6, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.892, + "snippet": "Transcript #29\n==================================\n<<>>: Hello—I'm having trouble with a double charge on" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.829, + "snippet": "m having trouble with a double charge on my card.\n<<>>: Thanks, can you also provide your <<>>:" + }, + { + "type": "<<>>", + "value": "SN-1234567890", + "confidence": 0.996, + "snippet": " Thanks, can you also provide your <<>>: SN-1234567890?\n<<>>: Can you check on that?\n<<>>: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.81, + "snippet": "u also provide your <<>>: SN-1234567890?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.765, + "snippet": "er: SN-1234567890?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.826, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.681, + "snippet": " is my Approximate Geolocation: San Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.793, + "snippet": "n Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.718, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.766, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.725, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.729, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.747, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.75, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.778, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Approximate Ge" + }, + { + "type": "<<>>", + "value": "San Francisco, CA", + "confidence": 0.594, + "snippet": "an you also provide your Approximate Geolocation: San Francisco, CA?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.804, + "snippet": " your Approximate Geolocation: San Francisco, CA?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.759, + "snippet": "San Francisco, CA?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Declared Inter" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.609, + "snippet": "k on that?\n<<>>: Thanks, can you also provide your <<>>: hiking, cooking?\n<<>>: Can you check on that?\n" + }, + { + "type": "<<>>", + "value": "hiking, cooking", + "confidence": 0.954, + "snippet": "ks, can you also provide your <<>>: hiking, cooking?\n<<>>: Can you check on that?\n<<>>: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.774, + "snippet": "provide your <<>>: hiking, cooking?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.774, + "snippet": ": hiking, cooking?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0029.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0029.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..46d3d8dd --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0029.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,435 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0029.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0029.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.858, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.799, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.953, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.912, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.954, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.911, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.524, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.779, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.768, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.73, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.797, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.952, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.889, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.848, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.564, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.515, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.937, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.867, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.817, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.658, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.905, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.928, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.611, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.783, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.964, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.919, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.851, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.793, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.979, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.955, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.681, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.598, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.576, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.552, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.514, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.733, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.798, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.698, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.653, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.591, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.674, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.969, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.866, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.582, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.921, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.612, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.838, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Social Security Number", + "confidence": 0.52, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.967, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.805, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.961, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.863, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.815, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.955, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.941, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.859, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.521, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.916, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\"" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.523, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.892,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.892, + "snippet": "uble charge on\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.523, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.829,\n \"snippet\":" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.736, + "snippet": " my card.\\nAva: Thanks, can you also provide your Serial Number:\"\n },\n {\n \"type\": \"Serial Number" + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.736, + "snippet": " my card.\\nAva: Thanks, can you also provide your Serial Number:\"\n },\n {\n \"type\": \"Serial Number" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0030.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0030.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..4e3e9f7b --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0030.txt.redacted.redaction.json.redacted @@ -0,0 +1,153 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0030.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0030.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.816, + "snippet": "Transcript #30\n==================================\n<<>>: Hi, I need help with updating my subscription.\nM" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.843, + "snippet": "n: Hi, I need help with updating my subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.758, + "snippet": " subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.856, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.762, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.761, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.879, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: " + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": " that up for you.\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: Let me look that up for you.\n<<>>: Here i" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.751, + "snippet": "<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.848, + "snippet": "1B:44:11:3A:B7\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: FaceID register" + }, + { + "type": "<<>>", + "value": "FaceID registered", + "confidence": 0.934, + "snippet": "for you.\n<<>>: Here is my <<>>: FaceID registered\n<<>>: Let me look that up for you.\n<<>>: Can yo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.729, + "snippet": "ere is my <<>>: FaceID registered\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.788, + "snippet": "eID registered\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.779, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.781, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.793, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.763, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.842, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0030.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0030.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..a6f12b6f --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0030.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0030.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0030.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.857, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.801, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.94, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.917, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.954, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.943, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.729, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.583, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.78, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.78, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.713, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.752, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.949, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.893, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.875, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.598, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.53, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.932, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.851, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.804, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.677, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.886, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.899, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.534, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.775, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.541, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.963, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.923, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.98, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.807, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.801, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.956, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.601, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.59, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.564, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.562, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.504, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.722, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.527, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.779, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.686, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.619, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.623, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.663, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.937, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.945, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.556, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.916, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.562, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.822, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.802, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.957, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.845, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.819, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.942, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.94, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.516, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.851, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.901, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\"" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.768, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.816,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Jordan", + "confidence": 0.667, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.816,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.879, + "snippet": "bscription.\\nM\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.753, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.843,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Maya", + "confidence": 0.647, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.843,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0031.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0031.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..72b5225d --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0031.txt.redacted.redaction.json.redacted @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0031.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0031.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.875, + "snippet": "ofia: Thanks, can you also provide your <<>>: <<>>?\nTaylor: Here is my <<>>: <<>>" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.826, + "snippet": "ofia: Thanks, can you also provide your <<>>: <<>>?\nTaylor: Here is my <<>>: <<>>" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.971, + "snippet": "r <<>>: <<>>?\nTaylor: Here is my <<>>: <<>>nks, can you also provide your <<>>:" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.61, + "snippet": "r <<>>: <<>>?\nTaylor: Here is my <<>>: <<>>nks, can you also provide your <<>>:" + }, + { + "type": "<<>>", + "value": "U123456", + "confidence": 0.967, + "snippet": "Sofia: Thanks, can you also provide your <<>>: U123456?\nTaylor: Can you check on that?\nSofia: Let me loo" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0031.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0031.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..6d313135 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0031.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,525 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0031.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0031.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.875, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.708, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.93, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.913, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.966, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.96, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.718, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.78, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.74, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.722, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.742, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.96, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.697, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.878, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.644, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.959, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Industry", + "confidence": 0.538, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.874, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.831, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.575, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.882, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.889, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.528, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.523, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.871, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.964, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.91, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.982, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.839, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.73, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.972, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.613, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.952, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Religion", + "value": "Profile Picture Image", + "confidence": 0.608, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.545, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.65, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.63, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.576, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.69, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.531, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.741, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.667, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.655, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Profile URL", + "value": "Region", + "confidence": 0.518, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.716, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Religion", + "confidence": 0.63, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.551, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.926, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.948, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.561, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.929, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.623, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.847, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.951, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.874, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.539, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.618, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "Username", + "value": "User Agent", + "confidence": 0.613, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.872, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Username", + "value": "User ID", + "confidence": 0.764, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.901, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.849, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.548, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.548, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Username", + "value": "User ID", + "confidence": 0.504, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User ID\",\n \"value\": \"jordan_92\",\n \"confiden" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.788, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.875,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "jordan_92", + "confidence": 0.759, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.875,\n \"snippet\":" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.765, + "snippet": "ofia: Thanks, can you also provide your Username: jordan_92?\\nTaylor: Here is my Gender: Non-binary\\nSofia: T" + }, + { + "type": "User ID", + "value": "jordan_92", + "confidence": 0.672, + "snippet": "ofia: Thanks, can you also provide your Username: jordan_92?\\nTaylor: Here is my Gender: Non-binary\\nSofia: T" + }, + { + "type": "User Agent", + "value": "jordan_92", + "confidence": 0.526, + "snippet": "ofia: Thanks, can you also provide your Username: jordan_92?\\nTaylor: Here is my Gender: Non-binary\\nSofia: T" + }, + { + "type": "Gender", + "value": "Non-binary\\nSofia: Tha", + "confidence": 0.606, + "snippet": " Username: jordan_92?\\nTaylor: Here is my Gender: Non-binary\\nSofia: Tha\"\n },\n {\n \"type\": \"Username\",\n " + }, + { + "type": "Name", + "value": "Non-binary\\nSofia: Tha", + "confidence": 0.552, + "snippet": " Username: jordan_92?\\nTaylor: Here is my Gender: Non-binary\\nSofia: Tha\"\n },\n {\n \"type\": \"Username\",\n " + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.835, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.826,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "jordan_92", + "confidence": 0.777, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.826,\n \"snippet\":" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.804, + "snippet": "ofia: Thanks, can you also provide your Username: jordan_92?\\nTaylor: Here is my Gender: Non-binary\\nSofia: T" + }, + { + "type": "User ID", + "value": "jordan_92", + "confidence": 0.743, + "snippet": "ofia: Thanks, can you also provide your Username: jordan_92?\\nTaylor: Here is my Gender: Non-binary\\nSofia: T" + }, + { + "type": "User Agent", + "value": "jordan_92", + "confidence": 0.61, + "snippet": "ofia: Thanks, can you also provide your Username: jordan_92?\\nTaylor: Here is my Gender: Non-binary\\nSofia: T" + }, + { + "type": "Name", + "value": "Non-binary\\nSofia: Tha", + "confidence": 0.694, + "snippet": " Username: jordan_92?\\nTaylor: Here is my Gender: Non-binary\\nSofia: Tha\"\n },\n {\n \"type\": \"Gender\",\n " + }, + { + "type": "Gender", + "value": "Non-binary\\nSofia: Tha", + "confidence": 0.625, + "snippet": " Username: jordan_92?\\nTaylor: Here is my Gender: Non-binary\\nSofia: Tha\"\n },\n {\n \"type\": \"Gender\",\n " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0032.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0032.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..47c48c12 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0032.txt.redacted.redaction.json.redacted @@ -0,0 +1,129 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0032.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0032.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.893, + "snippet": "Transcript #32\n==================================\n<<>>: Can you assist me with updating my subscription." + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.883, + "snippet": " Can you assist me with updating my subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.852, + "snippet": "y subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.864, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.833, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.872, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: Non-bi" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.982, + "snippet": "t?\n<<>>: Thanks, can you also provide your <<>>: Non-binary?\n<<>>: Can you check on that?\n<<>>: Let me look tha" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.825, + "snippet": "ks, can you also provide your <<>>: Non-binary?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.866, + "snippet": "r <<>>: Non-binary?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.882, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.889, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Biometric Iden" + }, + { + "type": "<<>>", + "value": "FaceID registered", + "confidence": 0.918, + "snippet": ", can you also provide your <<>>: FaceID registered?\n<<>>: Here is my <<>>: Hispanic" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.911, + "snippet": "ide your <<>>: FaceID registered?\n<<>>: Here is my <<>>: Hispanic" + }, + { + "type": "<<>>", + "value": "Hispanic", + "confidence": 0.989, + "snippet": "aceID registered?\n<<>>: Here is my <<>>: Hispanic" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0032.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0032.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..504152a4 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0032.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,441 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0032.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0032.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.87, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.812, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.942, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.912, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.944, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.94, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.647, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.501, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.796, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.778, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.725, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.762, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.954, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.893, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.879, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.615, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.525, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.938, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.86, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.808, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.701, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.886, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.901, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.569, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.79, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.566, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.962, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.924, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.981, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.803, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.793, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.975, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.955, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.576, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.566, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.561, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.528, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.523, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.724, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.755, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.683, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.639, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.598, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.65, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.939, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.946, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.597, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.915, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.566, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.825, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.806, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.973, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.861, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.777, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.943, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.932, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.505, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.853, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.917, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Sam", + "confidence": 0.772, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.893,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.542, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.893,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.864, + "snippet": " subscription.\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.811, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.883,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.566, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.883,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0033.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0033.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..5c80f3c0 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0033.txt.redacted.redaction.json.redacted @@ -0,0 +1,123 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0033.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0033.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.925, + "snippet": "Transcript #33\n==================================\n<<>>: Can you assist me with firmware update errors.\nA" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.838, + "snippet": "n: Can you assist me with firmware update errors.\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.891, + "snippet": " update errors.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.871, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.858, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.899, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.821, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.887, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: h" + }, + { + "type": "<<>>", + "value": "https://social.example.com", + "confidence": 0.66, + "snippet": "a: Thanks, can you also provide your <<>>: https://social.example.com/jordan?\n<<>>: Can you check on that?\n<<>>: Let m" + }, + { + "type": "<<>>", + "value": "jordan", + "confidence": 0.601, + "snippet": "vide your <<>>: https://social.example.com/jordan?\n<<>>: Can you check on that?\n<<>>: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.865, + "snippet": "r <<>>: https://social.example.com/jordan?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.897, + "snippet": "xample.com/jordan?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.849, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0033.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0033.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..618bbb25 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0033.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,423 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0033.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0033.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.87, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.804, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.951, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.868, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.952, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.94, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.992, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.827, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.775, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.738, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.779, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.954, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.896, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.874, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.6, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.948, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.522, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.866, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.832, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.764, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.887, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.892, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.605, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.781, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.524, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.963, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.926, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.829, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.821, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.971, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.953, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.568, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.585, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.569, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.547, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.724, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.771, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.706, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.608, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.649, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.7, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.507, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.934, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.947, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.523, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.916, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.638, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.813, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.961, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.817, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.546, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.964, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.822, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.742, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.952, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.928, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.851, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.912, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\"" + }, + { + "type": "User ID", + "value": "Jordan", + "confidence": 0.782, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.925,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.693, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.925,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.88, + "snippet": "ate errors.\\nA\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.745, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.838,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0034.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0034.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..29836694 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0034.txt.redacted.redaction.json.redacted @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0034.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0034.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Serial Number", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.99, + "snippet": " Thanks, can you also provide your <<>>: <<>>?\nAlex: Can you check on that?\nNoah: Thanks, can y" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.996, + "snippet": " Thanks, can you also provide your Serial Number: SN-1234567890?\nAlex: Can you check on that?\nNoah: Let me look t" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.992, + "snippet": "\nNoah: Thanks, can you also provide your <<>>: <<>>?\nAlex: Can you check on that?\nNoah: Let me look t" + }, + { + "type": "<<>>", + "value": "jordan@example.com", + "confidence": 0.998, + "snippet": " me look that up for you.\nAlex: Here is my <<>>: jordan@example.com\nNoah: Let me look that up for you.\nAlex: Can you " + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.995, + "snippet": " that up for you.\nAlex: Here is my Serial Number: SN-1234567890\nNoah: Let me look that up for you.\nAlex: Can you " + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0034.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0034.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..4bc1abda --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0034.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,435 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0034.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0034.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.835, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.827, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.941, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.878, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.969, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.949, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.616, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.698, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.823, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.795, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.787, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.961, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.57, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.911, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.871, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.683, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.541, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.955, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.867, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.826, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.538, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.892, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.943, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.814, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.781, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.561, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.969, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.918, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.984, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.829, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.803, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.981, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.525, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.953, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.692, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.639, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.625, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.596, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.5, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.727, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.815, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.73, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.704, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.563, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.639, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.972, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.923, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.836, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.963, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.823, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.571, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.941, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.891, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.853, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.94, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.934, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.842, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.84, + "snippet": " \"type\": \"Date of Birth\",\n \"value\": \"1990-05-12\",\n \"confidence\": 0.99,\n \"snippet\": " + }, + { + "type": "Credit Card Number", + "value": "1990-05-12", + "confidence": 0.648, + "snippet": " \"type\": \"Date of Birth\",\n \"value\": \"1990-05-12\",\n \"confidence\": 0.99,\n \"snippet\": " + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.907, + "snippet": " Thanks, can you also provide your Date of Birth: 1990-05-12?\\nAlex: Can you check on that?\\nNoah: Thanks, can" + }, + { + "type": "Credit Card Number", + "value": "1990-05-12", + "confidence": 0.737, + "snippet": " Thanks, can you also provide your Date of Birth: 1990-05-12?\\nAlex: Can you check on that?\\nNoah: Thanks, can" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.838, + "snippet": " Let me look t\"\n },\n {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.703, + "snippet": " Let me look t\"\n },\n {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.757, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence\": 0.992,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "U123456", + "confidence": 0.517, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence\": 0.992,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0035.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0035.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..f22d3606 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0035.txt.redacted.redaction.json.redacted @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0035.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0035.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Region", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.998, + "snippet": "at?\nAva: Thanks, can you also provide your <<>>: <<>>?\nJordan: Here is my <<>>: <<>>\nAva: Let " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.985, + "snippet": "ne: <<>>?\nJordan: Here is my <<>>: <<>>\nAva: Let me look that up for you.\nJordan: Can you" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.964, + "snippet": "t?\nAva: Thanks, can you also provide your <<>>: Non-binary?" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.549, + "snippet": "t?\nAva: Thanks, can you also provide your <<>>: Non-binary?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0035.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0035.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..7288b8cb --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0035.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,501 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0035.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0035.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.827, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.535, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.927, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.902, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.937, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.916, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.812, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.772, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.662, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.757, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.952, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.542, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.831, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.658, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.866, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.622, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.53, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.95, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.887, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.847, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.568, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.834, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.914, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.714, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.643, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.875, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.96, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.914, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.847, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.758, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.901, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.595, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.936, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.623, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.595, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.657, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.529, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.527, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.75, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.63, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Profile Picture URL", + "value": "Profile URL", + "confidence": 0.506, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.721, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.657, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.621, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.932, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.939, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.531, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.907, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.553, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.832, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.958, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.829, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Username", + "value": "User Agent", + "confidence": 0.751, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.643, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "Username", + "value": "User ID", + "confidence": 0.784, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.775, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Name", + "value": "User ID", + "confidence": 0.74, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.917, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.853, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Gender", + "value": "User Preferences", + "confidence": 0.621, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.701, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.816, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Phone\",\n \"value\": \"+1-202-555-0172\",\n \"co" + }, + { + "type": "Routing Number", + "value": "+1-202-555-0172", + "confidence": 0.846, + "snippet": " {\n \"type\": \"Phone\",\n \"value\": \"+1-202-555-0172\",\n \"confidence\": 0.998,\n \"snippet\":" + }, + { + "type": "MAC Address", + "value": "+1-202-555-0172", + "confidence": 0.633, + "snippet": " {\n \"type\": \"Phone\",\n \"value\": \"+1-202-555-0172\",\n \"confidence\": 0.998,\n \"snippet\":" + }, + { + "type": "Routing Number", + "value": "+1-202-555-0172", + "confidence": 0.894, + "snippet": "t?\\nAva: Thanks, can you also provide your Phone: +1-202-555-0172?\\nJordan: Here is my Username: jordan_92\\nAva: Le" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.852, + "snippet": "e: +1-202-555-0172?\\nJordan: Here is my Username: jordan_92\\nAva: Let \"\n },\n {\n \"type\": \"Use" + }, + { + "type": "User Agent", + "value": "jordan_92", + "confidence": 0.553, + "snippet": "e: +1-202-555-0172?\\nJordan: Here is my Username: jordan_92\\nAva: Let \"\n },\n {\n \"type\": \"Use" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.858, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.985,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "jordan_92", + "confidence": 0.502, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.985,\n \"snippet\":" + }, + { + "type": "Routing Number", + "value": "+1-202-555-0172", + "confidence": 0.833, + "snippet": " \"confidence\": 0.985,\n \"snippet\": \"ne: +1-202-555-0172?\\nJordan: Here is my Username: jordan_92\\nAva: Le" + }, + { + "type": "MAC Address", + "value": "+1-202-555-0172", + "confidence": 0.781, + "snippet": " \"confidence\": 0.985,\n \"snippet\": \"ne: +1-202-555-0172?\\nJordan: Here is my Username: jordan_92\\nAva: Le" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.75, + "snippet": "e: +1-202-555-0172?\\nJordan: Here is my Username: jordan_92\\nAva: Let me look that up for you.\\nJordan: Can y" + }, + { + "type": "User Agent", + "value": "jordan_92", + "confidence": 0.522, + "snippet": "e: +1-202-555-0172?\\nJordan: Here is my Username: jordan_92\\nAva: Let me look that up for you.\\nJordan: Can y" + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.697, + "snippet": "ordan: Can you\"\n },\n {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confide" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0036.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0036.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..460b1762 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0036.txt.redacted.redaction.json.redacted @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0036.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0036.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.953, + "snippet": "?\nNoah: Thanks, can you also provide your <<>>: <<>>?\nCasey: Can you check on that?\nNoah: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.996, + "snippet": "ks, can you also provide your <<>>: <<>>?\nCasey: Here is my Survey Data: Q1=Yes, Q2=No\nNoa" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.999, + "snippet": "h: Thanks, can you also provide your <<>>: 00:1B:44:11:3A:B7?\nCasey: Can you check on that?\nNoah: Let me look " + }, + { + "type": "<<>>", + "value": "U123456", + "confidence": 0.984, + "snippet": " look that up for you.\nCasey: Here is my <<>>: U123456\nNoah: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0036.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0036.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..c815c138 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0036.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,441 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0036.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0036.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.84, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.834, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.942, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.831, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Gender", + "value": "Biometric Identifier", + "confidence": 0.632, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.968, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.922, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.767, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.75, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.824, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.969, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.575, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Credit Card Number", + "value": "Email", + "confidence": 0.535, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.799, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.881, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.766, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.877, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Industry", + "confidence": 0.691, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.886, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.821, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.845, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.872, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.809, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.877, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.97, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.934, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.898, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.628, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.982, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.56, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.947, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.53, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.681, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.658, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.724, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.775, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.628, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.595, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.637, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.937, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.956, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.935, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.82, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.958, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.856, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.502, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.901, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.647, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.928, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Gender", + "value": "User Preferences", + "confidence": 0.867, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.779, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.784, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.673, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confide" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.773, + "snippet": " {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.953,\n \"snippet\":" + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.543, + "snippet": "pet\": \"?\\nNoah: Thanks, can you also provide your Gender: Non-binary?\\nCasey: Can you check on that?\\nNoah" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.88, + "snippet": "\\nNoah: Thanks, can you also provide your Gender: Non-binary?\\nCasey: Can you check on that?\\nNoah: Thanks, ca" + }, + { + "type": "User Preferences", + "value": "Non-binary", + "confidence": 0.653, + "snippet": "\\nNoah: Thanks, can you also provide your Gender: Non-binary?\\nCasey: Can you check on that?\\nNoah: Thanks, ca" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.596, + "snippet": ": Thanks, can \"\n },\n {\n \"type\": \"Sexual Orientation\",\n \"value\": \"Heterosexual\",\n \"confi" + }, + { + "type": "Gender", + "value": "Heterosexual", + "confidence": 0.532, + "snippet": " \"type\": \"Sexual Orientation\",\n \"value\": \"Heterosexual\",\n \"confidence\": 0.996,\n \"snippet\":" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.534, + "snippet": " \"snippet\": \"ks, can you also provide your Sexual Orientation: Heterosexual?\\nCasey: Here is my Survey Data: Q1" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.758, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\\nCasey: Here is my Survey Data: Q1=Yes, Q2=No\\nN" + }, + { + "type": "Race/Ethnicity", + "value": "Heterosexual", + "confidence": 0.538, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\\nCasey: Here is my Survey Data: Q1=Yes, Q2=No\\nN" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.69, + "snippet": "es, Q2=No\\nNoa\"\n },\n {\n \"type\": \"MAC Address\",\n \"value\": \"00:1B:44:11:3A:B7\",\n \"" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.516, + "snippet": "es, Q2=No\\nNoa\"\n },\n {\n \"type\": \"MAC Address\",\n \"value\": \"00:1B:44:11:3A:B7\",\n \"" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0037.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0037.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..cc0b9973 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0037.txt.redacted.redaction.json.redacted @@ -0,0 +1,159 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0037.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0037.txt.redacted", + "totalRedactions": 9, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Profile Picture Image", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.787, + "snippet": "an you assist me with a double charge on my card.\n<<>>: Thanks, can you also provide your Union Membersh" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.903, + "snippet": "anks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me loo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.612, + "snippet": "can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.863, + "snippet": " Membership: None?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.906, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Political Affi" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.532, + "snippet": "on that?\n<<>>: Thanks, can you also provide your <<>>: Independent?\n<<>>: Can you check on that?\nEtha" + }, + { + "type": "<<>>", + "value": "Independent", + "confidence": 0.97, + "snippet": " can you also provide your <<>>: Independent?\n<<>>: Can you check on that?\n<<>>: Let me loo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.791, + "snippet": "tion: Independent?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.896, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your User Preferenc" + }, + { + "type": "<<>>", + "value": "marketing_emails=false", + "confidence": 0.967, + "snippet": "anks, can you also provide your <<>>: marketing_emails=false?\n<<>>: Here is my Approximate Geolocation: San " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.632, + "snippet": "de your <<>>: marketing_emails=false?\n<<>>: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.654, + "snippet": "Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "<<>>", + "value": "San Francisco, CA", + "confidence": 0.626, + "snippet": " you.\n<<>>: Here is my Approximate Geolocation: San Francisco, CA\n<<>>: Thanks, can you also provide your <<>>" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.732, + "snippet": " is my Approximate Geolocation: San Francisco, CA\n<<>>: Thanks, can you also provide your <<>>: jord" + }, + { + "type": "<<>>", + "value": "jordan_92", + "confidence": 0.956, + "snippet": "than: Thanks, can you also provide your <<>>: jordan_92?\n<<>>: Can you check on that?\n<<>>: Thanks, ca" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.537, + "snippet": "s, can you also provide your <<>>: jordan_92?\n<<>>: Can you check on that?\n<<>>: Thanks, can you al" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.922, + "snippet": "ername: jordan_92?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: 0" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.995, + "snippet": "n: Thanks, can you also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.783, + "snippet": "also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0037.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0037.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..3a42bbc0 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0037.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,465 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0037.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0037.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.883, + "snippet": "cted\",\n \"totalRedactions\": 9,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.749, + "snippet": "cted\",\n \"totalRedactions\": 9,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.949, + "snippet": "ons\": 9,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.912, + "snippet": "ons\": 9,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.936, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.889, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.83, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.829, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.622, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.814, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Religion", + "value": "Driver's License", + "confidence": 0.51, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.96, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.886, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.866, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.672, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.58, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.922, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Union Membership", + "value": "Industry", + "confidence": 0.814, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.909, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.897, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.672, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.796, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.886, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.592, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.54, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.847, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.966, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.928, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.977, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.755, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.748, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.977, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.556, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.533, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.957, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.594, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.55, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.54, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.763, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Profile Picture URL", + "value": "Profile URL", + "confidence": 0.555, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.574, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.519, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.726, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.942, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.965, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.885, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.604, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.821, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Social Security Number", + "confidence": 0.503, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.963, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.975, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.624, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.676, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "Name", + "value": "User ID", + "confidence": 0.751, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.644, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.598, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Username", + "value": "User ID", + "confidence": 0.508, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.943, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.835, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.661, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.843, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.881, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\":" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.881, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\": 0.787,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Ethan", + "confidence": 0.509, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\": 0.787,\n \"snippet\":" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.932, + "snippet": "Union Membersh\"\n },\n {\n \"type\": \"Union Membership\",\n \"value\": \"None\",\n \"confidence\": " + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.951, + "snippet": " \"snippet\": \"anks, can you also provide your Union Membership: None?\\nTaylor: Can you check on that?\\nEthan: Le" + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.522, + "snippet": " \"snippet\": \"anks, can you also provide your Union Membership: None?\\nTaylor: Can you check on that?\\nEthan: Le" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.836, + "snippet": "an: Let me loo\"\n },\n {\n \"type\": \"Username\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "Username", + "value": "Taylor", + "confidence": 0.88, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Taylor\",\n \"confidence\": 0.612,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0038.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0038.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..77dea2af --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0038.txt.redacted.redaction.json.redacted @@ -0,0 +1,93 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0038.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0038.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.514, + "snippet": "Transcript #38\n==================================\n<<>>: Hello—I'm having trouble with a late delivery.\nE" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.711, + "snippet": "n: Hello—I'm having trouble with a late delivery.\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.562, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.517, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.524, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: 1990-05-12\n<<>>: Let " + }, + { + "type": "<<>>", + "value": "1990-05-12", + "confidence": 0.992, + "snippet": "hat up for you.\n<<>>: Here is my <<>>: 1990-05-12\n<<>>: Let me look that up for you.\n<<>>: Can y" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.505, + "snippet": "h: 1990-05-12\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.632, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0038.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0038.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..a066e14d --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0038.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,429 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0038.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0038.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.861, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.782, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.941, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.882, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.944, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.937, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.584, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.793, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.775, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.74, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.729, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.947, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.901, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.867, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.592, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.555, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.943, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.862, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.808, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.672, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.878, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.891, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.584, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.796, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.6, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.957, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.914, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.98, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.835, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.782, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.973, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.952, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.578, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.568, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.554, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.55, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.518, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.71, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.853, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.699, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.545, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.647, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.662, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.936, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.948, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.543, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.91, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.578, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.814, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.958, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.808, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.951, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.844, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.802, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.946, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.939, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.86, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.865, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\"" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.854, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.514,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Jordan", + "confidence": 0.797, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.514,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.797, + "snippet": "e delivery.\\nE\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Ethan", + "confidence": 0.774, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\": 0.711,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.737, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\": 0.711,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0039.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0039.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..40ac11a7 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0039.txt.redacted.redaction.json.redacted @@ -0,0 +1,141 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0039.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0039.txt.redacted", + "totalRedactions": 7, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.517, + "snippet": "update errors.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.725, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>:" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.982, + "snippet": " Thanks, can you also provide your <<>>: <<>>?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.527, + "snippet": "u also provide your <<>>: <<>>?\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.781, + "snippet": "ber: <<>>?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Profile Pictur" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.644, + "snippet": "ture URL: https://example.com/img/profile123.jpg?\n<<>>: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.717, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.662, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my R" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.862, + "snippet": "m: Let me look that up for you.\n<<>>: Here is my <<>>: None\n<<>>: Thanks, can you also provide your Use" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.711, + "snippet": "m: Let me look that up for you.\n<<>>: Here is my <<>>: None\n<<>>: Thanks, can you also provide your Use" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.963, + "snippet": "look that up for you.\n<<>>: Here is my <<>>: None\n<<>>: Thanks, can you also provide your <<>>:" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.886, + "snippet": "look that up for you.\n<<>>: Here is my <<>>: None\n<<>>: Thanks, can you also provide your <<>>:" + }, + { + "type": "<<>>", + "value": "jordan_92", + "confidence": 0.975, + "snippet": "<<>>: Thanks, can you also provide your <<>>: jordan_92?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.708, + "snippet": "sername: jordan_92?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Social Securit" + }, + { + "type": "<<>>", + "value": "123-45-6789", + "confidence": 0.997, + "snippet": "can you also provide your <<>>: 123-45-6789?\n<<>>: Can you check on that?" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.578, + "snippet": "provide your <<>>: 123-45-6789?\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0039.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0039.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..cac4bcec --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0039.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,417 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0039.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0039.txt.redacted.redaction.json.redacted", + "totalRedactions": 37, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.843, + "snippet": "cted\",\n \"totalRedactions\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.757, + "snippet": "cted\",\n \"totalRedactions\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.928, + "snippet": "ons\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.909, + "snippet": "ons\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.942, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.84, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.872, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.834, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.65, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.819, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.958, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.893, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.882, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.633, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.564, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.964, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.918, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.875, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.649, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.8, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.927, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.77, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.861, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.969, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.912, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.981, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.889, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.742, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.978, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.717, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.577, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.947, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.632, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.576, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.563, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.714, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.653, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.553, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.694, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.976, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.655, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.916, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.56, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.847, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Social Security Number", + "confidence": 0.617, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.968, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Social Security Number", + "value": "Tax Information", + "confidence": 0.54, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.819, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.608, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "Name", + "value": "User ID", + "confidence": 0.657, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.613, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.931, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.878, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.87, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.859, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Username\",\n \"value\": \"Riley\",\n \"confidence\":" + }, + { + "type": "Username", + "value": "Riley", + "confidence": 0.882, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Riley\",\n \"confidence\": 0.517,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Riley", + "confidence": 0.55, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Riley\",\n \"confidence\": 0.517,\n \"snippet\":" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.838, + "snippet": "s, can you als\"\n },\n {\n \"type\": \"Username\",\n \"value\": \"Liam\",\n \"confidence\": " + }, + { + "type": "Username", + "value": "Liam", + "confidence": 0.874, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Liam\",\n \"confidence\": 0.725,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Liam", + "confidence": 0.51, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Liam\",\n \"confidence\": 0.725,\n \"snippet\":" + }, + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.576, + "snippet": " \"type\": \"Serial Number\",\n \"value\": \"SN-1234567890\",\n \"confidence\": 0.982,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0040.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0040.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..b440ec5c --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0040.txt.redacted.redaction.json.redacted @@ -0,0 +1,135 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0040.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0040.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.715, + "snippet": "Transcript #40\n==================================\n<<>>: Hello—I'm having trouble with firmware update er" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.686, + "snippet": "o—I'm having trouble with firmware update errors.\n<<>>: Thanks, can you also provide your <<>>" + }, + { + "type": "<<>>", + "value": "Heterosexual", + "confidence": 0.991, + "snippet": "ks, can you also provide your <<>>: Heterosexual?\n<<>>: Here is my Approximate Geolocation: San F" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.783, + "snippet": "so provide your <<>>: Heterosexual?\n<<>>: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "<<>>", + "value": "San Francisco, CA", + "confidence": 0.561, + "snippet": "exual?\n<<>>: Here is my Approximate Geolocation: San Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Here i" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.589, + "snippet": " is my Approximate Geolocation: San Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Here is my S" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.671, + "snippet": "Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Here is my Survey Data: Q1=Yes, Q2=No\n<<>>: Let" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.723, + "snippet": "Q1=Yes, Q2=No\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.679, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.639, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.756, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.643, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you al" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.754, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>:" + }, + { + "type": "<<>>", + "value": "SN-1234567890", + "confidence": 0.999, + "snippet": " Thanks, can you also provide your <<>>: SN-1234567890?\n<<>>: Can you check on that?" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.674, + "snippet": "u also provide your <<>>: SN-1234567890?\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0040.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0040.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..2cf4d54c --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0040.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,465 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0040.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0040.txt.redacted.redaction.json.redacted", + "totalRedactions": 37, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.874, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.819, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.952, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.846, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.961, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.914, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.993, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.782, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.726, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.544, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.793, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.949, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.52, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.894, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.876, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.662, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.564, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.925, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.586, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.859, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.749, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.893, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.901, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.62, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.795, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.963, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.93, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.978, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.861, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.83, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.968, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.588, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.957, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.647, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.647, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.516, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.599, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.564, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.515, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.514, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.717, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.678, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.665, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.65, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.636, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.685, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.649, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.934, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.948, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.543, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.949, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.523, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.816, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.826, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.57, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.975, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.857, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.783, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.953, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.938, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.51, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.856, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.83, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Riley", + "confidence": 0.724, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.715,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.666, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.715,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.731, + "snippet": "ware update er\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Ethan", + "confidence": 0.716, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\": 0.686,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.603, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\": 0.686,\n \"snippet\":" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orienta", + "confidence": 0.842, + "snippet": "errors.\\nEthan: Thanks, can you also provide your Sexual Orienta\"\n },\n {\n \"type\": \"Sexual Orienta" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0041.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0041.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..ad506da3 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0041.txt.redacted.redaction.json.redacted @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0041.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0041.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.878, + "snippet": " on that?\nLiam: Thanks, can you also provide your <<>>: None?\nAlex: Can you check on that?\nLiam: Thanks," + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.976, + "snippet": "anks, can you also provide your <<>>: None?\nAlex: Can you check on that?\nLiam: Thanks, can y" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.79, + "snippet": " on that?\nLiam: Thanks, can you also provide your <<>>: hiking, cooking?\nAlex: Can you check on that?" + }, + { + "type": "<<>>", + "value": "hiking, cooking", + "confidence": 0.875, + "snippet": "ks, can you also provide your <<>>: hiking, cooking?\nAlex: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0041.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0041.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..0d9540a5 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0041.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,453 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0041.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0041.txt.redacted.redaction.json.redacted", + "totalRedactions": 34, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.801, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.784, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.95, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.902, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Declared Interests", + "value": "Biometric Identifier", + "confidence": 0.623, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.947, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.943, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.814, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.831, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Religion", + "value": "Driver's License", + "confidence": 0.541, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.958, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.524, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.903, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.851, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.675, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.524, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.943, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Union Membership", + "value": "Industry", + "confidence": 0.809, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.873, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.862, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.905, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.544, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.784, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.961, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.917, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.983, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.807, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.784, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.645, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.961, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.52, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.636, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.586, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.518, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.696, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.696, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.661, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.645, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.709, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.748, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.924, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.957, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.927, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.911, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.755, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.859, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.945, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.964, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.611, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.86, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.771, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.769, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.934, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.881, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.787, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.817, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.843, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Union Membership\",\n \"value\": \"Union Membership\",\n \"c" + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.662, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Union Membership\",\n \"value\": \"Union Membership\",\n \"c" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.827, + "snippet": " \"type\": \"Union Membership\",\n \"value\": \"Union Membership\",\n \"confidence\": 0.878,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.567, + "snippet": " \"type\": \"Union Membership\",\n \"value\": \"Union Membership\",\n \"confidence\": 0.878,\n \"snippet\":" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.867, + "snippet": "on that?\\nLiam: Thanks, can you also provide your Union Membership: None?\\nAlex: Can you check on that?\\nLiam: Thank" + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.566, + "snippet": "on that?\\nLiam: Thanks, can you also provide your Union Membership: None?\\nAlex: Can you check on that?\\nLiam: Thank" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.756, + "snippet": "nLiam: Thanks,\"\n },\n {\n \"type\": \"Union Membership\",\n \"value\": \"None\",\n \"confidence\": " + }, + { + "type": "Industry", + "value": "Union Membership", + "confidence": 0.564, + "snippet": "nLiam: Thanks,\"\n },\n {\n \"type\": \"Union Membership\",\n \"value\": \"None\",\n \"confidence\": " + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.876, + "snippet": " \"snippet\": \"anks, can you also provide your Union Membership: None?\\nAlex: Can you check on that?\\nLiam: Thank" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.671, + "snippet": " Thanks, can y\"\n },\n {\n \"type\": \"Declared Interests\",\n \"value\": \"Declared Interests\",\n " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0042.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0042.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..293dcd16 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0042.txt.redacted.redaction.json.redacted @@ -0,0 +1,51 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0042.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0042.txt.redacted", + "totalRedactions": 1, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.987, + "snippet": "e look that up for you.\nSam: Here is my <<>>: <<>>\nMaya: Let me look that up for you.\nSam: Can you c" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0042.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0042.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..960bdeaa --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0042.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,453 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0042.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0042.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.84, + "snippet": "cted\",\n \"totalRedactions\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.726, + "snippet": "cted\",\n \"totalRedactions\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.923, + "snippet": "ons\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.892, + "snippet": "ons\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.93, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.896, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.993, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.836, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.771, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.635, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.822, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.963, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.898, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.863, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.584, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.527, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.965, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.859, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.823, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.647, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.879, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.925, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.729, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.848, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "User ID", + "value": "Name", + "confidence": 0.616, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.955, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.902, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.984, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.901, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.846, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.975, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.959, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.586, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Religion", + "value": "Profile Picture Image", + "confidence": 0.536, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.506, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.577, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.542, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.509, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.686, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.543, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.702, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.679, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.688, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.949, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.957, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.582, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.926, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.691, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.876, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Social Security Number", + "confidence": 0.556, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.961, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.847, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Username", + "value": "User Agent", + "confidence": 0.754, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.75, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "Name", + "value": "User ID", + "confidence": 0.802, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.771, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Username", + "value": "User ID", + "confidence": 0.699, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.938, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.894, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.53, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.849, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.79, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.987,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "jordan_92", + "confidence": 0.691, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.987,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "jordan_92", + "confidence": 0.585, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.987,\n \"snippet\":" + }, + { + "type": "Name", + "value": "jordan_92", + "confidence": 0.547, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.987,\n \"snippet\":" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.808, + "snippet": " look that up for you.\\nSam: Here is my Username: jordan_92\\nMaya: Let me look that up for you.\\nSam: Can you" + }, + { + "type": "User Agent", + "value": "jordan_92", + "confidence": 0.636, + "snippet": " look that up for you.\\nSam: Here is my Username: jordan_92\\nMaya: Let me look that up for you.\\nSam: Can you" + }, + { + "type": "User ID", + "value": "jordan_92", + "confidence": 0.633, + "snippet": " look that up for you.\\nSam: Here is my Username: jordan_92\\nMaya: Let me look that up for you.\\nSam: Can you" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0043.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0043.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..ea4fe02f --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0043.txt.redacted.redaction.json.redacted @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0043.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0043.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.582, + "snippet": "Transcript #43\n==================================\n<<>>: Hi, I need help with updating my subscription.\nM" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.605, + "snippet": "check on that?\nMaya: Let me look that up for you.\n<<>>: Here is my <<>>: jordan@example.com\nMaya: Let m" + }, + { + "type": "<<>>", + "value": "jordan@example.com", + "confidence": 0.998, + "snippet": "t me look that up for you.\n<<>>: Here is my <<>>: jordan@example.com\nMaya: Let me look that up for you.\n<<>>: Can you c" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.517, + "snippet": "check on that?\nMaya: Let me look that up for you.\n<<>>: Can you check on that?\nMaya: Let me look that up" + }, + { + "type": "<<>>", + "value": "Maya", + "confidence": 0.542, + "snippet": "look that up for you.\n<<>>: Can you check on that?\nMaya: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0043.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0043.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..e20c5165 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0043.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,453 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0043.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0043.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.852, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.749, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.949, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.909, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.952, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.946, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.657, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.822, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.777, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.734, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.782, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.952, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.903, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.881, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.661, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.945, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.866, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.838, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.686, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.866, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.898, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.564, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.796, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.513, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.958, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.936, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.816, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.752, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.973, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.951, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.523, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.615, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.587, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.576, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.511, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.712, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.748, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.666, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.665, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.617, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.654, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.931, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.942, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.922, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.56, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.828, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.959, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.804, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.961, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.862, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.817, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.941, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.932, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.578, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.824, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.84, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0" + }, + { + "type": "Biometric Identifier", + "value": "Sam", + "confidence": 0.692, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.582,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Sam", + "confidence": 0.594, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.582,\n \"snippet\":" + }, + { + "type": "User Preferences", + "value": "Sam", + "confidence": 0.545, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.582,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "Sam", + "confidence": 0.537, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.582,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.827, + "snippet": "bscription.\\nM\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0" + }, + { + "type": "Biometric Identifier", + "value": "Sam", + "confidence": 0.737, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.605,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Sam", + "confidence": 0.659, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.605,\n \"snippet\":" + }, + { + "type": "User Preferences", + "value": "Sam", + "confidence": 0.613, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.605,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "Sam", + "confidence": 0.543, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.605,\n \"snippet\":" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.747, + "snippet": "ya: Let me look that up for you.\\nSam: Here is my Email: jordan@example.com\\nMaya: Let m\"\n },\n " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0044.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0044.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..106c6efc --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0044.txt.redacted.redaction.json.redacted @@ -0,0 +1,81 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0044.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0044.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.957, + "snippet": " me look that up for you.\nSam: Here is my <<>>: <<>>\nLiam: Let me look that up for you.\nSam: Can you c" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.811, + "snippet": " me look that up for you.\nSam: Here is my <<>>: <<>>\nLiam: Let me look that up for you.\nSam: Can you c" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.751, + "snippet": " on that?\nLiam: Thanks, can you also provide your <<>>: Independent?\nSam: Can you check on that?\nLiam: T" + }, + { + "type": "<<>>", + "value": "Independent", + "confidence": 0.979, + "snippet": " can you also provide your <<>>: Independent?\nSam: Can you check on that?\nLiam: Thanks, can yo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.917, + "snippet": " on that?\nLiam: Thanks, can you also provide your <<>>: Independent?\nSam: Can you check on that?\nLiam: L" + }, + { + "type": "<<>>", + "value": "Independent", + "confidence": 0.985, + "snippet": " can you also provide your <<>>: Independent?\nSam: Can you check on that?\nLiam: Let me look th" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0044.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0044.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..6a9218bb --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0044.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,471 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0044.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0044.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.85, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.758, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.942, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.827, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Gender", + "value": "Biometric Identifier", + "confidence": 0.691, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.968, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.961, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.648, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.993, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.709, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.706, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.736, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.97, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.581, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.826, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.886, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.751, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.519, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.849, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.88, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.795, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.882, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.929, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.784, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.865, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.531, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.967, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.927, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.99, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.949, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.976, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.694, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.513, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.944, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.622, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.539, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.704, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.632, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.579, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.674, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.537, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.854, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.829, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.532, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.639, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.94, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.954, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.97, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.877, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.957, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.861, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.515, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.919, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.68, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.927, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Gender", + "value": "User Preferences", + "confidence": 0.869, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.779, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.814, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.775, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confide" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.765, + "snippet": " {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.957,\n \"snippet\":" + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.673, + "snippet": "pet\": \" me look that up for you.\\nSam: Here is my Gender: Non-binary\\nLiam: Let me look that up for you.\\n" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.838, + "snippet": "me look that up for you.\\nSam: Here is my Gender: Non-binary\\nLiam: Let me look that up for you.\\nSam: Can you" + }, + { + "type": "User ID", + "value": "Non-binary", + "confidence": 0.651, + "snippet": "me look that up for you.\\nSam: Here is my Gender: Non-binary\\nLiam: Let me look that up for you.\\nSam: Can you" + }, + { + "type": "Race/Ethnicity", + "value": "Non-binary", + "confidence": 0.627, + "snippet": "me look that up for you.\\nSam: Here is my Gender: Non-binary\\nLiam: Let me look that up for you.\\nSam: Can you" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.905, + "snippet": "Sam: Can you c\"\n },\n {\n \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confide" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.803, + "snippet": " \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.811,\n \"snippet\":" + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.612, + "snippet": "pet\": \" me look that up for you.\\nSam: Here is my Gender: Non-binary\\nLiam: Let me look that up for you.\\n" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.797, + "snippet": "me look that up for you.\\nSam: Here is my Gender: Non-binary\\nLiam: Let me look that up for you.\\nSam: Can you" + }, + { + "type": "User ID", + "value": "Non-binary", + "confidence": 0.699, + "snippet": "me look that up for you.\\nSam: Here is my Gender: Non-binary\\nLiam: Let me look that up for you.\\nSam: Can you" + }, + { + "type": "Race/Ethnicity", + "value": "Non-binary", + "confidence": 0.633, + "snippet": "me look that up for you.\\nSam: Here is my Gender: Non-binary\\nLiam: Let me look that up for you.\\nSam: Can you" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.93, + "snippet": "Sam: Can you c\"\n },\n {\n \"type\": \"Political Affiliation\",\n \"value\": \"Political Affiliation\",\n " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0045.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0045.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..86889686 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0045.txt.redacted.redaction.json.redacted @@ -0,0 +1,57 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0045.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0045.txt.redacted", + "totalRedactions": 1, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.992, + "snippet": " me look that up for you.\nAlex: Here is my <<>>: <<>>\nSofia: Let me look that up for you.\nAlex: Can you" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.989, + "snippet": "?\nSofia: Thanks, can you also provide your <<>>: <<>>?\nAlex: Can you check on that?\nSofia: Let me look " + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0045.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0045.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..ec55040a --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0045.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,483 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0045.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0045.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.798, + "snippet": "cted\",\n \"totalRedactions\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.74, + "snippet": "cted\",\n \"totalRedactions\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.938, + "snippet": "ons\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.898, + "snippet": "ons\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.969, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.926, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.642, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.563, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.545, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.812, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.764, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.642, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.816, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.843, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.902, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.851, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.557, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.954, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.835, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.788, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.661, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.899, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.925, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.588, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.53, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.833, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.954, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.899, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.983, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.859, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.823, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.98, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.959, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.666, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.677, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.618, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.619, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.607, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.583, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.508, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.719, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.573, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.731, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.706, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.617, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.562, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.644, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.948, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.945, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.545, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.919, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.536, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.817, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.832, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.586, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.932, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.814, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.776, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.943, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.936, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.507, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.789, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.681, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Email\",\n \"value\": \"jordan@example.com\",\n " + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.875, + "snippet": " {\n \"type\": \"Email\",\n \"value\": \"jordan@example.com\",\n \"confidence\": 0.992,\n \"snippet\":" + }, + { + "type": "Username", + "value": "jordan@example.com", + "confidence": 0.554, + "snippet": " {\n \"type\": \"Email\",\n \"value\": \"jordan@example.com\",\n \"confidence\": 0.992,\n \"snippet\":" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.723, + "snippet": "me look that up for you.\\nAlex: Here is my Email: jordan@example.com\\nSofia: Let me look that up for you.\\nAlex: Can y" + }, + { + "type": "Username", + "value": "jordan@example.com", + "confidence": 0.687, + "snippet": "me look that up for you.\\nAlex: Here is my Email: jordan@example.com\\nSofia: Let me look that up for you.\\nAlex: Can y" + }, + { + "type": "Username", + "value": "jordan@example.com", + "confidence": 0.702, + "snippet": " {\n \"type\": \"Email\",\n \"value\": \"jordan@example.com\",\n \"confidence\": 0.989,\n \"snippet\":" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.686, + "snippet": " {\n \"type\": \"Email\",\n \"value\": \"jordan@example.com\",\n \"confidence\": 0.989,\n \"snippet\":" + }, + { + "type": "Username", + "value": "jordan@example.com", + "confidence": 0.732, + "snippet": "\\nSofia: Thanks, can you also provide your Email: jordan@example.com?\\nAlex: Can you check on that?\\nSofia: Let me loo" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.715, + "snippet": "\\nSofia: Thanks, can you also provide your Email: jordan@example.com?\\nAlex: Can you check on that?\\nSofia: Let me loo" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0046.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0046.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..f51c9971 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0046.txt.redacted.redaction.json.redacted @@ -0,0 +1,153 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0046.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0046.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.945, + "snippet": "Transcript #46\n==================================\n<<>>: Can you assist me with updating my subscription." + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.924, + "snippet": " Can you assist me with updating my subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.93, + "snippet": " subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.956, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.906, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.955, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my So" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.938, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: 123-45-6789\nN" + }, + { + "type": "<<>>", + "value": "123-45-6789", + "confidence": 0.998, + "snippet": "for you.\n<<>>: Here is my <<>>: 123-45-6789\n<<>>: Let me look that up for you.\n<<>>: Can you " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.895, + "snippet": "x: Here is my <<>>: 123-45-6789\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.92, + "snippet": "r: 123-45-6789\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.936, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.918, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.94, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.902, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.963, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.918, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.951, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: 0" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": "h: Thanks, can you also provide your <<>>: 00:1B:44:11:3A:B7?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0046.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0046.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..72a57034 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0046.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0046.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0046.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.846, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.81, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.941, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.919, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.949, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.934, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.564, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.791, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.791, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.717, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.768, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.95, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.901, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.877, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.597, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.529, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.938, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.857, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.81, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.705, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.886, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.903, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.565, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.801, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.549, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.96, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.921, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.981, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.824, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.811, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.957, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.615, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.569, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.56, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.543, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.509, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.715, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.509, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.744, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.672, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.634, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.593, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.661, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.941, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.947, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.578, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.919, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.607, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.819, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.795, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.964, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.879, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.82, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.946, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.942, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.583, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.857, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.525, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.87, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.769, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": 0.945,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Alex", + "confidence": 0.505, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": 0.945,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.804, + "snippet": " subscription.\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Noah\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.697, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Noah\",\n \"confidence\": 0.924,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Noah", + "confidence": 0.57, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Noah\",\n \"confidence\": 0.924,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0047.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0047.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..635da48a --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0047.txt.redacted.redaction.json.redacted @@ -0,0 +1,147 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0047.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0047.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.732, + "snippet": "Transcript #47\n==================================\n<<>>: Hi, I need help with a double charge on my card." + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.736, + "snippet": " Hi, I need help with a double charge on my card.\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.766, + "snippet": "ge on my card.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.765, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Union Membersh" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.799, + "snippet": " on that?\n<<>>: Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.969, + "snippet": "anks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.763, + "snippet": "can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.713, + "snippet": "n Membership: None?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.759, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.676, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my A" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.85, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "<<>>", + "value": "San Francisco, CA", + "confidence": 0.608, + "snippet": "r you.\n<<>>: Here is my Approximate Geolocation: San Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Can you" + }, + { + "type": "<<>>", + "value": "San Francisco, CA", + "confidence": 0.551, + "snippet": "r you.\n<<>>: Here is my Approximate Geolocation: San Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Can you" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.655, + "snippet": " is my Approximate Geolocation: San Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.767, + "snippet": " Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.755, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.748, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0047.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0047.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..7b35e147 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0047.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0047.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0047.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.874, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.785, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.949, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.911, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.954, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.945, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.568, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.582, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.795, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.782, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.703, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.774, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.958, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.893, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.871, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.577, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.571, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.936, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.854, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.813, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.691, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.895, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.903, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.53, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.524, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.783, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.533, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.96, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.923, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.819, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.79, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.979, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.956, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.609, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.574, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.57, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.535, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.53, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.723, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.742, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.674, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.616, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.572, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.665, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.941, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.951, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Page views", + "value": "Serial Number", + "confidence": 0.535, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.52, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.908, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.602, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.828, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.805, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.966, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.869, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.816, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.953, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.942, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.869, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.91, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Riley", + "confidence": 0.747, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.732,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.735, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.732,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.891, + "snippet": "ge on my card.\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.799, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.736,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Maya", + "confidence": 0.769, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.736,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0048.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0048.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..6ffe9e4f --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0048.txt.redacted.redaction.json.redacted @@ -0,0 +1,141 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0048.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0048.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.96, + "snippet": "Transcript #48\n==================================\n<<>>: Can you assist me with a double charge on my car" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.96, + "snippet": "an you assist me with a double charge on my card.\n<<>>: Thanks, can you also provide your <<>>" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.505, + "snippet": "n my card.\n<<>>: Thanks, can you also provide your <<>>: Independent?\n<<>>: Can you check on that?\n<<>>: L" + }, + { + "type": "<<>>", + "value": "Independent", + "confidence": 0.971, + "snippet": " can you also provide your <<>>: Independent?\n<<>>: Can you check on that?\n<<>>: Let me look th" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.955, + "snippet": " provide your <<>>: Independent?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.949, + "snippet": "iation: Independent?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.939, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.949, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.937, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.943, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.946, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.935, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.952, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.953, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Pr" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.965, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Private Key: -----BEGIN PRIVATE KEY--" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.96, + "snippet": "ATE KEY----- [redacted] -----END PRIVATE KEY-----\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0048.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0048.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..ea6cfde8 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0048.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,453 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0048.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0048.txt.redacted.redaction.json.redacted", + "totalRedactions": 37, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.874, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.745, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.939, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.925, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.928, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.922, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.989, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.82, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.815, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.67, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Driver's License", + "confidence": 0.6, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.599, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.951, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.895, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.878, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.618, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.591, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.904, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.866, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.82, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.635, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.902, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.913, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.657, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.54, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.794, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.525, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.956, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.93, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.987, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.899, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.976, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.902, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.583, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.577, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.564, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.541, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.539, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.734, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.845, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.739, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Industry", + "value": "Race/Ethnicity", + "confidence": 0.549, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.525, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.514, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.694, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.947, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.954, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Page views", + "value": "Serial Number", + "confidence": 0.546, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.537, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.917, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.587, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.856, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.966, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.82, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.965, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.893, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.807, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.948, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.943, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.586, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.863, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.933, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.616, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": 0.96,\n \"snippet\": " + }, + { + "type": "User ID", + "value": "Alex", + "confidence": 0.55, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": 0.96,\n \"snippet\": " + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.926, + "snippet": "arge on my car\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.633, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.96,\n \"snippet\": " + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.555, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.96,\n \"snippet\": " + }, + { + "type": "Political Affiliation", + "value": "Political Affi", + "confidence": 0.664, + "snippet": " my card.\\nAva: Thanks, can you also provide your Political Affi\"\n },\n {\n \"type\": \"Political Affi" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0049.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0049.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..6c4b478f --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0049.txt.redacted.redaction.json.redacted @@ -0,0 +1,87 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0049.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0049.txt.redacted", + "totalRedactions": 7, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.995, + "snippet": "Thanks, can you also provide your <<>>: <<>>?\nAlex: Can you check on that?\nNoah: Let me look t" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.861, + "snippet": ", can you also provide your <<>>: <<>>?\nAlex: Here is my <<>>: <<>>" + }, + { + "type": "<<>>", + "value": "<<>>6789", + "confidence": 0.999, + "snippet": "istered?\nAlex: Here is my <<>>: <<>>6789\nNoah: Let me look that up for you.\nAlex: Can you " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.611, + "snippet": " on that?\nNoah: Thanks, can you also provide your <<>>: None?\nAlex: Can you check on that?\nNoah: Thanks," + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.933, + "snippet": "Noah: Thanks, can you also provide your <<>>: None?\nAlex: Can you check on that?\nNoah: Thanks, can y" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.539, + "snippet": " on that?\nNoah: Thanks, can you also provide your <<>>: Non-binary?" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.978, + "snippet": "?\nNoah: Thanks, can you also provide your <<>>: Non-binary?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0049.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0049.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..40a1a088 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0049.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,501 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0049.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0049.txt.redacted.redaction.json.redacted", + "totalRedactions": 37, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.806, + "snippet": "cted\",\n \"totalRedactions\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.788, + "snippet": "cted\",\n \"totalRedactions\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.948, + "snippet": "ons\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Religion", + "value": "Biometric Identifier", + "confidence": 0.63, + "snippet": "ons\": 7,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.968, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.904, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.985, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.812, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.752, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.564, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.724, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.948, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.899, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.669, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.867, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.701, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.67, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Race/Ethnicity", + "value": "Industry", + "confidence": 0.928, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.893, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.875, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.836, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.582, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.88, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.916, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.653, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.808, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.624, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.958, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.937, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.643, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.982, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.838, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.79, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.969, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.947, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.798, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Religion", + "value": "Profile Picture Image", + "confidence": 0.544, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.53, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.741, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.657, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.605, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.664, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.582, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.587, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.798, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Religion", + "confidence": 0.531, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.501, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.945, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.958, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.908, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.596, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.964, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.859, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.729, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.871, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.891, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.857, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.922, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.567, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Gender", + "value": "User Preferences", + "confidence": 0.525, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.868, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.525, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.544, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidenc" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.889, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.995,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.731, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.995,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.875, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\\nAlex: Can you check on that?\\nNoah: Let me look" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.758, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\\nAlex: Can you check on that?\\nNoah: Let me look" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.818, + "snippet": " Let me look t\"\n },\n {\n \"type\": \"Biometric Identifier\",\n \"value\": \"FaceID registered\",\n \"" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.894, + "snippet": "\"type\": \"Biometric Identifier\",\n \"value\": \"FaceID registered\",\n \"confidence\": 0.861,\n \"snippet\":" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.642, + "snippet": ",\n \"snippet\": \", can you also provide your Biometric Identifier: FaceID registered?\\nAlex: Here is my Social Secu" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.904, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\\nAlex: Here is my Social Security Number: 123-45" + }, + { + "type": "User Preferences", + "value": "FaceID registered", + "confidence": 0.581, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\\nAlex: Here is my Social Security Number: 123-45" + }, + { + "type": "Political Affiliation", + "value": "123-45-", + "confidence": 0.737, + "snippet": "stered?\\nAlex: Here is my Social Security Number: 123-45-\"\n },\n {\n \"type\": \"Social Securit" + }, + { + "type": "Passport Number", + "value": "123-45-", + "confidence": 0.505, + "snippet": "stered?\\nAlex: Here is my Social Security Number: 123-45-\"\n },\n {\n \"type\": \"Social Securit" + }, + { + "type": "Social Security Number", + "value": "123-45-6789", + "confidence": 0.772, + "snippet": "ype\": \"Social Security Number\",\n \"value\": \"123-45-6789\",\n \"confidence\": 0.999,\n \"snippet\":" + }, + { + "type": "Political Affiliation", + "value": "123-45-6789", + "confidence": 0.748, + "snippet": "ype\": \"Social Security Number\",\n \"value\": \"123-45-6789\",\n \"confidence\": 0.999,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0050.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0050.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..e2509d18 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0050.txt.redacted.redaction.json.redacted @@ -0,0 +1,111 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0050.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0050.txt.redacted", + "totalRedactions": 2, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.885, + "snippet": "Transcript #50\n==================================\n<<>>: Good morning, question about firmware update err" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.774, + "snippet": "d morning, question about firmware update errors.\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.883, + "snippet": "update errors.\n<<>>: Let me look that up for you.\n<<>>: Here is my Cookies: sessionid=abcd1234\n<<>>: Let" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.76, + "snippet": "u.\n<<>>: Here is my Cookies: sessionid=abcd1234\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.836, + "snippet": "ionid=abcd1234\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.775, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.839, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.814, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.789, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.852, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.826, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0050.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0050.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..7dce59d4 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0050.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,453 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0050.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0050.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.865, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.827, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.951, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.864, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.962, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.933, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.574, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.823, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.78, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.764, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.798, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.951, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.898, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.879, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.685, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.946, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.533, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.879, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.834, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.774, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.884, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.893, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.552, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.795, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.965, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.932, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.978, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.858, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.814, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.966, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.527, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.953, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.672, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.54, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.598, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.562, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.532, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.528, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.721, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.687, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.622, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.542, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.661, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.702, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.556, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.933, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.946, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.57, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.925, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.696, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.817, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.964, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.82, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.577, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.958, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.827, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.785, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.954, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.935, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.548, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.858, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.877, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\"" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.812, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.885,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Jordan", + "confidence": 0.592, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.885,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.778, + "snippet": "are update err\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.706, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.774,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Maya", + "confidence": 0.652, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.774,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0051.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0051.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..95beb36c --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0051.txt.redacted.redaction.json.redacted @@ -0,0 +1,129 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0051.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0051.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.71, + "snippet": "Transcript #51\n==================================\n<<>>: Good morning, question about a late delivery.\nMa" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.598, + "snippet": "ex: Good morning, question about a late delivery.\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.665, + "snippet": "late delivery.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.546, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.57, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.652, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.534, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.614, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: U1234" + }, + { + "type": "<<>>", + "value": "U123456", + "confidence": 0.963, + "snippet": "\n<<>>: Thanks, can you also provide your <<>>: U123456?\n<<>>: Can you check on that?\n<<>>: Let me look t" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.628, + "snippet": "anks, can you also provide your <<>>: U123456?\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.535, + "snippet": "ur <<>>: U123456?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.553, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.637, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.539, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0051.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0051.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..c6c6c912 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0051.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,441 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0051.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0051.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.853, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.799, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.94, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.898, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.95, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.938, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.642, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.544, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.523, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.78, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.759, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.715, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.765, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.95, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.899, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.874, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.593, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.551, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.941, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.856, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.806, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.655, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.889, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.898, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.787, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.963, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.92, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.98, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.822, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.796, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.975, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.957, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.531, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.616, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.573, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.572, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.545, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.518, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.716, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.773, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.665, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.588, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.587, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.658, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.939, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.949, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.594, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.911, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.606, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.819, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.964, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.799, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.517, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.951, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.852, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.808, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.944, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.939, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.862, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.812, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Alex", + "confidence": 0.87, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": 0.71,\n \"snippet\": " + }, + { + "type": "User ID", + "value": "Alex", + "confidence": 0.716, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Alex\",\n \"confidence\": 0.71,\n \"snippet\": " + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.732, + "snippet": " delivery.\\nMa\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.814, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.598,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Maya", + "confidence": 0.679, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.598,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0052.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0052.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..91fdf59d --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0052.txt.redacted.redaction.json.redacted @@ -0,0 +1,147 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0052.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0052.txt.redacted", + "totalRedactions": 6, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.605, + "snippet": "Transcript #52\n==================================\n<<>>: Hi, I need help with updating my subscription.\nA" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.642, + "snippet": "r: Hi, I need help with updating my subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.545, + "snippet": "y subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.585, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.546, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.592, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Sexual Orienta" + }, + { + "type": "<<>>", + "value": "Heterosexual", + "confidence": 0.991, + "snippet": "ks, can you also provide your <<>>: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.503, + "snippet": "so provide your <<>>: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.573, + "snippet": "ion: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Cookies: sessi" + }, + { + "type": "<<>>", + "value": "sessionid=abcd1234", + "confidence": 0.508, + "snippet": "?\n<<>>: Thanks, can you also provide your Cookies: sessionid=abcd1234?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.539, + "snippet": "ou also provide your Cookies: sessionid=abcd1234?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.577, + "snippet": "essionid=abcd1234?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: U1234" + }, + { + "type": "<<>>", + "value": "U123456", + "confidence": 0.985, + "snippet": "?\n<<>>: Thanks, can you also provide your <<>>: U123456?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.528, + "snippet": "anks, can you also provide your <<>>: U123456?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.575, + "snippet": " <<>>: U123456?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: jordan@" + }, + { + "type": "<<>>", + "value": "jordan@example.com", + "confidence": 0.998, + "snippet": "at?\n<<>>: Thanks, can you also provide your <<>>: jordan@example.com?\n<<>>: Can you check on that?" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.552, + "snippet": " you also provide your <<>>: jordan@example.com?\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0052.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0052.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..90d5abfc --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0052.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,435 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0052.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0052.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.865, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.797, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.945, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.915, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.954, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.941, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.697, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.57, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.791, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.781, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.699, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.768, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.948, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.895, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.876, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.588, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.539, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.933, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.521, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.846, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.8, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.691, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.886, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.905, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.574, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.779, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.519, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.963, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.925, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.829, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.72, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.975, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.952, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.619, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.58, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.572, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.54, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.517, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.725, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.509, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.677, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.676, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.624, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.6, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.669, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.937, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.945, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.517, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.912, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.586, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.822, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.799, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.972, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.876, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.801, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.946, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.939, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.859, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.93, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "User ID", + "value": "Taylor", + "confidence": 0.757, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.605,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.921, + "snippet": "bscription.\\nA\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.765, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.642,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0053.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0053.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..793f1ee8 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0053.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0053.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0053.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.791, + "snippet": "password.\nNoah: Thanks, can you also provide your <<>>: <<>>?\nJordan: Can you check on that?\n" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.909, + "snippet": "ks, can you also provide your <<>>: <<>>?\nJordan: Can you check on that?\nNoah: Let me look" + }, + { + "type": "<<>>", + "value": "FaceID registered", + "confidence": 0.88, + "snippet": ", can you also provide your <<>>: FaceID registered?\nJordan: Can you check on that?\nNoah: Let me look" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0053.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0053.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..dac8a283 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0053.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,459 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0053.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0053.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.798, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.71, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.768, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.708, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Race/Ethnicity", + "value": "Biometric Identifier", + "confidence": 0.641, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.963, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.928, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.988, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.833, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.811, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Sexual Orientation", + "value": "Driver's License", + "confidence": 0.64, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.959, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.927, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.852, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.734, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.981, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.912, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Income", + "value": "Interaction Events", + "confidence": 0.63, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.883, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.923, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.815, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.962, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.946, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.69, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.966, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.888, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.55, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.975, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.617, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.963, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.761, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.603, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.614, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.751, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.714, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.527, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.704, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.537, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.857, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.795, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Industry", + "value": "Race/Ethnicity", + "confidence": 0.51, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.717, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.775, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.922, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.955, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.825, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.815, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.561, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.87, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.951, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.825, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.67, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.929, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.715, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.557, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.934, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.768, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.741, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.647, + "snippet": " \"type\": \"Declared Interests\",\n \"value\": \"Declared Interests\",\n \"confidence\": 0.791,\n \"snippet\":" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.598, + "snippet": "assword.\\nNoah: Thanks, can you also provide your Declared Interests: hiking, cooking?\\nJordan: Can you check on that?" + }, + { + "type": "Industry", + "value": "hiking, cooking", + "confidence": 0.936, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\\nJordan: Can you check on that?\\n\"\n },\n " + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.632, + "snippet": "eck on that?\\n\"\n },\n {\n \"type\": \"Declared Interests\",\n \"value\": \"hiking, cooking\",\n \"co" + }, + { + "type": "Industry", + "value": "hiking, cooking", + "confidence": 0.938, + "snippet": " \"type\": \"Declared Interests\",\n \"value\": \"hiking, cooking\",\n \"confidence\": 0.909,\n \"snippet\":" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.522, + "snippet": " \"snippet\": \"ks, can you also provide your Declared Interests: hiking, cooking?\\nJordan: Can you check on that?" + }, + { + "type": "Industry", + "value": "hiking, cooking", + "confidence": 0.939, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\\nJordan: Can you check on that?\\nNoah: Let me lo" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.829, + "snippet": "h: Let me look\"\n },\n {\n \"type\": \"Biometric Identifier\",\n \"value\": \"FaceID registered\",\n \"" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.789, + "snippet": "h: Let me look\"\n },\n {\n \"type\": \"Biometric Identifier\",\n \"value\": \"FaceID registered\",\n \"" + }, + { + "type": "Race/Ethnicity", + "value": "Biometric Identifier", + "confidence": 0.73, + "snippet": "h: Let me look\"\n },\n {\n \"type\": \"Biometric Identifier\",\n \"value\": \"FaceID registered\",\n \"" + }, + { + "type": "Declared Interests", + "value": "Biometric Identifier", + "confidence": 0.61, + "snippet": "h: Let me look\"\n },\n {\n \"type\": \"Biometric Identifier\",\n \"value\": \"FaceID registered\",\n \"" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0054.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0054.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..2b498a61 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0054.txt.redacted.redaction.json.redacted @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0054.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0054.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Profile Picture Image", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.545, + "snippet": "on that?\nSofia: Thanks, can you also provide your <<>>: marketing_emails=false?\nRiley: Can you check on " + }, + { + "type": "<<>>", + "value": "marketing_emails=false", + "confidence": 0.938, + "snippet": "anks, can you also provide your <<>>: marketing_emails=false?\nRiley: Can you check on that?\nSofia: Let me look" + }, + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.97, + "snippet": "ia: Thanks, can you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nRiley: Can you check" + }, + { + "type": "<<>>", + "value": "Macintosh; Intel Mac OS X", + "confidence": 0.523, + "snippet": "an you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nRiley: Can you check on that?\nSofia: Let me loo" + }, + { + "type": "<<>>", + "value": "+1-202-555-0172", + "confidence": 0.995, + "snippet": "me look that up for you.\nRiley: Here is my <<>>: +1-202-555-0172\nSofia: Thanks, can you also provide your Private " + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0054.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0054.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..deeb5c6b --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0054.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,411 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0054.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0054.txt.redacted.redaction.json.redacted", + "totalRedactions": 33, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.846, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.779, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.858, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.934, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.93, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.554, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.784, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.714, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.592, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.762, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.915, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.897, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.87, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.624, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.568, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.959, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.845, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.829, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.542, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.875, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.897, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.778, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.943, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.928, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.642, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.978, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.798, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.771, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.971, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.956, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.644, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.64, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.508, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.693, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.569, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.502, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.605, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.582, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.939, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.952, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.903, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.521, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.838, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.956, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.831, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.551, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.948, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.571, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.565, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.975, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.811, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.784, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Preferences\",\n \"value\": \"User Preferences\",\n \"c" + }, + { + "type": "User Agent", + "value": "User Preferences", + "confidence": 0.567, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Preferences\",\n \"value\": \"User Preferences\",\n \"c" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.913, + "snippet": " \"type\": \"User Preferences\",\n \"value\": \"User Preferences\",\n \"confidence\": 0.545,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Preferences", + "confidence": 0.597, + "snippet": " \"type\": \"User Preferences\",\n \"value\": \"User Preferences\",\n \"confidence\": 0.545,\n \"snippet\":" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.851, + "snippet": "n that?\\nSofia: Thanks, can you also provide your User Preferences: marketing_emails=false?\\nRiley: Can you check on" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.864, + "snippet": " you check on \"\n },\n {\n \"type\": \"User Preferences\",\n \"value\": \"marketing_emails=false\",\n " + }, + { + "type": "User Agent", + "value": "User Preferences", + "confidence": 0.593, + "snippet": " you check on \"\n },\n {\n \"type\": \"User Preferences\",\n \"value\": \"marketing_emails=false\",\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.896, + "snippet": " \"snippet\": \"anks, can you also provide your User Preferences: marketing_emails=false?\\nRiley: Can you check on" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.934, + "snippet": "a: Let me look\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Mozilla/5.0\",\n \"confid" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0055.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0055.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..3726b25c --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0055.txt.redacted.redaction.json.redacted @@ -0,0 +1,57 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0055.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0055.txt.redacted", + "totalRedactions": 2, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Profile Picture Image", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.947, + "snippet": ", can you also provide your <<>>: <<>> registered?\nSam: Can you check on that?\nEthan: Le" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.987, + "snippet": "me look that up for you.\nSam: Here is my <<>>: <<>>\nEthan: Let me look that up for you.\nSam: Can you " + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0055.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0055.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..bdb3d23d --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0055.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,471 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0055.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0055.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.815, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.797, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.914, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User ID", + "value": "Biometric Identifier", + "confidence": 0.855, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.947, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.905, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.986, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.819, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.786, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.639, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.819, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.932, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.916, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.702, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.852, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.688, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.945, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.885, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.831, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.615, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.901, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.937, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.669, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.522, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.823, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.666, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.973, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.931, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.978, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.815, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.807, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.979, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.948, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.504, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.679, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.677, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.513, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.735, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.829, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.617, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.57, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.503, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.638, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.94, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.949, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.857, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Biometric Identifier", + "value": "Sexual Orientation", + "confidence": 0.62, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.837, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.816, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.539, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.929, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.921, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.769, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User ID", + "confidence": 0.52, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.949, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.9, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.577, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.784, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Biometric Identifier\",\n \"value\": \"FaceID\",\n \"confidence\"" + }, + { + "type": "User ID", + "value": "Biometric Identifier", + "confidence": 0.661, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Biometric Identifier\",\n \"value\": \"FaceID\",\n \"confidence\"" + }, + { + "type": "Biometric Identifier", + "value": "FaceID", + "confidence": 0.875, + "snippet": "\"type\": \"Biometric Identifier\",\n \"value\": \"FaceID\",\n \"confidence\": 0.947,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "FaceID", + "confidence": 0.862, + "snippet": "\"type\": \"Biometric Identifier\",\n \"value\": \"FaceID\",\n \"confidence\": 0.947,\n \"snippet\":" + }, + { + "type": "Biometric Identifier", + "value": "FaceID", + "confidence": 0.919, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\\nSam: Can you check on that?\\nEthan: " + }, + { + "type": "User ID", + "value": "FaceID", + "confidence": 0.734, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\\nSam: Can you check on that?\\nEthan: " + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.672, + "snippet": "at?\\nEthan: Le\"\n },\n {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.56, + "snippet": "at?\\nEthan: Le\"\n },\n {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.777, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence\": 0.987,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "U123456", + "confidence": 0.74, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence\": 0.987,\n \"snippet\":" + }, + { + "type": "Biometric Identifier", + "value": "U123456", + "confidence": 0.589, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence\": 0.987,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "U123456", + "confidence": 0.713, + "snippet": "e look that up for you.\\nSam: Here is my User ID: U123456\\nEthan: Let me look that up for you.\\nSam: Can yo" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.56, + "snippet": "e look that up for you.\\nSam: Here is my User ID: U123456\\nEthan: Let me look that up for you.\\nSam: Can yo" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0056.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0056.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..74611d8f --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0056.txt.redacted.redaction.json.redacted @@ -0,0 +1,129 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0056.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0056.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.584, + "snippet": "Transcript #56\n==================================\n<<>>: Hello—I'm having trouble with a late delivery.\nL" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.626, + "snippet": "y: Hello—I'm having trouble with a late delivery.\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.56, + "snippet": "late delivery.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.684, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.53, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.635, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>" + }, + { + "type": "<<>>", + "value": "Hispanic", + "confidence": 0.992, + "snippet": "Thanks, can you also provide your <<>>: Hispanic?\n<<>>: Can you check on that?\n<<>>: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.575, + "snippet": "n you also provide your <<>>: Hispanic?\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.54, + "snippet": "thnicity: Hispanic?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.686, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.57, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.544, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.682, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.529, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0056.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0056.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..767d897c --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0056.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,441 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0056.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0056.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.86, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.785, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.939, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.878, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.942, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.938, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.574, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.788, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.78, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.691, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.733, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.946, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.903, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.869, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.597, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.586, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.944, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.853, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.809, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.622, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.884, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.895, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.569, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.795, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.571, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.958, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.917, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.98, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.841, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.78, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.973, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.502, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.953, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.618, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.519, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.568, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.563, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.556, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.554, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.702, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.733, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.657, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.631, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.609, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.667, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.94, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.945, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.556, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.907, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.623, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.815, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.957, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.801, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.97, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.895, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.81, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.947, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.941, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.864, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.852, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Riley", + "confidence": 0.767, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.584,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.764, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.584,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.811, + "snippet": "e delivery.\\nL\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": " + }, + { + "type": "User ID", + "value": "Liam", + "confidence": 0.749, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": 0.626,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.705, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": 0.626,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0057.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0057.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..c2e1d179 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0057.txt.redacted.redaction.json.redacted @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0057.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0057.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.982, + "snippet": " can you also provide your <<>>: <<>>?\nRiley: Can you check on that?\nLiam: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.975, + "snippet": "ks, can you also provide your <<>>: <<>>?\nRiley: Can you check on that?\nLiam: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.978, + "snippet": "Thanks, can you also provide your <<>>: <<>>?\nRiley: Here is my <<>>: Heterosexu" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.975, + "snippet": ": <<>>?\nRiley: Here is my <<>>: <<>>\nLiam: Let me look that up for you.\nRiley: Can you" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0057.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0057.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..f0bb4f3d --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0057.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,465 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0057.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0057.txt.redacted.redaction.json.redacted", + "totalRedactions": 37, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.809, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.703, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.936, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.884, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.963, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.952, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.733, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.993, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.523, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.832, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.566, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.815, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.962, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.532, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.877, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.67, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Biometric Identifier", + "value": "Gender", + "confidence": 0.659, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.887, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.812, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.626, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.898, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.871, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.901, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.936, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.834, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.786, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.965, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.948, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.541, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.99, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.947, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.978, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.589, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.584, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.917, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.581, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.761, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.694, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.66, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.57, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.789, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.784, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.657, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.711, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.949, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.952, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.952, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.866, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.946, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.877, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.536, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.925, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.718, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.692, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.926, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.869, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Gender", + "value": "User Preferences", + "confidence": 0.518, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.778, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.918, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Political Affiliation\",\n \"value\": \"Independent\",\n \"confid" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.815, + "snippet": "2,\n \"snippet\": \" can you also provide your Political Affiliation: Independent?\\nRiley: Can you check on that?\\nLia" + }, + { + "type": "Region", + "value": "Political Affiliation", + "confidence": 0.513, + "snippet": "2,\n \"snippet\": \" can you also provide your Political Affiliation: Independent?\\nRiley: Can you check on that?\\nLia" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.634, + "snippet": " can you also provide your Political Affiliation: Independent?\\nRiley: Can you check on that?\\nLiam: Let me loo" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.718, + "snippet": ": Let me look \"\n },\n {\n \"type\": \"Sexual Orientation\",\n \"value\": \"Heterosexual\",\n \"confi" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.74, + "snippet": " \"type\": \"Sexual Orientation\",\n \"value\": \"Heterosexual\",\n \"confidence\": 0.975,\n \"snippet\":" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.681, + "snippet": " \"snippet\": \"ks, can you also provide your Sexual Orientation: Heterosexual?\\nRiley: Can you check on that?\\nLi" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.841, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\\nRiley: Can you check on that?\\nLiam: Thanks, ca" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.752, + "snippet": ": Thanks, can \"\n },\n {\n \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidenc" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.734, + "snippet": ": Thanks, can \"\n },\n {\n \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidenc" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.745, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.978,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.6, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.978,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0058.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0058.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..d0a88811 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0058.txt.redacted.redaction.json.redacted @@ -0,0 +1,117 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0058.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0058.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.693, + "snippet": "Transcript #58\n==================================\n<<>>: Hello—I'm having trouble with firmware update er" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.712, + "snippet": "o—I'm having trouble with firmware update errors.\n<<>>: Thanks, can you also provide your <<>>: h" + }, + { + "type": "<<>>", + "value": "https://social.example.com/jordan", + "confidence": 0.98, + "snippet": "n: Thanks, can you also provide your <<>>: https://social.example.com/jordan?\n<<>>: Can you check on that?\n<<>>: Let me look" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.664, + "snippet": "r <<>>: https://social.example.com/jordan?\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.693, + "snippet": "example.com/jordan?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my R" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.663, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: Hispanic\n<<>>: Thank" + }, + { + "type": "<<>>", + "value": "Hispanic", + "confidence": 0.997, + "snippet": "hat up for you.\n<<>>: Here is my <<>>: Hispanic\n<<>>: Thanks, can you also provide your MAC Addr" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.669, + "snippet": "r you.\n<<>>: Here is my <<>>: Hispanic\n<<>>: Thanks, can you also provide your <<>>: 0" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.997, + "snippet": "n: Thanks, can you also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.716, + "snippet": "also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.703, + "snippet": " 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.648, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0058.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0058.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..3a4d60e7 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0058.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,483 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0058.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0058.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.878, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.811, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.954, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.851, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.962, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.919, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.992, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.511, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.834, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.781, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.736, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.827, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.949, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.533, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.898, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.867, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.629, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.936, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.527, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.861, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.84, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.759, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.902, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.912, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.636, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.806, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.967, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.934, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.848, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.84, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.97, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.58, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.953, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.642, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.696, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.643, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.599, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.525, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.515, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.502, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.733, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.698, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.584, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.503, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.613, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.713, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.554, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.514, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.934, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.949, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.521, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.918, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.688, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.827, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.837, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.524, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.976, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.831, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.771, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.957, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.94, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.525, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.864, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.81, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Riley", + "confidence": 0.705, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.693,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.642, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.693,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.71, + "snippet": "ware update er\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Ethan", + "confidence": 0.694, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\": 0.712,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Ethan", + "confidence": 0.58, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\": 0.712,\n \"snippet\":" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.582, + "snippet": "errors.\\nEthan: Thanks, can you also provide your Profile URL: h\"\n },\n {\n \"type\": \"Profile URL" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.574, + "snippet": "errors.\\nEthan: Thanks, can you also provide your Profile URL: h\"\n },\n {\n \"type\": \"Profile URL" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0059.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0059.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..065d1d09 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0059.txt.redacted.redaction.json.redacted @@ -0,0 +1,159 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0059.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0059.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.929, + "snippet": "Transcript #59\n==================================\n<<>>: Hi, I need help with updating my subscription.\nA" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.911, + "snippet": "r: Hi, I need help with updating my subscription.\n<<>>: Thanks, can you also provide your Approximate Ge" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.905, + "snippet": " your Approximate Geolocation: San Francisco, CA?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.882, + "snippet": "San Francisco, CA?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.912, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.863, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.919, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: marketing_emails=fa" + }, + { + "type": "<<>>", + "value": "marketing_emails=false", + "confidence": 0.941, + "snippet": " up for you.\n<<>>: Here is my <<>>: marketing_emails=false\n<<>>: Let me look that up for you.\n<<>>: Can you" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.793, + "snippet": "re is my <<>>: marketing_emails=false\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.865, + "snippet": "ng_emails=false\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.885, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.863, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.855, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.887, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.855, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.928, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: L" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.512, + "snippet": ": Let me look that up for you.\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: Let me look that up for y" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.999, + "snippet": " that up for you.\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: Let me look that up for you." + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.877, + "snippet": "<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0059.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0059.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..65c539c4 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0059.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,441 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0059.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0059.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.863, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.804, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.949, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.916, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.953, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.943, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.698, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.512, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.784, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.783, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.686, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.764, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.945, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.899, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.871, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.57, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.555, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.936, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.845, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.802, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.673, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.886, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.902, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.568, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.789, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.522, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.961, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.924, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.98, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.833, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.717, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.975, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.605, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.584, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.584, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.552, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.523, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.725, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.701, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.679, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.63, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.594, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.674, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.936, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.943, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.502, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.913, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.608, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.821, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.806, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.971, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.883, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.808, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.95, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.941, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.511, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.858, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.506, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.929, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "User ID", + "value": "Taylor", + "confidence": 0.717, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.929,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.511, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.929,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.914, + "snippet": "bscription.\\nA\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.712, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.911,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0060.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0060.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..bbccc745 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0060.txt.redacted.redaction.json.redacted @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0060.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0060.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Serial Number", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "Serial Number", + "value": "SN-1234567890", + "confidence": 0.998, + "snippet": " Thanks, can you also provide your Serial Number: SN-1234567890?\nJordan: Here is my <<>>: <<>>\nEthan:" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.993, + "snippet": " Number: SN-1234567890?\nJordan: Here is my <<>>: <<>>\nEthan: Let me look that up for you.\nJordan: Can y" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.527, + "snippet": "on that?\nEthan: Thanks, can you also provide your <<>>: hiking, cooking?\nJordan: Can you check on that?\n" + }, + { + "type": "<<>>", + "value": "hiking, cooking", + "confidence": 0.94, + "snippet": "ks, can you also provide your <<>>: hiking, cooking?\nJordan: Can you check on that?\nEthan: Let me loo" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0060.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0060.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..84079eaa --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0060.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,411 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0060.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0060.txt.redacted.redaction.json.redacted", + "totalRedactions": 34, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.73, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.686, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.934, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.891, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Declared Interests", + "value": "Biometric Identifier", + "confidence": 0.79, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.931, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.801, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.987, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.872, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.841, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.943, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.583, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.912, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.912, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.736, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.7, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.971, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.928, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Income", + "value": "Interaction Events", + "confidence": 0.656, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.848, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.946, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.789, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.835, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.973, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.941, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.659, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.843, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.819, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.861, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.952, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.515, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.631, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.58, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.696, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.668, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.736, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.845, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.747, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.65, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.512, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.69, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.976, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.878, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.854, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.552, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.828, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.963, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.811, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.611, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.942, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.782, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.777, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.929, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.879, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.696, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.751, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.809, + "snippet": "Number: SN-1234567890?\\nJordan: Here is my Phone: +1-202-555-0172\\nEthan:\"\n },\n {\n \"type\": \"Phone\"" + }, + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.534, + "snippet": " {\n \"type\": \"Phone\",\n \"value\": \"+1-202-555-0172\",\n \"confidence\": 0.993,\n \"snippet\":" + }, + { + "type": "Phone", + "value": "+1-202-555-0172", + "confidence": 0.789, + "snippet": "Number: SN-1234567890?\\nJordan: Here is my Phone: +1-202-555-0172\\nEthan: Let me look that up for you.\\nJordan: Can" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.698, + "snippet": " \"type\": \"Declared Interests\",\n \"value\": \"Declared Interests\",\n \"confidence\": 0.527,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0061.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0061.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..e87d30dd --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0061.txt.redacted.redaction.json.redacted @@ -0,0 +1,159 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0061.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0061.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.722, + "snippet": "Transcript #61\n==================================\n<<>>: Hi, I need help with logging into my account.\nSo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.632, + "snippet": "or: Hi, I need help with logging into my account.\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.689, + "snippet": "o my account.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.622, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.629, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: 1990-05-12\n<<>>: Let " + }, + { + "type": "<<>>", + "value": "1990-05-12", + "confidence": 0.997, + "snippet": "hat up for you.\n<<>>: Here is my <<>>: 1990-05-12\n<<>>: Let me look that up for you.\n<<>>: Can y" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.594, + "snippet": "you.\n<<>>: Here is my <<>>: 1990-05-12\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.58, + "snippet": "h: 1990-05-12\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you al" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.672, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: None" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.986, + "snippet": "on that?\n<<>>: Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let " + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.979, + "snippet": "ofia: Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me loo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.575, + "snippet": "Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.581, + "snippet": "ur <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.629, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.538, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.63, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.582, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.601, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.615, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0061.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0061.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..9e4584c7 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0061.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,423 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0061.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0061.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.831, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.812, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.946, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.907, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.953, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.938, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.66, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.538, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.783, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.773, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.713, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.772, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.95, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.896, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.872, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.593, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.559, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.94, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.848, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.805, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.664, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.889, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.907, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.563, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.786, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.522, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.96, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.923, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.98, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.821, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.773, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.976, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.956, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.616, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.595, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.583, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.572, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.723, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.739, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.665, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.628, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.595, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.656, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.938, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.949, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.916, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.559, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.819, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.806, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.96, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.859, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.816, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.948, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.94, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.862, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.899, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.627, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.722,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Taylor", + "confidence": 0.596, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.722,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.876, + "snippet": "y account.\\nSo\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Sofia\",\n \"confidence\":" + }, + { + "type": "User Agent", + "value": "Sofia", + "confidence": 0.677, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sofia\",\n \"confidence\": 0.632,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Sofia", + "confidence": 0.642, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sofia\",\n \"confidence\": 0.632,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0062.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0062.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..5d6bd939 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0062.txt.redacted.redaction.json.redacted @@ -0,0 +1,117 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0062.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0062.txt.redacted", + "totalRedactions": 2, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.952, + "snippet": "Transcript #62\n==================================\n<<>>: Can you assist me with a late delivery.\n<<>>: L" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.916, + "snippet": "==\n<<>>: Can you assist me with a late delivery.\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.949, + "snippet": "ate delivery.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.952, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.921, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.949, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.939, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.919, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.952, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.929, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.94, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.954, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0062.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0062.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..7f1044df --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0062.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,423 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0062.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0062.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.863, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.783, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.943, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.884, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.944, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.939, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.664, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.512, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.782, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.753, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.704, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.739, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.949, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.896, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.862, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.577, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.526, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.941, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.845, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.794, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.674, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.893, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.905, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.526, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.793, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.539, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.963, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.922, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.806, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.787, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.976, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.53, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.604, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.578, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.571, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.553, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.717, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.791, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.67, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.544, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.582, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.659, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.938, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.945, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.603, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.901, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.565, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.823, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.959, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.805, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.973, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.869, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.775, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.945, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.936, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.857, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.868, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Riley", + "confidence": 0.601, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.952,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.856, + "snippet": "ery.\\nEthan: L\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Ethan", + "confidence": 0.707, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\": 0.916,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0063.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0063.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..833b0ebe --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0063.txt.redacted.redaction.json.redacted @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0063.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0063.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.989, + "snippet": " can you also provide your <<>>: <<>>?\nCasey: Can you check on that?\nNoah: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.98, + "snippet": "anks, can you also provide your <<>>: <<>>?\nCasey: Here is my <<>>: <<>>\nNoah" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.999, + "snippet": "embership: <<>>?\nCasey: Here is my <<>>: <<>>\nNoah: Thanks, can you also provide your Private K" + }, + { + "type": "<<>>", + "value": "SN-1234567890", + "confidence": 0.99, + "snippet": " Thanks, can you also provide your <<>>: SN-1234567890?\nCasey: Can you check on that?\nNoah: Let me look " + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0063.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0063.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..588eba86 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0063.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,489 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0063.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0063.txt.redacted.redaction.json.redacted", + "totalRedactions": 37, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.79, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.712, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.941, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.933, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Declared Interests", + "value": "Biometric Identifier", + "confidence": 0.559, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.943, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.898, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.533, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.797, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.877, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.848, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.727, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Sexual Orientation", + "value": "Driver's License", + "confidence": 0.555, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.943, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.53, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.895, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.876, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.743, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.569, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.692, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.915, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.862, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.673, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Income", + "value": "Interaction Events", + "confidence": 0.636, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.811, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.928, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.864, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.828, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.966, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.932, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.534, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.991, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.934, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.981, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.696, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.509, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.866, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.552, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.639, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.584, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.74, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.58, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.699, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.698, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.664, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.594, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.747, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.541, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.951, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Passport Number", + "value": "Routing Number", + "confidence": 0.52, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.96, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.903, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.785, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.887, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.949, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.953, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.87, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.821, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.809, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.937, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.912, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.818, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.845, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.901, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Political Affiliation\",\n \"value\": \"Independent\",\n \"confid" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.691, + "snippet": "9,\n \"snippet\": \" can you also provide your Political Affiliation: Independent?\\nCasey: Can you check on that?\\nNoa" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.694, + "snippet": " can you also provide your Political Affiliation: Independent?\\nCasey: Can you check on that?\\nNoah: Thanks, ca" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.939, + "snippet": ": Thanks, can \"\n },\n {\n \"type\": \"Union Membership\",\n \"value\": \"None\",\n \"confidence\": " + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.933, + "snippet": " \"snippet\": \"anks, can you also provide your Union Membership: None?\\nCasey: Here is my Date of Birth: 1990-05-" + }, + { + "type": "Industry", + "value": "None", + "confidence": 0.572, + "snippet": "anks, can you also provide your Union Membership: None?\\nCasey: Here is my Date of Birth: 1990-05-12\\nNo" + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.896, + "snippet": "mbership: None?\\nCasey: Here is my Date of Birth: 1990-05-12\\nNoah\"\n },\n {\n \"type\": \"Date of " + }, + { + "type": "Credit Card Number", + "value": "1990-05-12", + "confidence": 0.7, + "snippet": "mbership: None?\\nCasey: Here is my Date of Birth: 1990-05-12\\nNoah\"\n },\n {\n \"type\": \"Date of " + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.929, + "snippet": " \"type\": \"Date of Birth\",\n \"value\": \"1990-05-12\",\n \"confidence\": 0.999,\n \"snippet\":" + }, + { + "type": "Credit Card Number", + "value": "1990-05-12", + "confidence": 0.833, + "snippet": " \"type\": \"Date of Birth\",\n \"value\": \"1990-05-12\",\n \"confidence\": 0.999,\n \"snippet\":" + }, + { + "type": "Phone", + "value": "1990-05-12", + "confidence": 0.668, + "snippet": " \"type\": \"Date of Birth\",\n \"value\": \"1990-05-12\",\n \"confidence\": 0.999,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0064.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0064.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..b461b96f --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0064.txt.redacted.redaction.json.redacted @@ -0,0 +1,153 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0064.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0064.txt.redacted", + "totalRedactions": 6, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.785, + "snippet": "Transcript #64\n==================================\n<<>>: Hello—I'm having trouble with a double charge on" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.769, + "snippet": "m having trouble with a double charge on my card.\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.79, + "snippet": "rge on my card.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.778, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Social Securit" + }, + { + "type": "<<>>", + "value": "123-45-6789", + "confidence": 0.998, + "snippet": "can you also provide your <<>>: 123-45-6789?\n<<>>: Can you check on that?\n<<>>: Thanks, can y" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.792, + "snippet": "provide your <<>>: 123-45-6789?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.758, + "snippet": "umber: 123-45-6789?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Sexual Orienta" + }, + { + "type": "<<>>", + "value": "Heterosexual", + "confidence": 0.99, + "snippet": "ks, can you also provide your <<>>: Heterosexual?\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.842, + "snippet": "so provide your <<>>: Heterosexual?\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: T" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.994, + "snippet": "ion: Heterosexual?\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: Thanks, can you also provide your Profile UR" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.744, + "snippet": "\n<<>>: Here is my <<>>: 00:1B:44:11:3A:B7\n<<>>: Thanks, can you also provide your <<>>: h" + }, + { + "type": "<<>>", + "value": "https://social.example.com/jordan", + "confidence": 0.99, + "snippet": "a: Thanks, can you also provide your <<>>: https://social.example.com/jordan?\n<<>>: Can you check on that?\n<<>>: Let me look t" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.808, + "snippet": "r <<>>: https://social.example.com/jordan?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.714, + "snippet": "example.com/jordan?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.789, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.766, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.813, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.776, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0064.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0064.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..088c0284 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0064.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,429 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0064.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0064.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.872, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.806, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.949, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.904, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.952, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.921, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.509, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.774, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.773, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.703, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.793, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.954, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.889, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.855, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.563, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.56, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.933, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.851, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.807, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.683, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.901, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.919, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.552, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.51, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.795, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.962, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.929, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.81, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.799, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.978, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.953, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.625, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.567, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.547, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.546, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.517, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.732, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.677, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.676, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.613, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.574, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.67, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.939, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.95, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Page views", + "value": "Serial Number", + "confidence": 0.528, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.907, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.625, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.83, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.802, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.965, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.875, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.814, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.952, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.938, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.866, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.913, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Casey\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Casey", + "confidence": 0.656, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Casey\",\n \"confidence\": 0.785,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Casey", + "confidence": 0.632, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Casey\",\n \"confidence\": 0.785,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.897, + "snippet": "uble charge on\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.775, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.769,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.758, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.769,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0065.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0065.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..e83fba6c --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0065.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0065.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0065.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.994, + "snippet": "?\nSofia: Thanks, can you also provide your <<>>: <<>>?\nCasey: Here is my Survey Data: Q1=Yes, Q2=No\nSof" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.988, + "snippet": "s, Q2=No\nSofia: Thanks, can you also provide your <<>>: None?\nCasey: Can you check on that?\nSofia: Let m" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.916, + "snippet": "ofia: Thanks, can you also provide your <<>>: None?\nCasey: Can you check on that?\nSofia: Let me look" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0065.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0065.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..a94ebaf2 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0065.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,465 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0065.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0065.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.797, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.773, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.934, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.934, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.966, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.938, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.642, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.829, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.767, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.757, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.836, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.892, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.841, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.613, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.526, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.832, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Religion", + "value": "Industry", + "confidence": 0.567, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.52, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.842, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.815, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.674, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.879, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.922, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.551, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.821, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.954, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.929, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.981, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.802, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.794, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.95, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.538, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.628, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.62, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Religion", + "value": "Profile Picture Image", + "confidence": 0.589, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.625, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.589, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.574, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.513, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.723, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.707, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.65, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.61, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.896, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.661, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.925, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.942, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.894, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.515, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.819, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.955, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.803, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.911, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.781, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.738, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.94, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.933, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.557, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.777, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.576, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Email\",\n \"value\": \"jordan@example.com\",\n " + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.944, + "snippet": " {\n \"type\": \"Email\",\n \"value\": \"jordan@example.com\",\n \"confidence\": 0.994,\n \"snippet\":" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.936, + "snippet": "\\nSofia: Thanks, can you also provide your Email: jordan@example.com?\\nCasey: Here is my Survey Data: Q1=Yes, Q2=No\\nS" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.814, + "snippet": "es, Q2=No\\nSof\"\n },\n {\n \"type\": \"Religion\",\n \"value\": \"Religion\",\n \"confidenc" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.775, + "snippet": "es, Q2=No\\nSof\"\n },\n {\n \"type\": \"Religion\",\n \"value\": \"Religion\",\n \"confidenc" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.698, + "snippet": " {\n \"type\": \"Religion\",\n \"value\": \"Religion\",\n \"confidence\": 0.988,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.658, + "snippet": " {\n \"type\": \"Religion\",\n \"value\": \"Religion\",\n \"confidence\": 0.988,\n \"snippet\":" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.815, + "snippet": ", Q2=No\\nSofia: Thanks, can you also provide your Religion: None?\\nCasey: Can you check on that?\\nSofia: Let" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.799, + "snippet": ", Q2=No\\nSofia: Thanks, can you also provide your Religion: None?\\nCasey: Can you check on that?\\nSofia: Let" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0066.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0066.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..d2296559 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0066.txt.redacted.redaction.json.redacted @@ -0,0 +1,51 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0066.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0066.txt.redacted", + "totalRedactions": 1, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.997, + "snippet": "ah: Thanks, can you also provide your <<>>: <<>>?\nSam: Can you check on that?\nNoah: Let me look th" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0066.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0066.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..65fc48e6 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0066.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,441 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0066.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0066.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.725, + "snippet": "cted\",\n \"totalRedactions\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.932, + "snippet": "ons\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.902, + "snippet": "ons\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.939, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.89, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.994, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.844, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.712, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.657, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.821, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.96, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.555, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.879, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.862, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.562, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.536, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.967, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.85, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.8, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.65, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.839, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.891, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.719, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.635, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.817, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.57, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.953, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.916, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.985, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.939, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.755, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.985, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.959, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.68, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.669, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.678, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.642, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.583, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.501, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.647, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.549, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.764, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.746, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.659, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.697, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.686, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.555, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.932, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.956, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.509, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.939, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.723, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.863, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.951, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Social Security Number", + "value": "Tax Information", + "confidence": 0.516, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.861, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.964, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.814, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.791, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.928, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.921, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.741, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "IP Address", + "value": "192.168.1.42", + "confidence": 0.88, + "snippet": "{\n \"type\": \"IP Address\",\n \"value\": \"192.168.1.42\",\n \"confidence\": 0.997,\n \"snippet\":" + }, + { + "type": "MAC Address", + "value": "192.168.1.42", + "confidence": 0.862, + "snippet": "{\n \"type\": \"IP Address\",\n \"value\": \"192.168.1.42\",\n \"confidence\": 0.997,\n \"snippet\":" + }, + { + "type": "MAC Address", + "value": "192.168.1.42", + "confidence": 0.843, + "snippet": "ah: Thanks, can you also provide your IP Address: 192.168.1.42?\\nSam: Can you check on that?\\nNoah: Let me look " + }, + { + "type": "IP Address", + "value": "192.168.1.42", + "confidence": 0.825, + "snippet": "ah: Thanks, can you also provide your IP Address: 192.168.1.42?\\nSam: Can you check on that?\\nNoah: Let me look " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0067.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0067.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..6af07bac --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0067.txt.redacted.redaction.json.redacted @@ -0,0 +1,147 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0067.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0067.txt.redacted", + "totalRedactions": 6, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.951, + "snippet": "Transcript #67\n==================================\n<<>>: Can you assist me with logging into my account.\n" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.929, + "snippet": ": Can you assist me with logging into my account.\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.952, + "snippet": "nto my account.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.925, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.946, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: https://social.example.c" + }, + { + "type": "<<>>", + "value": "https://social.example.com/jordan", + "confidence": 0.975, + "snippet": " that up for you.\n<<>>: Here is my <<>>: https://social.example.com/jordan\n<<>>: Let me look that up for you.\n<<>>: Can you" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.925, + "snippet": "my <<>>: https://social.example.com/jordan\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.925, + "snippet": "mple.com/jordan\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.93, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.929, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.927, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.948, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.92, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.941, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: marketing_emails=fa" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.579, + "snippet": ": Let me look that up for you.\n<<>>: Here is my <<>>: marketing_emails=false" + }, + { + "type": "<<>>", + "value": "marketing_emails", + "confidence": 0.568, + "snippet": " up for you.\n<<>>: Here is my <<>>: marketing_emails=false" + }, + { + "type": "<<>>", + "value": "false", + "confidence": 0.685, + "snippet": "or: Here is my <<>>: marketing_emails=false" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0067.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0067.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..69b776f0 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0067.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,423 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0067.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0067.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.82, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.807, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.942, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.892, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.944, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.938, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.612, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.512, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.785, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.781, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.721, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.756, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.951, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.898, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.872, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.58, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.571, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.942, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.856, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.811, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.659, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.883, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.906, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.596, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.801, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.582, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.958, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.924, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.981, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.853, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.74, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.955, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.579, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.594, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.589, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.562, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.711, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.735, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.674, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.633, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.624, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.665, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.937, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.946, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.92, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.602, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.817, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.804, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.967, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.872, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.811, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.947, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.936, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.861, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.898, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "User ID", + "value": "Taylor", + "confidence": 0.775, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.951,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.736, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.951,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.872, + "snippet": " my account.\\n\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.794, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.929,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.753, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.929,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0068.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0068.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..4aacf27c --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0068.txt.redacted.redaction.json.redacted @@ -0,0 +1,165 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0068.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0068.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.973, + "snippet": "Transcript #68\n==================================\n<<>>: Hello—I'm having trouble with updating my subscr" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.952, + "snippet": "I'm having trouble with updating my subscription.\n<<>>: Thanks, can you also provide your Survey Data: Q" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.954, + "snippet": "you also provide your Survey Data: Q1=Yes, Q2=No?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.961, + "snippet": " Data: Q1=Yes, Q2=No?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: 0" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": "a: Thanks, can you also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look tha" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.941, + "snippet": "also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.943, + "snippet": "s: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Ser" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.966, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: SN-1234567890\n<<>>: Let" + }, + { + "type": "<<>>", + "value": "SN-1234567890", + "confidence": 0.998, + "snippet": "k that up for you.\n<<>>: Here is my <<>>: SN-1234567890\n<<>>: Let me look that up for you.\n<<>>: Can you ch" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.844, + "snippet": "you.\n<<>>: Here is my <<>>: SN-1234567890\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.936, + "snippet": ": SN-1234567890\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.927, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Coo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.957, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Cookies: sessionid=abcd1234\n<<>>: Let " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.807, + "snippet": " you.\n<<>>: Here is my Cookies: sessionid=abcd1234\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.926, + "snippet": "sionid=abcd1234\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.924, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.922, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.925, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.954, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.95, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0068.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0068.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..0c50038a --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0068.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0068.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0068.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.865, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.807, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.945, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.915, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.946, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.943, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.678, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.805, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.78, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.72, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.78, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.945, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.892, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.873, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.608, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.551, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.932, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.538, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.863, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.815, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.656, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.877, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.901, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.577, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.797, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.539, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.96, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.92, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.98, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.836, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.757, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.97, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.951, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.592, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.589, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.572, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.561, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.532, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.719, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.538, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.711, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.678, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.636, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.619, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.676, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.937, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.948, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Page views", + "value": "Serial Number", + "confidence": 0.5, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.921, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.624, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.816, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.809, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.971, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.85, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.803, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.947, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.939, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.534, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.854, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.91, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Sam", + "confidence": 0.536, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.973,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.523, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.973,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.902, + "snippet": "ting my subscr\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.539, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.952,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.528, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.952,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0069.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0069.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..7be2dd7e --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0069.txt.redacted.redaction.json.redacted @@ -0,0 +1,111 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0069.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0069.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.582, + "snippet": "Transcript #69\n==================================\n<<>>: Hello—I'm having trouble with a late delivery.\nM" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.645, + "snippet": "m: Hello—I'm having trouble with a late delivery.\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.572, + "snippet": "late delivery.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.627, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.51, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.51, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.607, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.631, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: +1-202-" + }, + { + "type": "<<>>", + "value": "+1-202-555-0172", + "confidence": 0.993, + "snippet": "t?\n<<>>: Thanks, can you also provide your <<>>: +1-202-555-0172?\n<<>>: Here is my <<>>: 1990-05-12" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.532, + "snippet": "can you also provide your <<>>: +1-202-555-0172?\n<<>>: Here is my <<>>: 1990-05-12" + }, + { + "type": "<<>>", + "value": "1990-05-12", + "confidence": 0.966, + "snippet": ": +1-202-555-0172?\n<<>>: Here is my <<>>: 1990-05-12" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0069.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0069.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..72e61683 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0069.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,429 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0069.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0069.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.86, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.786, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.937, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.886, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.942, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.937, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.627, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.793, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.781, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.711, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.742, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.948, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.899, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.867, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.604, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.581, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.933, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.86, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.822, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.6, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.874, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.893, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.562, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.799, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.581, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.958, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.916, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.846, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.779, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.972, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.952, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.567, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.585, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.569, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.559, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.558, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.702, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.794, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.673, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.569, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.62, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.656, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.934, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.944, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.543, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.912, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.576, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.814, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.958, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.802, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.966, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.852, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.781, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.945, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.936, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.863, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.798, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Sam", + "confidence": 0.77, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.582,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.69, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.582,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.738, + "snippet": "e delivery.\\nM\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": " + }, + { + "type": "User ID", + "value": "Maya", + "confidence": 0.789, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.645,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.741, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.645,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0070.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0070.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..b7acdfba --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0070.txt.redacted.redaction.json.redacted @@ -0,0 +1,159 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0070.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0070.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.9, + "snippet": "Transcript #70\n==================================\n<<>>: Can you assist me with a late delivery.\n<<>>: Le" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.813, + "snippet": "==\n<<>>: Can you assist me with a late delivery.\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.894, + "snippet": "late delivery.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.822, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.859, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.872, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.812, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.873, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: None" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.991, + "snippet": " on that?\n<<>>: Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.863, + "snippet": "<<>>: Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.786, + "snippet": "Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.83, + "snippet": "our <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.866, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.788, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.852, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.835, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.803, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.869, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.826, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0070.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0070.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..6c44d3d1 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0070.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0070.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0070.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.854, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.796, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.938, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.891, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.945, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.941, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.643, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.507, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.774, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.765, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.691, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.744, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.949, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.895, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.863, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.556, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.55, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.937, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.843, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.795, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.661, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.894, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.901, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.524, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.794, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.539, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.962, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.921, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.807, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.8, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.977, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.572, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.508, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.575, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.571, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.554, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.528, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.716, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.769, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.661, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.614, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.585, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.656, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.939, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.943, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.608, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.907, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.579, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.821, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.959, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.798, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.972, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.884, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.785, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.942, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.936, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.854, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.504, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.859, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Riley", + "confidence": 0.693, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.9,\n \"snippet\": \"" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.509, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.9,\n \"snippet\": \"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.846, + "snippet": "ery.\\nNoah: Le\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Noah\",\n \"confidence\": " + }, + { + "type": "User ID", + "value": "Noah", + "confidence": 0.754, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Noah\",\n \"confidence\": 0.813,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.527, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Noah\",\n \"confidence\": 0.813,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0071.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0071.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..e72814f1 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0071.txt.redacted.redaction.json.redacted @@ -0,0 +1,123 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0071.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0071.txt.redacted", + "totalRedactions": 6, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.8, + "snippet": "Transcript #71\n==================================\n<<>>: Good morning, question about updating my subscri" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.811, + "snippet": "morning, question about updating my subscription.\n<<>>: Thanks, can you also provide your Declared Inter" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.514, + "snippet": "cription.\n<<>>: Thanks, can you also provide your <<>>: hiking, cooking?\n<<>>: Can you check on that?\nN" + }, + { + "type": "<<>>", + "value": "hiking, cooking", + "confidence": 0.902, + "snippet": "ks, can you also provide your <<>>: hiking, cooking?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.742, + "snippet": "provide your <<>>: hiking, cooking?\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.79, + "snippet": "s: hiking, cooking?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>:" + }, + { + "type": "<<>>", + "value": "SN-1234567890", + "confidence": 0.994, + "snippet": " Thanks, can you also provide your <<>>: SN-1234567890?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.731, + "snippet": "u also provide your <<>>: SN-1234567890?\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.782, + "snippet": "ber: SN-1234567890?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: 0" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": "h: Thanks, can you also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.75, + "snippet": "also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.786, + "snippet": " 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Cookies: sessi" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.771, + "snippet": "ou also provide your Cookies: sessionid=abcd1234?\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0071.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0071.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..3ab979f5 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0071.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,453 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0071.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0071.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.857, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.799, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.941, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.93, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Declared Interests", + "value": "Biometric Identifier", + "confidence": 0.633, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.954, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.94, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.654, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.531, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.793, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.619, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.772, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.94, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.899, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.889, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.688, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.934, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.855, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.687, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.608, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.886, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.904, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.505, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.792, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.966, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.935, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.824, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.777, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.973, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.957, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.533, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.649, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.586, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.571, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.539, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.518, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.72, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.552, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.698, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.648, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.602, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.609, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.656, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.939, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.944, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.568, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.9, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.637, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.62, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.819, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.964, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.805, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.971, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.883, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.834, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.946, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.945, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.803, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.85, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.506, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.945, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Riley", + "confidence": 0.687, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.8,\n \"snippet\": \"" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.674, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.8,\n \"snippet\": \"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.925, + "snippet": "ing my subscri\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Noah\",\n \"confidence\": " + }, + { + "type": "User ID", + "value": "Noah", + "confidence": 0.738, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Noah\",\n \"confidence\": 0.811,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Noah", + "confidence": 0.623, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Noah\",\n \"confidence\": 0.811,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0072.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0072.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..1bcc8026 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0072.txt.redacted.redaction.json.redacted @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0072.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0072.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.57, + "snippet": " you.\nTaylor: Here is my Approximate Geolocation: <<>>\nEthan: Thanks, can you also provide your <<>>: " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.983, + "snippet": "\nEthan: Thanks, can you also provide your <<>>: <<>>?\nTaylor: Here is my <<>>: <<>>" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.755, + "snippet": "\nEthan: Thanks, can you also provide your <<>>: <<>>?\nTaylor: Here is my <<>>: <<>>" + }, + { + "type": "<<>>", + "value": "1990-05-12", + "confidence": 0.996, + "snippet": "er: <<>>?\nTaylor: Here is my <<>>: <<>>an: Let me look that up for you.\nTaylor: Can y" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0072.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0072.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..374668b8 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0072.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,489 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0072.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0072.txt.redacted.redaction.json.redacted", + "totalRedactions": 37, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.86, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.656, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.93, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.856, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Gender", + "value": "Biometric Identifier", + "confidence": 0.72, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.967, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.914, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.502, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.943, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.702, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.617, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.728, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.953, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.523, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.699, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.864, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.643, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.856, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.508, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.828, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.748, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.886, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.902, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.657, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.546, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.836, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.96, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.913, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.845, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.803, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.977, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.96, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.505, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.76, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.578, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.645, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.622, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.707, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.506, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.734, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.578, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.882, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.618, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.568, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.936, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.951, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.9, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.798, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.946, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.828, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.88, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.706, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.621, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.925, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Gender", + "value": "User Preferences", + "confidence": 0.903, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.835, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.765, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.907, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Region\",\n \"value\": \"San Francisco, CA\",\n \"" + }, + { + "type": "IP Address", + "value": "San Francisco, CA", + "confidence": 0.633, + "snippet": " {\n \"type\": \"Region\",\n \"value\": \"San Francisco, CA\",\n \"confidence\": 0.57,\n \"snippet\": " + }, + { + "type": "IP Address", + "value": "San Francisco, CA", + "confidence": 0.676, + "snippet": "you.\\nTaylor: Here is my Approximate Geolocation: San Francisco, CA\\nEthan: Thanks, can you also provide your Gender:" + }, + { + "type": "MAC Address", + "value": "San Francisco, CA", + "confidence": 0.581, + "snippet": "you.\\nTaylor: Here is my Approximate Geolocation: San Francisco, CA\\nEthan: Thanks, can you also provide your Gender:" + }, + { + "type": "User Preferences", + "value": "Non-binary", + "confidence": 0.716, + "snippet": " {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.983,\n \"snippet\":" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.674, + "snippet": " {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.983,\n \"snippet\":" + }, + { + "type": "Biometric Identifier", + "value": "Non-binary", + "confidence": 0.585, + "snippet": " {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.983,\n \"snippet\":" + }, + { + "type": "User Preferences", + "value": "Non-binary", + "confidence": 0.774, + "snippet": "nEthan: Thanks, can you also provide your Gender: Non-binary?\\nTaylor: Here is my Date of Birth: 1990-05-12\\nE" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.7, + "snippet": "nEthan: Thanks, can you also provide your Gender: Non-binary?\\nTaylor: Here is my Date of Birth: 1990-05-12\\nE" + }, + { + "type": "Biometric Identifier", + "value": "Non-binary", + "confidence": 0.65, + "snippet": "nEthan: Thanks, can you also provide your Gender: Non-binary?\\nTaylor: Here is my Date of Birth: 1990-05-12\\nE" + }, + { + "type": "Date of Birth", + "value": "1990-05-12\\nEth", + "confidence": 0.627, + "snippet": "r: Non-binary?\\nTaylor: Here is my Date of Birth: 1990-05-12\\nEth\"\n },\n {\n \"type\": \"Sexual Orienta" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.813, + "snippet": "990-05-12\\nEth\"\n },\n {\n \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confide" + }, + { + "type": "Industry", + "value": "Sexual Orientation", + "confidence": 0.542, + "snippet": "990-05-12\\nEth\"\n },\n {\n \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confide" + }, + { + "type": "User Preferences", + "value": "Non-binary", + "confidence": 0.704, + "snippet": " \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.755,\n \"snippet\":" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.694, + "snippet": " \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.755,\n \"snippet\":" + }, + { + "type": "Biometric Identifier", + "value": "Non-binary", + "confidence": 0.575, + "snippet": " \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.755,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0073.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0073.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..239d7ed3 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0073.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0073.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0073.txt.redacted", + "totalRedactions": 2, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Profile Picture Image", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.983, + "snippet": "Ethan: Thanks, can you also provide your <<>>: <<>>?\nAlex: Can you check on that?\nEthan: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.961, + "snippet": "\nEthan: Thanks, can you also provide your <<>>: <<>>?\nAlex: Can you check on that?\nEthan: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.658, + "snippet": "\nEthan: Thanks, can you also provide your <<>>: <<>>?\nAlex: Can you check on that?\nEthan: Let me look " + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0073.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0073.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..31a0ef4a --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0073.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,489 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0073.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0073.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.866, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.85, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.933, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.889, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.967, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.925, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.587, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.764, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.712, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.782, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.962, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.558, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.661, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.874, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.688, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.926, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.87, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.822, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.889, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.939, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.675, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.51, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.864, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.63, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.971, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.928, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.798, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.764, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.981, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.57, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.521, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.952, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.669, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.636, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.528, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.719, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.501, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.915, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.725, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.507, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.547, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.588, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.945, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.95, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.556, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.911, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.829, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.834, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.868, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.887, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.602, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.937, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.867, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Gender", + "value": "User Preferences", + "confidence": 0.787, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.885, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.509, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.664, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.874, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence\": 0.983,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "U123456", + "confidence": 0.633, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence\": 0.983,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.903, + "snippet": "Ethan: Thanks, can you also provide your User ID: U123456?\\nAlex: Can you check on that?\\nEthan: Thanks, ca" + }, + { + "type": "Race/Ethnicity", + "value": "U123456", + "confidence": 0.739, + "snippet": "Ethan: Thanks, can you also provide your User ID: U123456?\\nAlex: Can you check on that?\\nEthan: Thanks, ca" + }, + { + "type": "User Agent", + "value": "U123456", + "confidence": 0.551, + "snippet": "Ethan: Thanks, can you also provide your User ID: U123456?\\nAlex: Can you check on that?\\nEthan: Thanks, ca" + }, + { + "type": "User Preferences", + "value": "Non-binary", + "confidence": 0.669, + "snippet": " {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.961,\n \"snippet\":" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.579, + "snippet": " {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.961,\n \"snippet\":" + }, + { + "type": "User Preferences", + "value": "Non-binary", + "confidence": 0.779, + "snippet": "nEthan: Thanks, can you also provide your Gender: Non-binary?\\nAlex: Can you check on that?\\nEthan: Let me loo" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.682, + "snippet": "nEthan: Thanks, can you also provide your Gender: Non-binary?\\nAlex: Can you check on that?\\nEthan: Let me loo" + }, + { + "type": "Biometric Identifier", + "value": "Non-binary", + "confidence": 0.645, + "snippet": "nEthan: Thanks, can you also provide your Gender: Non-binary?\\nAlex: Can you check on that?\\nEthan: Let me loo" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.848, + "snippet": ": Let me look \"\n },\n {\n \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confide" + }, + { + "type": "User Preferences", + "value": "Non-binary", + "confidence": 0.691, + "snippet": " \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.658,\n \"snippet\":" + }, + { + "type": "Gender", + "value": "Non-binary", + "confidence": 0.612, + "snippet": " \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.658,\n \"snippet\":" + }, + { + "type": "Biometric Identifier", + "value": "Non-binary", + "confidence": 0.515, + "snippet": " \"type\": \"Sexual Orientation\",\n \"value\": \"Non-binary\",\n \"confidence\": 0.658,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0074.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0074.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..b9a2a06b --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0074.txt.redacted.redaction.json.redacted @@ -0,0 +1,57 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0074.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0074.txt.redacted", + "totalRedactions": 2, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.974, + "snippet": "m: Thanks, can you also provide your <<>>: <<>>?\nTaylor: Can you check on that?\nLiam: Let me look" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.951, + "snippet": ", can you also provide your <<>>: <<>>?\nTaylor: Can you check on that?\nLiam: Let me look" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0074.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0074.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..29a40206 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0074.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0074.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0074.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.819, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.814, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.933, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.945, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.908, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.986, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.769, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.727, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.719, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.757, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.932, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.896, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.552, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.861, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.569, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.551, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.946, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.861, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.78, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.606, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.864, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.909, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.658, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.8, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.631, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.955, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.908, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.982, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.894, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.755, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.967, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.95, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.51, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.648, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.644, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.673, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.639, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.636, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.583, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.579, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.743, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.694, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.53, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.618, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.673, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.906, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.939, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.922, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Biometric Identifier", + "value": "Sexual Orientation", + "confidence": 0.505, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.863, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.945, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.858, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.645, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.93, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.907, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.904, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.925, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.642, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.86, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "IP Address", + "value": "https://social.example.com/jordan", + "confidence": 0.543, + "snippet": "\n \"type\": \"Profile URL\",\n \"value\": \"https://social.example.com/jordan\",\n \"confidence\": 0.974,\n \"snippet\":" + }, + { + "type": "IP Address", + "value": "https://social.example.com/jordan", + "confidence": 0.627, + "snippet": "m: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\\nTaylor: Can you check on that?\\nLiam: Let me lo" + }, + { + "type": "Account Number", + "value": "https://social.example.com/jordan", + "confidence": 0.551, + "snippet": "m: Thanks, can you also provide your Profile URL: https://social.example.com/jordan?\\nTaylor: Can you check on that?\\nLiam: Let me lo" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.752, + "snippet": "m: Let me look\"\n },\n {\n \"type\": \"Biometric Identifier\",\n \"value\": \"FaceID registered\",\n \"" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.923, + "snippet": "\"type\": \"Biometric Identifier\",\n \"value\": \"FaceID registered\",\n \"confidence\": 0.951,\n \"snippet\":" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.568, + "snippet": ",\n \"snippet\": \", can you also provide your Biometric Identifier: FaceID registered?\\nTaylor: Can you check on tha" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.946, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\\nTaylor: Can you check on that?\\nLiam: Let me lo" + }, + { + "type": "User Preferences", + "value": "FaceID registered", + "confidence": 0.504, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\\nTaylor: Can you check on that?\\nLiam: Let me lo" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0075.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0075.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..a00db6d6 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0075.txt.redacted.redaction.json.redacted @@ -0,0 +1,87 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0075.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0075.txt.redacted", + "totalRedactions": 6, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "https://example.com", + "confidence": 0.538, + "snippet": "p for you.\nRiley: Here is my <<>>: https://example.com/img/profile123.jpg\nNoah: Thanks, can you also pro" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.982, + "snippet": "pg\nNoah: Thanks, can you also provide your <<>>: <<>>?\nRiley: Can you check on that?\nNoah: Thanks, can " + }, + { + "type": "<<>>", + "value": "https://social.example.com", + "confidence": 0.784, + "snippet": "h: Thanks, can you also provide your <<>>: https://social.example.com/jordan?\nRiley: Can you check on that?\nNoah: Let m" + }, + { + "type": "<<>>", + "value": "jordan", + "confidence": 0.506, + "snippet": "vide your <<>>: https://social.example.com/jordan?\nRiley: Can you check on that?\nNoah: Let me look " + }, + { + "type": "<<>>", + "value": "+1-202-555-0172", + "confidence": 0.994, + "snippet": "me look that up for you.\nRiley: Here is my <<>>: +1-202-555-0172\nNoah: Let me look that up for you.\nRiley: Here is" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.976, + "snippet": "e look that up for you.\nRiley: Here is my <<>>: Non-binary" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.706, + "snippet": "e look that up for you.\nRiley: Here is my <<>>: Non-binary" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0075.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0075.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..4d803142 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0075.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,417 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0075.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0075.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.828, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.717, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.938, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.901, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.954, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.919, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.582, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.989, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.535, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.841, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.753, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.627, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.776, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.852, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.887, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.534, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.851, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.549, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.933, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.86, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.794, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.647, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.857, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.904, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.661, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.842, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.945, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.909, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.983, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.906, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.744, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.95, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.739, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture Image", + "confidence": 0.553, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.547, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.698, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.678, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.673, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.593, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.621, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.738, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Profile Picture Image", + "value": "Race/Ethnicity", + "confidence": 0.712, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.706, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.639, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.68, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.918, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.94, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.942, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.599, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.84, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.942, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.884, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.506, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.914, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.807, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.686, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.935, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.913, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.787, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.78, + "snippet": " {\n \"type\": \"Email\",\n \"value\": \"jordan@example.com\",\n \"confidence\": 0.982,\n \"snippet\":" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.782, + "snippet": "g\\nNoah: Thanks, can you also provide your Email: jordan@example.com?\\nRiley: Can you check on that?\\nNoah: Thanks, ca" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0076.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0076.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..bcf83827 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0076.txt.redacted.redaction.json.redacted @@ -0,0 +1,51 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0076.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0076.txt.redacted", + "totalRedactions": 1, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.978, + "snippet": "ah: Thanks, can you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nJordan: Can you chec" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0076.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0076.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..60262b64 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0076.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,441 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0076.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0076.txt.redacted.redaction.json.redacted", + "totalRedactions": 34, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.831, + "snippet": "cted\",\n \"totalRedactions\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.77, + "snippet": "cted\",\n \"totalRedactions\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.933, + "snippet": "ons\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.914, + "snippet": "ons\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.942, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.88, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.994, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.503, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.891, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.719, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.663, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.824, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.946, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.897, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.883, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.727, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.943, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.535, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.836, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.812, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.797, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.846, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.91, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.629, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.79, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.707, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.944, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.936, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.913, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.65, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.972, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.95, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.606, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.666, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.618, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.601, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.565, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.501, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.652, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.565, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.749, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.739, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.696, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Industry", + "value": "Region", + "confidence": 0.514, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.703, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.57, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.932, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.952, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.508, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.95, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.769, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.859, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Social Security Number", + "confidence": 0.52, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.953, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Social Security Number", + "value": "Tax Information", + "confidence": 0.512, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.895, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.987, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.962, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.844, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.93, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.929, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.821, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.928, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Mozilla/5.0\",\n \"confid" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.957, + "snippet": "\"snippet\": \"ah: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\\nJordan" + }, + { + "type": "User ID", + "value": "User Agent", + "confidence": 0.538, + "snippet": "\"snippet\": \"ah: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\\nJordan" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0077.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0077.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..b2286164 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0077.txt.redacted.redaction.json.redacted @@ -0,0 +1,105 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0077.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0077.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.678, + "snippet": "Transcript #77\n==================================\n<<>>: Hi, I need help with firmware update errors.\n<<>>" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.554, + "snippet": "<<>>: Hi, I need help with firmware update errors.\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.673, + "snippet": " update errors.\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: SN-1234567890\n<<>>: Let" + }, + { + "type": "<<>>", + "value": "SN-1234567890", + "confidence": 0.999, + "snippet": "hat up for you.\n<<>>: Here is my <<>>: SN-1234567890\n<<>>: Let me look that up for you.\n<<>>: Can you" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.506, + "snippet": ".\n<<>>: Here is my <<>>: SN-1234567890\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.522, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.585, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: Non-binary\n<<>>: Thanks, can y" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.977, + "snippet": " look that up for you.\n<<>>: Here is my <<>>: Non-binary\n<<>>: Thanks, can you also provide your Declared I" + }, + { + "type": "<<>>", + "value": "hiking, cooking", + "confidence": 0.914, + "snippet": "ks, can you also provide your <<>>: hiking, cooking?\n<<>>: Can you check on that?\n<<>>: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.559, + "snippet": "provide your <<>>: hiking, cooking?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0077.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0077.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..c09f738e --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0077.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0077.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0077.txt.redacted.redaction.json.redacted", + "totalRedactions": 37, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.873, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.812, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.949, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.869, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.963, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.941, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.992, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.54, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.84, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.772, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.718, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.795, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.952, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.892, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.873, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.591, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.527, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.945, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.524, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.858, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.833, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.763, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.893, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.901, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.611, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.781, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.965, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.929, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.977, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.833, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.829, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.971, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.952, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.548, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.601, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.569, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.523, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.513, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.728, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.723, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.699, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.615, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.647, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.71, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.516, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.933, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.946, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.519, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.916, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.655, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.813, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.818, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.563, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.966, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.837, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.765, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.952, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.932, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.86, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.906, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\"" + }, + { + "type": "User ID", + "value": "Jordan", + "confidence": 0.774, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.678,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.609, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.678,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.865, + "snippet": "e errors.\\nAva\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.78, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.554,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.619, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.554,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "dan", + "confidence": 0.587, + "snippet": "\n \"confidence\": 0.554,\n \"snippet\": \"dan: Hi, I need help with firmware update errors.\\nAv" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0078.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0078.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..14eadf36 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0078.txt.redacted.redaction.json.redacted @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0078.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0078.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.856, + "snippet": ": Let me look that up for you.\nJordan: Here is my <<>>: <<>>\nEthan: Thanks, can you also provide your Us" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.888, + "snippet": "ook that up for you.\nJordan: Here is my <<>>: <<>>\nEthan: Thanks, can you also provide your User Age" + }, + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.964, + "snippet": "an: Thanks, can you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nJordan: Can you chec" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.999, + "snippet": "n: Thanks, can you also provide your <<>>: 00:1B:44:11:3A:B7?\nJordan: Can you check on that?\nEthan: Let me loo" + }, + { + "type": "<<>>", + "value": "hiking, cooking", + "confidence": 0.572, + "snippet": "p for you.\nJordan: Here is my <<>>: hiking, cooking" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0078.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0078.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..84624764 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0078.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,507 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0078.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0078.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.777, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.718, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.943, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.933, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.937, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.898, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.989, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.846, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.79, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.522, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.749, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.946, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.899, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.841, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.611, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.598, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.821, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Religion", + "value": "Industry", + "confidence": 0.651, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.572, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.908, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.885, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.611, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.797, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.921, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.649, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.849, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.965, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.907, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.846, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.76, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.98, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.737, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.545, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.942, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.605, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.508, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.566, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.56, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.722, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.522, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.517, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.746, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.671, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.639, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.611, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.874, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.68, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Religion", + "confidence": 0.584, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.917, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.95, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.867, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.833, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Social Security Number", + "confidence": 0.503, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.955, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.838, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.963, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.749, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.728, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.941, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.927, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.614, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.835, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.801, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Religion\",\n \"value\": \"Religion\",\n \"confidenc" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.769, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Religion\",\n \"value\": \"Religion\",\n \"confidenc" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.688, + "snippet": " {\n \"type\": \"Religion\",\n \"value\": \"Religion\",\n \"confidence\": 0.856,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.639, + "snippet": " {\n \"type\": \"Religion\",\n \"value\": \"Religion\",\n \"confidence\": 0.856,\n \"snippet\":" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.798, + "snippet": " Let me look that up for you.\\nJordan: Here is my Religion: None\\nEthan: Thanks, can you also provide your U" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.714, + "snippet": " Let me look that up for you.\\nJordan: Here is my Religion: None\\nEthan: Thanks, can you also provide your U" + }, + { + "type": "User Preferences", + "value": "None", + "confidence": 0.56, + "snippet": "ok that up for you.\\nJordan: Here is my Religion: None\\nEthan: Thanks, can you also provide your Us\"\n " + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.669, + "snippet": "rovide your Us\"\n },\n {\n \"type\": \"Religion\",\n \"value\": \"None\",\n \"confidence\": " + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.547, + "snippet": "rovide your Us\"\n },\n {\n \"type\": \"Religion\",\n \"value\": \"None\",\n \"confidence\": " + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.733, + "snippet": "ippet\": \"ook that up for you.\\nJordan: Here is my Religion: None\\nEthan: Thanks, can you also provide your U" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.649, + "snippet": "ippet\": \"ook that up for you.\\nJordan: Here is my Religion: None\\nEthan: Thanks, can you also provide your U" + }, + { + "type": "User Preferences", + "value": "None", + "confidence": 0.69, + "snippet": "ok that up for you.\\nJordan: Here is my Religion: None\\nEthan: Thanks, can you also provide your User Ag" + }, + { + "type": "Biometric Identifier", + "value": "None", + "confidence": 0.555, + "snippet": "ok that up for you.\\nJordan: Here is my Religion: None\\nEthan: Thanks, can you also provide your User Ag" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.947, + "snippet": " your User Age\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Mozilla/5.0\",\n \"confid" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0079.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0079.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..bee84156 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0079.txt.redacted.redaction.json.redacted @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0079.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0079.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.648, + "snippet": "elivery.\nEthan: Thanks, can you also provide your <<>>: Independent?\nJordan: Can you check on that?\nEtha" + }, + { + "type": "<<>>", + "value": "Independent", + "confidence": 0.977, + "snippet": " can you also provide your <<>>: Independent?\nJordan: Can you check on that?\nEthan: Thanks, ca" + }, + { + "type": "<<>>", + "value": "marketing_emails", + "confidence": 0.948, + "snippet": "anks, can you also provide your <<>>: marketing_emails=false?\nJordan: Here is my <<>>: +1-202-555-0172\n" + }, + { + "type": "<<>>", + "value": "+1-202-555-0172", + "confidence": 0.995, + "snippet": "marketing_emails=false?\nJordan: Here is my <<>>: +1-202-555-0172\nEthan: Thanks, can you also provide your User Pre" + }, + { + "type": "<<>>", + "value": "marketing_emails=false", + "confidence": 0.95, + "snippet": "anks, can you also provide your <<>>: marketing_emails=false?\nJordan: Can you check on that?\nEthan: Let me loo" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0079.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0079.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..f30b1bd5 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0079.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,459 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0079.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0079.txt.redacted.redaction.json.redacted", + "totalRedactions": 34, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.831, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.685, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.949, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.896, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.95, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.935, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.677, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.992, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.589, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.859, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.824, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.758, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.93, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.512, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.92, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.887, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.765, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.671, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Industry", + "confidence": 0.804, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.609, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.901, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.816, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.632, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.877, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.92, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.771, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.803, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.965, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.949, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.989, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.952, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.977, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.588, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Date of Birth", + "value": "Phone", + "confidence": 0.56, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.558, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.912, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.705, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "User Preferences", + "value": "Profile Picture Image", + "confidence": 0.62, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.693, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.684, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.709, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.663, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.743, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.658, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.623, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.814, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.706, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.94, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.957, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.925, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.832, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.899, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.925, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.874, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.531, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.93, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.752, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.739, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.934, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.694, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.514, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.748, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.857, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Political Affiliation\",\n \"value\": \"Political Affiliation\",\n " + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.894, + "snippet": "type\": \"Political Affiliation\",\n \"value\": \"Political Affiliation\",\n \"confidence\": 0.648,\n \"snippet\":" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.83, + "snippet": "livery.\\nEthan: Thanks, can you also provide your Political Affiliation: Independent?\\nJordan: Can you check on that?\\nEt" + }, + { + "type": "Industry", + "value": "Political Affiliation", + "confidence": 0.557, + "snippet": "livery.\\nEthan: Thanks, can you also provide your Political Affiliation: Independent?\\nJordan: Can you check on that?\\nEt" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.891, + "snippet": "on that?\\nEtha\"\n },\n {\n \"type\": \"Political Affiliation\",\n \"value\": \"Independent\",\n \"confid" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.711, + "snippet": "7,\n \"snippet\": \" can you also provide your Political Affiliation: Independent?\\nJordan: Can you check on that?\\nEt" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.818, + "snippet": "an: Thanks, ca\"\n },\n {\n \"type\": \"User Preferences\",\n \"value\": \"marketing_emails\",\n \"c" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0080.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0080.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..243912ce --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0080.txt.redacted.redaction.json.redacted @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0080.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0080.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.522, + "snippet": "Transcript #80\n==================================\n<<>>: Good morning, question about logging into my acc" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.559, + "snippet": " morning, question about logging into my account.\n<<>>: Thanks, can you also provide your <<>>: <<>>" + }, + { + "type": "<<>>", + "value": "<<>>nary", + "confidence": 0.919, + "snippet": ".\n<<>>: Thanks, can you also provide your <<>>: <<>>nary?\n<<>>: Can you check on that?\n<<>>: Thanks, can" + }, + { + "type": "<<>>", + "value": "<<>>nary", + "confidence": 0.711, + "snippet": ".\n<<>>: Thanks, can you also provide your <<>>: <<>>nary?\n<<>>: Can you check on that?\n<<>>: Thanks, can" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0080.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0080.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..6bdc661e --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0080.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0080.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0080.txt.redacted.redaction.json.redacted", + "totalRedactions": 37, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.843, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.815, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.95, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.834, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.964, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.954, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.729, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.638, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.501, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.77, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.763, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.684, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.774, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.953, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.856, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.731, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.866, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.589, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.546, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.93, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.85, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.797, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.574, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.892, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.91, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.81, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.574, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.962, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.91, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.978, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.802, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.793, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.641, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.597, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.582, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.573, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.514, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.73, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.693, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.636, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.613, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.64, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.934, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.948, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.91, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.823, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.961, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.8, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.579, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.952, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.823, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.814, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.934, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.933, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.867, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.565, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.877, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\"" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.782, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.522,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Jordan", + "confidence": 0.561, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.522,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.865, + "snippet": "ng into my acc\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.741, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.559,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Maya", + "confidence": 0.532, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.559,\n \"snippet\":" + }, + { + "type": "Gender", + "value": "Non-bi", + "confidence": 0.751, + "snippet": "\\nMaya: Thanks, can you also provide your Gender: Non-bi\"\n },\n {\n \"type\": \"Gender\",\n " + }, + { + "type": "User Preferences", + "value": "Non-bi", + "confidence": 0.616, + "snippet": "\\nMaya: Thanks, can you also provide your Gender: Non-bi\"\n },\n {\n \"type\": \"Gender\",\n " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0081.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0081.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..b159c07b --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0081.txt.redacted.redaction.json.redacted @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0081.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0081.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.883, + "snippet": "k that up for you.\nRiley: Here is my <<>>: <<>>\nNoah: Let me look that up for you.\nRiley: Can you" + }, + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.913, + "snippet": "ok that up for you.\nRiley: Here is my <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)\nNoah: Let me look tha" + }, + { + "type": "<<>>", + "value": "123-45-6789", + "confidence": 0.999, + "snippet": "or you.\nRiley: Here is my <<>>: 123-45-6789\nNoah: Let me look that up for you.\nRiley: Can you" + }, + { + "type": "<<>>", + "value": "Noah", + "confidence": 0.531, + "snippet": "ok that up for you.\nRiley: Can you check on that?\nNoah: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0081.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0081.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..83c3b0b5 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0081.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,423 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0081.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0081.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.837, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.821, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.927, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.9, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.937, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.912, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.516, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.989, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.505, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.831, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.751, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.712, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.747, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.929, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.893, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.861, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.597, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.921, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.855, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.783, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.755, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.837, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.897, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.65, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.792, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.546, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.949, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.92, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.982, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.906, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.612, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.964, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.947, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.678, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.595, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.756, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.576, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.502, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.569, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.523, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.737, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.624, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.594, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.734, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.647, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.905, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.941, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.953, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.665, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.827, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.937, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.899, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.976, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.867, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.811, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.926, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.922, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.854, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Account Number", + "value": "https://social.example.com/jordan", + "confidence": 0.636, + "snippet": " that up for you.\\nRiley: Here is my Profile URL: https://social.example.com/jordan\\nNoah: Let me look that up for you.\\nRiley: Can y" + }, + { + "type": "IP Address", + "value": "https://social.example.com/jordan", + "confidence": 0.502, + "snippet": " that up for you.\\nRiley: Here is my Profile URL: https://social.example.com/jordan\\nNoah: Let me look that up for you.\\nRiley: Can y" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.822, + "snippet": "Riley: Can you\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Mozilla/5.0\",\n \"confid" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.956, + "snippet": "snippet\": \"ok that up for you.\\nRiley: Here is my User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)\\nNoah: L" + }, + { + "type": "User ID", + "value": "User Agent", + "confidence": 0.505, + "snippet": "snippet\": \"ok that up for you.\\nRiley: Here is my User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)\\nNoah: L" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0082.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0082.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..3ae68130 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0082.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0082.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0082.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Profile Picture Image", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.51, + "snippet": "late delivery.\nMaya: Let me look that up for you.\n<<>>: Can you check on that?\nMaya: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.991, + "snippet": "Thanks, can you also provide your <<>>: <<>>?\n<<>>: Here is my <<>>: <<>>\nMaya: Let " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.969, + "snippet": "/Ethnicity: <<>>?\n<<>>: Here is my <<>>: <<>>\nMaya: Let me look that up for you.\n<<>>: Can you" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0082.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0082.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..af2f9bd4 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0082.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,495 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0082.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0082.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.856, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.684, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.928, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.896, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Declared Interests", + "value": "Biometric Identifier", + "confidence": 0.544, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.923, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.873, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.572, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.989, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.853, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.806, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.533, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.741, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.948, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.894, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.875, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.629, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.62, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Race/Ethnicity", + "value": "Industry", + "confidence": 0.91, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.857, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.907, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.891, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.633, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.831, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.914, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.63, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.618, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.863, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "User ID", + "value": "Name", + "confidence": 0.642, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.966, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.943, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.513, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.972, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.801, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.69, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.981, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.711, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Date of Birth", + "value": "Phone", + "confidence": 0.519, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.941, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.521, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.672, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.606, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.705, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.675, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.773, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.572, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.761, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Religion", + "confidence": 0.523, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.948, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.961, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.539, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.893, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.587, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.835, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.958, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.85, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.628, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "Name", + "value": "User ID", + "confidence": 0.825, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.73, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.926, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.836, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.652, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.735, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.767, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Username\",\n \"value\": \"Riley\",\n \"confidence\":" + }, + { + "type": "Username", + "value": "Riley", + "confidence": 0.568, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Riley\",\n \"confidence\": 0.51,\n \"snippet\": " + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.618, + "snippet": "s, can you als\"\n },\n {\n \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidenc" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.743, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.991,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.669, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.991,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.768, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\\nRiley: Here is my Username: jordan_92\\nMaya: Le" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.742, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\\nRiley: Here is my Username: jordan_92\\nMaya: Le" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.591, + "snippet": "Ethnicity: Hispanic?\\nRiley: Here is my Username: jordan_92\\nMaya: Let \"\n },\n {\n \"type\": \"Us" + }, + { + "type": "User ID", + "value": "jordan_92", + "confidence": 0.509, + "snippet": "Ethnicity: Hispanic?\\nRiley: Here is my Username: jordan_92\\nMaya: Let \"\n },\n {\n \"type\": \"Us" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.715, + "snippet": "92\\nMaya: Let \"\n },\n {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confiden" + }, + { + "type": "Name", + "value": "jordan_92", + "confidence": 0.611, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.969,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "jordan_92", + "confidence": 0.536, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.969,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0083.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0083.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..0f603898 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0083.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0083.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0083.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.982, + "snippet": "ia: Thanks, can you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nSam: Can you check o" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.993, + "snippet": " that up for you.\nSam: Here is my <<>>: <<>>\nSofia: Let me look that up for you.\nSam: Here is " + }, + { + "type": "<<>>", + "value": "SN-1234567890", + "confidence": 0.998, + "snippet": "k that up for you.\nSam: Here is my <<>>: SN-1234567890\nSofia: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0083.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0083.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..37e43216 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0083.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,459 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0083.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0083.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.816, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.814, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.935, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.896, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.946, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.94, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.624, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.99, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.53, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.851, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.805, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.573, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.714, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.934, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.888, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.875, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.745, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.835, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Industry", + "confidence": 0.726, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.533, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.857, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.839, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.803, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.841, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.876, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.811, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.61, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.952, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.948, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.975, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.888, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.568, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.973, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.508, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.947, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.523, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Religion", + "value": "Profile Picture Image", + "confidence": 0.659, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.542, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.504, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.685, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.647, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.61, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.589, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.615, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.593, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.684, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.68, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.781, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.581, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.917, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.948, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.94, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.749, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.844, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.941, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.91, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.952, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.809, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.754, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.918, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.911, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.512, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.859, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.858, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Mozilla/5.0\",\n \"confid" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.871, + "snippet": "\"snippet\": \"ia: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\\nSam: C" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.804, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.993,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.692, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.993,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.77, + "snippet": "that up for you.\\nSam: Here is my Race/Ethnicity: Hispanic\\nSofia: Let me look that up for you.\\nSam: Here i" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.658, + "snippet": "that up for you.\\nSam: Here is my Race/Ethnicity: Hispanic\\nSofia: Let me look that up for you.\\nSam: Here i" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0084.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0084.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..b182ae45 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0084.txt.redacted.redaction.json.redacted @@ -0,0 +1,63 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0084.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0084.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.988, + "snippet": "ks, can you also provide your <<>>: <<>>?\nTaylor: Can you check on that?\nNoah: Let me look" + }, + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.967, + "snippet": "ah: Thanks, can you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\nTaylor: Here is my S" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.97, + "snippet": "o\nNoah: Thanks, can you also provide your <<>>: Non-binary?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0084.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0084.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..87b006e7 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0084.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,453 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0084.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0084.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.871, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.843, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.936, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.836, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Gender", + "value": "Biometric Identifier", + "confidence": 0.733, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.971, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.926, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.636, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.993, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.519, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.78, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.748, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.826, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.942, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.622, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.82, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.718, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Biometric Identifier", + "value": "Gender", + "confidence": 0.577, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.876, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.824, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.847, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.65, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.877, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.815, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.88, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.919, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.644, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.838, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.545, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.964, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.942, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.977, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.864, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.612, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.976, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.516, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.944, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.541, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.71, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.688, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.522, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.725, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.717, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.624, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.709, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.933, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.933, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.846, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.82, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.958, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.865, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.946, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.782, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.76, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.937, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Gender", + "value": "User Preferences", + "confidence": 0.906, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.818, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.858, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.56, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.626, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Sexual Orientation\",\n \"value\": \"Heterosexual\",\n \"confi" + }, + { + "type": "Industry", + "value": "Sexual Orientation", + "confidence": 0.539, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Sexual Orientation\",\n \"value\": \"Heterosexual\",\n \"confi" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.7, + "snippet": "ks, can you also provide your Sexual Orientation: Heterosexual?\\nTaylor: Can you check on that?\\nNoah: Let me lo" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.87, + "snippet": "h: Let me look\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Mozilla/5.0\",\n \"confid" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.942, + "snippet": "\"snippet\": \"ah: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\\nTaylor" + }, + { + "type": "User ID", + "value": "User Agent", + "confidence": 0.566, + "snippet": "\"snippet\": \"ah: Thanks, can you also provide your User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X)?\\nTaylor" + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.822, + "snippet": ": Here is my S\"\n },\n {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confide" + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.714, + "snippet": ": Here is my S\"\n },\n {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confide" + }, + { + "type": "Biometric Identifier", + "value": "Gender", + "confidence": 0.569, + "snippet": ": Here is my S\"\n },\n {\n \"type\": \"Gender\",\n \"value\": \"Non-binary\",\n \"confide" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0085.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0085.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..cf845b29 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0085.txt.redacted.redaction.json.redacted @@ -0,0 +1,165 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0085.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0085.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.917, + "snippet": "Transcript #85\n==================================\n<<>>: Hi, I need help with updating my subscription.\nM" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.834, + "snippet": "y: Hi, I need help with updating my subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.933, + "snippet": " subscription.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.857, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.931, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.861, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.895, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.87, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.892, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.852, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my I" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.932, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: 192.168.1.42\n<<>>: Thanks" + }, + { + "type": "<<>>", + "value": "192.168.1.42", + "confidence": 0.994, + "snippet": "ok that up for you.\n<<>>: Here is my <<>>: 192.168.1.42\n<<>>: Thanks, can you also provide your <<>>: jo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.766, + "snippet": "r you.\n<<>>: Here is my <<>>: 192.168.1.42\n<<>>: Thanks, can you also provide your <<>>: jordan@" + }, + { + "type": "<<>>", + "value": "jordan@example.com", + "confidence": 0.997, + "snippet": "42\n<<>>: Thanks, can you also provide your <<>>: jordan@example.com?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.852, + "snippet": " you also provide your <<>>: jordan@example.com?\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.868, + "snippet": "jordan@example.com?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: 0" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.993, + "snippet": "a: Thanks, can you also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.856, + "snippet": "also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.888, + "snippet": " 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.894, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0085.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0085.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..9e7a6df3 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0085.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,441 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0085.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0085.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.862, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.807, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.946, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.91, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.957, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.939, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.73, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.592, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.783, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.78, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.691, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.774, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.949, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.897, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.875, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.601, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.541, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.928, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.85, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.801, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.675, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.888, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.904, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.578, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.776, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.527, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.965, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.926, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.981, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.817, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.78, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.975, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.615, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.584, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.573, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.558, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.722, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.537, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.705, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.668, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.624, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.592, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.66, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.939, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.942, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.55, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.913, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.571, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.826, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.961, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.805, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.971, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.875, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.821, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.947, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.942, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.52, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.859, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.897, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Riley", + "confidence": 0.759, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.917,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.699, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.917,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.882, + "snippet": "bscription.\\nM\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": " + }, + { + "type": "User ID", + "value": "Maya", + "confidence": 0.776, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.834,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.773, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.834,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0086.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0086.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..d1cd88e8 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0086.txt.redacted.redaction.json.redacted @@ -0,0 +1,51 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0086.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0086.txt.redacted", + "totalRedactions": 1, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.989, + "snippet": "Thanks, can you also provide your <<>>: <<>>?\nAlex: Can you check on that?\nSofia: Let me look " + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0086.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0086.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..7fa6f96b --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0086.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,435 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0086.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0086.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.814, + "snippet": "cted\",\n \"totalRedactions\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.662, + "snippet": "cted\",\n \"totalRedactions\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.941, + "snippet": "ons\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.916, + "snippet": "ons\": 1,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.933, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.916, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.994, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.83, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.83, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.797, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.964, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.902, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.882, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.669, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.664, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Race/Ethnicity", + "value": "Industry", + "confidence": 0.938, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.885, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.869, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.832, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.585, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.895, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.939, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.784, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.8, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.558, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.953, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.934, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.561, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.984, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.888, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.829, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.98, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.962, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.509, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Religion", + "value": "Profile Picture Image", + "confidence": 0.699, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.616, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.662, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.605, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.682, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.608, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.684, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.657, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.776, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Religion", + "confidence": 0.541, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.504, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.957, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.964, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.59, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.899, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.509, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.86, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.966, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.877, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.899, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.734, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.666, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.956, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.926, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.68, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.753, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.56, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidenc" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.845, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.989,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.708, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.989,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.836, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\\nAlex: Can you check on that?\\nSofia: Let me loo" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.728, + "snippet": "Thanks, can you also provide your Race/Ethnicity: Hispanic?\\nAlex: Can you check on that?\\nSofia: Let me loo" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0087.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0087.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..6b3c8c46 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0087.txt.redacted.redaction.json.redacted @@ -0,0 +1,111 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0087.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0087.txt.redacted", + "totalRedactions": 2, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.783, + "snippet": "Transcript #87\n==================================\n<<>>: Hello—I'm having trouble with firmware update er" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.849, + "snippet": "o—I'm having trouble with firmware update errors.\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.739, + "snippet": "update errors.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.843, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.779, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.786, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.799, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.833, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.709, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.88, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.767, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0087.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0087.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..aa9e5101 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0087.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,459 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0087.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0087.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.868, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.818, + "snippet": "cted\",\n \"totalRedactions\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.953, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.865, + "snippet": "ons\": 2,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.959, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.93, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.993, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.533, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.832, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.782, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.743, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.811, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.947, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.501, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.898, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.878, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.66, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.944, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.527, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.873, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.831, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.766, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.886, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.898, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.631, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.799, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.504, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.962, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.927, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.978, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.859, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.851, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.965, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.564, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.955, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.643, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.636, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.541, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.554, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.518, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.504, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.714, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.695, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.649, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.604, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.673, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.709, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.564, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.513, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.935, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.946, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.53, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.93, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.705, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.814, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.815, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.536, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.962, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.848, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.795, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.953, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.936, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.536, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.851, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.88, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\"" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.583, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.783,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.82, + "snippet": "ware update er\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.574, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": 0.849,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0088.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0088.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..42e3da84 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0088.txt.redacted.redaction.json.redacted @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0088.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0088.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.892, + "snippet": ", can you also provide your <<>>: <<>>?\nSam: Can you check on that?\nSofia: Let me look t" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.971, + "snippet": "k that up for you.\nSam: Here is my <<>>: <<>>\nSofia: Let me look that up for you.\nSam: Here is " + }, + { + "type": "<<>>", + "value": "192.168.1.42", + "confidence": 0.997, + "snippet": "look that up for you.\nSam: Here is my <<>>: 192.168.1.42\nSofia: Thanks, can you also provide your Date of " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.977, + "snippet": " Thanks, can you also provide your <<>>: <<>>?\nSam: Can you check on that?\nSofia: Let me look t" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.59, + "snippet": " Thanks, can you also provide your <<>>: <<>>?\nSam: Can you check on that?\nSofia: Let me look t" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0088.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0088.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..87728cdd --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0088.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,471 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0088.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0088.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.807, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.716, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.919, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.933, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.933, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.757, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.798, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.758, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.688, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.793, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.948, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.522, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Credit Card Number", + "value": "Email", + "confidence": 0.501, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.932, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.701, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.884, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.717, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.599, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.951, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.877, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.856, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.631, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.865, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.935, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.684, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.677, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.855, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.602, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.97, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.92, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.821, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.817, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.985, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.951, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.595, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.608, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.555, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.672, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.626, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.544, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.716, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.558, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.645, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.643, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.643, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.523, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.653, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.941, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.951, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.845, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Biometric Identifier", + "value": "Sexual Orientation", + "confidence": 0.533, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.834, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.958, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.849, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.61, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.936, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.909, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.88, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.937, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.881, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.53, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.87, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Biometric Identifier\",\n \"value\": \"FaceID registered\",\n \"" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.919, + "snippet": "\"type\": \"Biometric Identifier\",\n \"value\": \"FaceID registered\",\n \"confidence\": 0.892,\n \"snippet\":" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.614, + "snippet": ",\n \"snippet\": \", can you also provide your Biometric Identifier: FaceID registered?\\nSam: Can you check on that?\\" + }, + { + "type": "Biometric Identifier", + "value": "FaceID registered", + "confidence": 0.962, + "snippet": ", can you also provide your Biometric Identifier: FaceID registered?\\nSam: Can you check on that?\\nSofia: Let me look" + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.957, + "snippet": " \"type\": \"Date of Birth\",\n \"value\": \"1990-05-12\",\n \"confidence\": 0.971,\n \"snippet\":" + }, + { + "type": "Credit Card Number", + "value": "1990-05-12", + "confidence": 0.826, + "snippet": " \"type\": \"Date of Birth\",\n \"value\": \"1990-05-12\",\n \"confidence\": 0.971,\n \"snippet\":" + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.954, + "snippet": " that up for you.\\nSam: Here is my Date of Birth: 1990-05-12\\nSofia: Let me look that up for you.\\nSam: Here i" + }, + { + "type": "Credit Card Number", + "value": "1990-05-12", + "confidence": 0.891, + "snippet": " that up for you.\\nSam: Here is my Date of Birth: 1990-05-12\\nSofia: Let me look that up for you.\\nSam: Here i" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.709, + "snippet": "nSam: Here is \"\n },\n {\n \"type\": \"IP Address\",\n \"value\": \"192.168.1.42\",\n \"confi" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0089.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0089.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..f4be8437 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0089.txt.redacted.redaction.json.redacted @@ -0,0 +1,153 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0089.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0089.txt.redacted", + "totalRedactions": 6, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.783, + "snippet": "Transcript #89\n==================================\n<<>>: Can you assist me with resetting my password.\nLi" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.741, + "snippet": "or: Can you assist me with resetting my password.\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.819, + "snippet": "g my password.\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: hiking, cooking\nL" + }, + { + "type": "<<>>", + "value": "hiking, cooking", + "confidence": 0.894, + "snippet": "p for you.\n<<>>: Here is my <<>>: hiking, cooking\n<<>>: Thanks, can you also provide your Race/Ethn" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.714, + "snippet": "r: Here is my <<>>: hiking, cooking\n<<>>: Thanks, can you also provide your <<>>" + }, + { + "type": "<<>>", + "value": "Hispanic", + "confidence": 0.984, + "snippet": "Thanks, can you also provide your <<>>: Hispanic?\n<<>>: Here is my <<>>: None\n<<>>: " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.768, + "snippet": "n you also provide your <<>>: Hispanic?\n<<>>: Here is my <<>>: None\n<<>>: Let me l" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.967, + "snippet": "y: Hispanic?\n<<>>: Here is my <<>>: None\n<<>>: Let me look that up for you.\n<<>>: Can yo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.71, + "snippet": "spanic?\n<<>>: Here is my <<>>: None\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.687, + "snippet": "mbership: None\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.712, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.698, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.769, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.693, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.728, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: 0" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.998, + "snippet": "m: Thanks, can you also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.755, + "snippet": "also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.676, + "snippet": "00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0089.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0089.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..695de5de --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0089.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,441 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0089.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0089.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.876, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.806, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.943, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.91, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.947, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.932, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.589, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.793, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.784, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.71, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.761, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.946, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.9, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.876, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.592, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.589, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.941, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.857, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.805, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.649, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.888, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.905, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.535, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.791, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.565, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.962, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.918, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.975, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.835, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.803, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.973, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.955, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.511, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.592, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.51, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.583, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.578, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.55, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.716, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.665, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.656, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.629, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.594, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.679, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.938, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.949, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.558, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.915, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.639, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.812, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.961, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.795, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.969, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.883, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.805, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.947, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.939, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.856, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.504, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.889, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "User ID", + "value": "Taylor", + "confidence": 0.755, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.783,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Taylor", + "confidence": 0.638, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Taylor\",\n \"confidence\": 0.783,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.85, + "snippet": " password.\\nLi\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": " + }, + { + "type": "User ID", + "value": "Liam", + "confidence": 0.762, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": 0.741,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Liam", + "confidence": 0.552, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Liam\",\n \"confidence\": 0.741,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0090.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0090.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..7f69b046 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0090.txt.redacted.redaction.json.redacted @@ -0,0 +1,111 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0090.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0090.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.807, + "snippet": "Transcript #90\n==================================\n<<>>: Hi, I need help with firmware update errors.\n<<>>" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.828, + "snippet": "<<>>: Hi, I need help with firmware update errors.\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.773, + "snippet": " update errors.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.818, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Union Membersh" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.983, + "snippet": "anks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Thanks, can you" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.781, + "snippet": "can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.809, + "snippet": "ion Membership: None?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: jordan@" + }, + { + "type": "<<>>", + "value": "jordan@example.com", + "confidence": 0.998, + "snippet": "at?\n<<>>: Thanks, can you also provide your <<>>: jordan@example.com?\n<<>>: Can you check on that?\n<<>>: Let me look tha" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.791, + "snippet": " you also provide your <<>>: jordan@example.com?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.842, + "snippet": ": jordan@example.com?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.796, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0090.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0090.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..143449e3 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0090.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,477 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0090.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0090.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.872, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.813, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.951, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.875, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.96, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.94, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.53, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.836, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.781, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.727, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.801, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.956, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.89, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.873, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.597, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.511, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.944, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.543, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.861, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.829, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.768, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.889, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.897, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.584, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.791, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.964, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.93, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.977, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.836, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.82, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.97, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.514, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.952, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.582, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.592, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.511, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.567, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.543, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.521, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.727, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.694, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.631, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.55, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.626, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.703, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.542, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.501, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.933, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.945, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.536, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.918, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.653, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.811, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.96, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.818, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.541, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.96, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.809, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.753, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.952, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.928, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.521, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.856, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.834, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Sam", + "confidence": 0.775, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.807,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.737, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.807,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.772, + "snippet": "e errors.\\nAva\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.77, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.828,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.699, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.828,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Sam", + "confidence": 0.761, + "snippet": "\n \"confidence\": 0.828,\n \"snippet\": \"Sam: Hi, I need help with firmware update errors.\\nAv" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.668, + "snippet": "\n \"confidence\": 0.828,\n \"snippet\": \"Sam: Hi, I need help with firmware update errors.\\nAv" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0091.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0091.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..3c849f03 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0091.txt.redacted.redaction.json.redacted @@ -0,0 +1,141 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0091.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0091.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.792, + "snippet": "Transcript #91\n==================================\n<<>>: Good morning, question about logging into my acc" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.773, + "snippet": " morning, question about logging into my account.\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.717, + "snippet": "to my account.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.796, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Union Membersh" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.902, + "snippet": " on that?\n<<>>: Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let m" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.967, + "snippet": "anks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me look" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.725, + "snippet": "can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.745, + "snippet": " Membership: None?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.69, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.72, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.829, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Approximate Geolocation: San Francisc" + }, + { + "type": "<<>>", + "value": "San Francisco, CA", + "confidence": 0.599, + "snippet": " you.\n<<>>: Here is my Approximate Geolocation: San Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Can yo" + }, + { + "type": "<<>>", + "value": "San Francisco, CA", + "confidence": 0.581, + "snippet": " you.\n<<>>: Here is my Approximate Geolocation: San Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Can yo" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.705, + "snippet": " is my Approximate Geolocation: San Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.716, + "snippet": " Francisco, CA\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.766, + "snippet": "k that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0091.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0091.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..709c4abd --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0091.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,441 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0091.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0091.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.841, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.816, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.944, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.91, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.96, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.946, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.669, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.989, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.575, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.544, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.778, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.768, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.719, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.768, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.946, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.89, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.872, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.634, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.579, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.938, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.861, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.815, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.622, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.888, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.907, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.507, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.778, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.531, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.962, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.925, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.803, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.792, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.972, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.578, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.592, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.563, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.546, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.535, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.721, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.518, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.797, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.665, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.599, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.63, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.661, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.936, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.948, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.925, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.602, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.813, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.961, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.802, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.944, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.85, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.838, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.946, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.94, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.513, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.856, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.865, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\"" + }, + { + "type": "User Agent", + "value": "Jordan", + "confidence": 0.796, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Jordan\",\n \"confidence\": 0.792,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.799, + "snippet": "ng into my acc\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": " + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.722, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.773,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Maya", + "confidence": 0.556, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.773,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0092.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0092.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..2982b26b --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0092.txt.redacted.redaction.json.redacted @@ -0,0 +1,129 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0092.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0092.txt.redacted", + "totalRedactions": 6, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.869, + "snippet": "Transcript #92\n==================================\n<<>>: Hi, I need help with firmware update errors.\nEth" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.886, + "snippet": "<<>>: Hi, I need help with firmware update errors.\n<<>>: Thanks, can you also provide your <<>>: None" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.679, + "snippet": " errors.\n<<>>: Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Thank" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.596, + "snippet": " errors.\n<<>>: Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Thank" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.863, + "snippet": "than: Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Thanks, can" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.877, + "snippet": "Thanks, can you also provide your <<>>: None?\n<<>>: Can you check on that?\n<<>>: Thanks, can you al" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.898, + "snippet": "our <<>>: None?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Sexual Orienta" + }, + { + "type": "<<>>", + "value": "Heterosexual", + "confidence": 0.975, + "snippet": "ks, can you also provide your <<>>: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Thanks, can" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.859, + "snippet": "so provide your <<>>: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Thanks, can you al" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.896, + "snippet": "tion: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: 19" + }, + { + "type": "<<>>", + "value": "192.168.1.42", + "confidence": 0.998, + "snippet": "an: Thanks, can you also provide your <<>>: 192.168.1.42?\n<<>>: Can you check on that?\n<<>>: Let me look" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.897, + "snippet": "n you also provide your <<>>: 192.168.1.42?\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.909, + "snippet": "ress: 192.168.1.42?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.877, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0092.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0092.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..e90b7abe --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0092.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,459 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0092.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0092.txt.redacted.redaction.json.redacted", + "totalRedactions": 37, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.87, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.816, + "snippet": "cted\",\n \"totalRedactions\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.953, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.872, + "snippet": "ons\": 6,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.964, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.924, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.992, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.836, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.795, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.685, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.8, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.954, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.501, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.9, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.87, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.634, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.889, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.598, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.858, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.814, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.782, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.906, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.917, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.64, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.804, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.968, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.938, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.978, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.833, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.83, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.973, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.585, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.955, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.582, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.618, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Religion", + "value": "Profile Picture Image", + "confidence": 0.569, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.509, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.554, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.522, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.735, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.704, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.613, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.581, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.637, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.833, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.675, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Religion", + "confidence": 0.513, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.932, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.945, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.548, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.909, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.833, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.819, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.973, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.843, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.774, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.958, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.94, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.564, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.861, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.813, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Riley", + "confidence": 0.752, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.869,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.57, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.869,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.743, + "snippet": "e errors.\\nEth\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\":" + }, + { + "type": "User ID", + "value": "Ethan", + "confidence": 0.713, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ethan\",\n \"confidence\": 0.886,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "ley", + "confidence": 0.66, + "snippet": "\n \"confidence\": 0.886,\n \"snippet\": \"ley: Hi, I need help with firmware update errors.\\nEt" + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.862, + "snippet": "errors.\\nEthan: Thanks, can you also provide your Religion: None\"\n },\n {\n \"type\": \"Sexual O" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.753, + "snippet": "errors.\\nEthan: Thanks, can you also provide your Religion: None\"\n },\n {\n \"type\": \"Sexual O" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0093.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0093.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..56418817 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0093.txt.redacted.redaction.json.redacted @@ -0,0 +1,87 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0093.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0093.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Profile Picture Image", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.63, + "snippet": "late delivery.\nMaya: Let me look that up for you.\n<<>>: Can you check on that?\nMaya: Thanks, can you als" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.972, + "snippet": "Maya: Thanks, can you also provide your <<>>: <<>>?\n<<>>: Can you check on that?\nMaya: Let me look" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.572, + "snippet": "check on that?\nMaya: Let me look that up for you.\n<<>>: Here is my <<>>: Independent\nMa" + }, + { + "type": "<<>>", + "value": "Independent", + "confidence": 0.978, + "snippet": "or you.\n<<>>: Here is my <<>>: Independent\nMaya: Let me look that up for you.\n<<>>: Can yo" + }, + { + "type": "<<>>", + "value": "Maya", + "confidence": 0.513, + "snippet": "or: Here is my <<>>: Independent\nMaya: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "Maya", + "confidence": 0.51, + "snippet": "k that up for you.\n<<>>: Can you check on that?\nMaya: Let me look that up for you.\n<<>>: Can you che" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.504, + "snippet": "check on that?\nMaya: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0093.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0093.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..1af62471 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0093.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,447 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0093.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0093.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.876, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.666, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.926, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.904, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.93, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.872, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.988, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.825, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.817, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.67, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.771, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Religion", + "value": "Driver's License", + "confidence": 0.525, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.95, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.901, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.87, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.623, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.564, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.958, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.902, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.888, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.604, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.821, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.905, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.625, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.53, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.869, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "User ID", + "value": "Name", + "confidence": 0.676, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.966, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.923, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.978, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.817, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.733, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.979, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.637, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.943, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.578, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.572, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.501, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.742, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.606, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.682, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.522, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.694, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.946, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.954, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.569, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.89, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.572, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.825, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.961, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.831, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.627, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "Name", + "value": "User ID", + "confidence": 0.796, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.705, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.924, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.839, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.762, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.773, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Username\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "User Preferences", + "value": "Taylor", + "confidence": 0.531, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Taylor\",\n \"confidence\": 0.63,\n \"snippet\": " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.825, + "snippet": "s, can you als\"\n },\n {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confiden" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.795, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.972,\n \"snippet\":" + }, + { + "type": "Name", + "value": "jordan_92", + "confidence": 0.681, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.972,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "jordan_92", + "confidence": 0.663, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"jordan_92\",\n \"confidence\": 0.972,\n \"snippet\":" + }, + { + "type": "Username", + "value": "jordan_92", + "confidence": 0.85, + "snippet": "Maya: Thanks, can you also provide your Username: jordan_92?\\nTaylor: Can you check on that?\\nMaya: Let me lo" + }, + { + "type": "User ID", + "value": "jordan_92", + "confidence": 0.519, + "snippet": "Maya: Thanks, can you also provide your Username: jordan_92?\\nTaylor: Can you check on that?\\nMaya: Let me lo" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.73, + "snippet": "a: Let me look\"\n },\n {\n \"type\": \"Username\",\n \"value\": \"Taylor\",\n \"confidence\"" + }, + { + "type": "User Preferences", + "value": "Taylor", + "confidence": 0.575, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Taylor\",\n \"confidence\": 0.572,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0094.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0094.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..6d25c40f --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0094.txt.redacted.redaction.json.redacted @@ -0,0 +1,177 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0094.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0094.txt.redacted", + "totalRedactions": 8, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "Profile Picture URL", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.534, + "snippet": "Transcript #94\n==================================\n<<>>: Hi, I need help with firmware update errors.\nEth" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.589, + "snippet": "<<>>: Hi, I need help with firmware update errors.\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.518, + "snippet": "pdate errors.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that u" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.858, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my Bi" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.538, + "snippet": "heck on that?\n<<>>: Let me look that up for you.\n<<>>: Here is my <<>>: FaceID register" + }, + { + "type": "<<>>", + "value": "FaceID registered", + "confidence": 0.945, + "snippet": "p for you.\n<<>>: Here is my <<>>: FaceID registered\n<<>>: Let me look that up for you.\n<<>>: Can you" + }, + { + "type": "<<>>", + "value": "FaceID registered", + "confidence": 0.759, + "snippet": "p for you.\n<<>>: Here is my <<>>: FaceID registered\n<<>>: Let me look that up for you.\n<<>>: Can you" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.872, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.871, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: jord" + }, + { + "type": "<<>>", + "value": "jordan_92", + "confidence": 0.945, + "snippet": "than: Thanks, can you also provide your <<>>: jordan_92?\n<<>>: Here is my <<>>: None\n<<>>: T" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.529, + "snippet": "s, can you also provide your <<>>: jordan_92?\n<<>>: Here is my <<>>: None\n<<>>: Thanks," + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.93, + "snippet": "me: jordan_92?\n<<>>: Here is my <<>>: None\n<<>>: Thanks, can you also provide your MAC Addr" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.756, + "snippet": "me: jordan_92?\n<<>>: Here is my <<>>: None\n<<>>: Thanks, can you also provide your MAC Addr" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.503, + "snippet": "ordan_92?\n<<>>: Here is my <<>>: None\n<<>>: Thanks, can you also provide your <<>>: 0" + }, + { + "type": "<<>>", + "value": "00:1B:44:11:3A:B7", + "confidence": 0.999, + "snippet": "n: Thanks, can you also provide your <<>>: 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.815, + "snippet": ": 00:1B:44:11:3A:B7?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.808, + "snippet": "ook that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your User Preferenc" + }, + { + "type": "<<>>", + "value": "marketing_emails=false", + "confidence": 0.902, + "snippet": "anks, can you also provide your <<>>: marketing_emails=false?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "marketing_emails=false", + "confidence": 0.565, + "snippet": "anks, can you also provide your <<>>: marketing_emails=false?\n<<>>: Can you check on that?\n<<>>: Thanks, can " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.86, + "snippet": "keting_emails=false?\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your <<>>: Mo" + }, + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.825, + "snippet": "an: Thanks, can you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?" + }, + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.536, + "snippet": "an: Thanks, can you also provide your <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0094.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0094.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..1dd830ff --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0094.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,417 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0094.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0094.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.91, + "snippet": "cted\",\n \"totalRedactions\": 8,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.883, + "snippet": "cted\",\n \"totalRedactions\": 8,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.936, + "snippet": "ons\": 8,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.893, + "snippet": "ons\": 8,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.962, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.848, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.989, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.857, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.809, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.715, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.815, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.975, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.563, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.944, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.905, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.672, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.557, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.936, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.845, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.784, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.746, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.836, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.922, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.721, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.858, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.97, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.925, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.977, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.848, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.8, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.974, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.623, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.948, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.59, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.615, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.736, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.698, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.594, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.729, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.524, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Religion", + "confidence": 0.517, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.936, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.953, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.585, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.9, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.681, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.836, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.963, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.829, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.617, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "Name", + "value": "User ID", + "confidence": 0.641, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.636, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.927, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.917, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.525, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.868, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.845, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Username\",\n \"value\": \"Alex\",\n \"confidence\": " + }, + { + "type": "Username", + "value": "Alex", + "confidence": 0.909, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Alex\",\n \"confidence\": 0.534,\n \"snippet\":" + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.824, + "snippet": "e errors.\\nEth\"\n },\n {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\":" + }, + { + "type": "Username", + "value": "Ethan", + "confidence": 0.866, + "snippet": " {\n \"type\": \"Username\",\n \"value\": \"Ethan\",\n \"confidence\": 0.589,\n \"snippet\":" + }, + { + "type": "Username", + "value": "lex", + "confidence": 0.779, + "snippet": "\n \"confidence\": 0.589,\n \"snippet\": \"lex: Hi, I need help with firmware update errors.\\nEt" + }, + { + "type": "Name", + "value": "lex", + "confidence": 0.501, + "snippet": "\n \"confidence\": 0.589,\n \"snippet\": \"lex: Hi, I need help with firmware update errors.\\nEt" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0095.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0095.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..ff0cb245 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0095.txt.redacted.redaction.json.redacted @@ -0,0 +1,75 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0095.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0095.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.995, + "snippet": " Thanks, can you also provide your <<>>: <<>>?\nSam: Here is my <<>>: <<>>\nLiam: L" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.994, + "snippet": "irth: <<>>?\nSam: Here is my <<>>: <<>>\nLiam: Let me look that up for you.\nSam: Can you c" + }, + { + "type": "<<>>", + "value": "None", + "confidence": 0.956, + "snippet": "anks, can you also provide your <<>>: None?\nSam: Can you check on that?\nLiam: Thanks, can yo" + }, + { + "type": "<<>>", + "value": "+1-202-555-0172", + "confidence": 0.999, + "snippet": "t?\nLiam: Thanks, can you also provide your <<>>: +1-202-555-0172?\nSam: Here is my <<>>: Mozilla/5.0 (Macinto" + }, + { + "type": "<<>>", + "value": "Mozilla/5.0", + "confidence": 0.893, + "snippet": "one: +1-202-555-0172?\nSam: Here is my <<>>: Mozilla/5.0 (Macintosh; Intel Mac OS X)\nLiam: Let me look tha" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0095.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0095.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..1ce158ba --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0095.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,525 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0095.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0095.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.851, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.843, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.923, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.913, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.975, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.937, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.832, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.702, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.859, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.684, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.796, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Sexual Orientation", + "value": "Driver's License", + "confidence": 0.599, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.963, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.664, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Credit Card Number", + "value": "Email", + "confidence": 0.554, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.9, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.564, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.853, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.664, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.529, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.842, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Race/Ethnicity", + "value": "Industry", + "confidence": 0.56, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.863, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.796, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.559, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.895, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.942, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.666, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.55, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.811, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.963, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.922, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.985, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.821, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.756, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.981, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.699, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.505, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.956, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Social Security Number", + "value": "Political Affiliation", + "confidence": 0.525, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Religion", + "value": "Profile Picture Image", + "confidence": 0.664, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.534, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.623, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.6, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.529, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.714, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.672, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.584, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.748, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Union Membership", + "value": "Region", + "confidence": 0.684, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.581, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.756, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Industry", + "value": "Religion", + "confidence": 0.549, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.943, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.953, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.888, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.723, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.594, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.838, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.955, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.92, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.811, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.77, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.702, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.945, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.922, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.794, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.831, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.746, + "snippet": " \"type\": \"Date of Birth\",\n \"value\": \"1990-05-12\",\n \"confidence\": 0.995,\n \"snippet\":" + }, + { + "type": "Date of Birth", + "value": "1990-05-12", + "confidence": 0.729, + "snippet": " Thanks, can you also provide your Date of Birth: 1990-05-12?\\nSam: Here is my Race/Ethnicity: Hispanic\\nLiam:" + }, + { + "type": "Page views", + "value": "1990-05-12", + "confidence": 0.545, + "snippet": " Thanks, can you also provide your Date of Birth: 1990-05-12?\\nSam: Here is my Race/Ethnicity: Hispanic\\nLiam:" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.505, + "snippet": " your Date of Birth: 1990-05-12?\\nSam: Here is my Race/Ethnicity: Hispanic\\nLiam: L\"\n },\n {\n \"typ" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.855, + "snippet": "rth: 1990-05-12?\\nSam: Here is my Race/Ethnicity: Hispanic\\nLiam: L\"\n },\n {\n \"type\": \"Race/" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.549, + "snippet": "rth: 1990-05-12?\\nSam: Here is my Race/Ethnicity: Hispanic\\nLiam: L\"\n },\n {\n \"type\": \"Race/" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.545, + "snippet": "panic\\nLiam: L\"\n },\n {\n \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidenc" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.657, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.994,\n \"snippet\":" + }, + { + "type": "Industry", + "value": "Hispanic", + "confidence": 0.55, + "snippet": " \"type\": \"Race/Ethnicity\",\n \"value\": \"Hispanic\",\n \"confidence\": 0.994,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.646, + "snippet": " \"snippet\": \"irth: 1990-05-12?\\nSam: Here is my Race/Ethnicity: Hispanic\\nLiam: Let me look that up for you.\\nSa" + }, + { + "type": "Race/Ethnicity", + "value": "Hispanic", + "confidence": 0.743, + "snippet": "rth: 1990-05-12?\\nSam: Here is my Race/Ethnicity: Hispanic\\nLiam: Let me look that up for you.\\nSam: Can you" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.903, + "snippet": "Sam: Can you c\"\n },\n {\n \"type\": \"Union Membership\",\n \"value\": \"None\",\n \"confidence\": " + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0096.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0096.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..e6424dc2 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0096.txt.redacted.redaction.json.redacted @@ -0,0 +1,123 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0096.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0096.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.976, + "snippet": "Transcript #96\n==================================\n<<>>: Good morning, question about logging into my acc" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.957, + "snippet": " morning, question about logging into my account.\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.973, + "snippet": "nto my account.\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.958, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.973, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.952, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.973, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.954, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you chec" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.96, + "snippet": " check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.96, + "snippet": "ok that up for you.\n<<>>: Can you check on that?\n<<>>: Thanks, can you also provide your Sexual Orienta" + }, + { + "type": "<<>>", + "value": "Heterosexual", + "confidence": 0.994, + "snippet": "ks, can you also provide your <<>>: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Let me look t" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.969, + "snippet": "so provide your <<>>: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Let me look that up " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.96, + "snippet": "tion: Heterosexual?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you." + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0096.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0096.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..56e30c2d --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0096.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,465 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0096.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0096.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.856, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.833, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.947, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.894, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.962, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.939, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.633, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.989, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.561, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.519, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.774, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.762, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.686, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.803, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.948, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.895, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.856, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.589, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.569, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.936, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.86, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.813, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.645, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.901, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.928, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.581, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.528, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.792, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.506, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.967, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.933, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.98, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.799, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.773, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.977, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.951, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.59, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.58, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.54, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.533, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.524, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.728, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Union Membership", + "value": "Profile URL", + "confidence": 0.501, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.717, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.667, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.657, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.581, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.673, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.939, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.947, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.518, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.909, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.617, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.827, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.959, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.806, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.965, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.877, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.843, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.948, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.942, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.539, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.867, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Name", + "value": "Username", + "confidence": 0.513, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.902, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\":" + }, + { + "type": "User Agent", + "value": "Riley", + "confidence": 0.816, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.976,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Riley", + "confidence": 0.719, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Riley\",\n \"confidence\": 0.976,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.867, + "snippet": "ng into my acc\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0" + }, + { + "type": "User Agent", + "value": "Ava", + "confidence": 0.788, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.957,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "Ava", + "confidence": 0.737, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Ava\",\n \"confidence\": 0.957,\n \"snippet\":" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0097.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0097.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..9d903fdc --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0097.txt.redacted.redaction.json.redacted @@ -0,0 +1,117 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0097.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0097.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.75, + "snippet": "Transcript #97\n==================================\n<<>>: Hi, I need help with firmware update errors.\nMay" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.753, + "snippet": "<<>>: Hi, I need help with firmware update errors.\n<<>>: Thanks, can you also provide your <<>>:" + }, + { + "type": "<<>>", + "value": "SN-1234567890", + "confidence": 0.998, + "snippet": " Thanks, can you also provide your <<>>: SN-1234567890?\n<<>>: Can you check on that?\n<<>>: Let me look th" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.681, + "snippet": "u also provide your <<>>: SN-1234567890?\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.657, + "snippet": "umber: SN-1234567890?\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.687, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.585, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.725, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.655, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.633, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.744, + "snippet": "look that up for you.\n<<>>: Can you check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.697, + "snippet": "check on that?\n<<>>: Let me look that up for you.\n<<>>: Can you check on that?" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0097.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0097.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..e9444c24 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0097.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,483 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0097.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0097.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.872, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.817, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.956, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.87, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.961, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.927, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.992, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Credit Card Number", + "value": "Date of Birth", + "confidence": 0.502, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.85, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.798, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.747, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.818, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.955, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.893, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.874, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.574, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.511, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.939, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Region", + "value": "Industry", + "confidence": 0.551, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.872, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Sexual Orientation", + "value": "Interaction Events", + "confidence": 0.839, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.791, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.902, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.912, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.633, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.792, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.968, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.924, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.978, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.846, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.845, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.971, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.503, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.955, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.612, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.661, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.533, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.574, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.527, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.731, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.691, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.608, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.517, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.64, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.7, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Sexual Orientation", + "value": "Religion", + "confidence": 0.518, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.96, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.905, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.637, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.931, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.626, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.824, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Social Security Number", + "confidence": 0.504, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.967, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Serial Number", + "value": "Tax Information", + "confidence": 0.54, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.818, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.557, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.958, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.84, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.779, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.959, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.934, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.866, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.782, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.778, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.75,\n \"snippet\": " + }, + { + "type": "User ID", + "value": "Sam", + "confidence": 0.749, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Sam\",\n \"confidence\": 0.75,\n \"snippet\": " + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.709, + "snippet": "e errors.\\nMay\"\n },\n {\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": " + }, + { + "type": "User ID", + "value": "Maya", + "confidence": 0.71, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.753,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Maya", + "confidence": 0.647, + "snippet": "{\n \"type\": \"User Agent\",\n \"value\": \"Maya\",\n \"confidence\": 0.753,\n \"snippet\":" + }, + { + "type": "User Agent", + "value": "Sam", + "confidence": 0.802, + "snippet": "\n \"confidence\": 0.753,\n \"snippet\": \"Sam: Hi, I need help with firmware update errors.\\nMa" + }, + { + "type": "User ID", + "value": "Sam", + "confidence": 0.724, + "snippet": "\n \"confidence\": 0.753,\n \"snippet\": \"Sam: Hi, I need help with firmware update errors.\\nMa" + }, + { + "type": "Tax Information", + "value": "Serial", + "confidence": 0.767, + "snippet": " errors.\\nMaya: Thanks, can you also provide your Serial Number:\"\n },\n {\n \"type\": \"Serial" + }, + { + "type": "Serial Number", + "value": "Serial", + "confidence": 0.693, + "snippet": " errors.\\nMaya: Thanks, can you also provide your Serial Number:\"\n },\n {\n \"type\": \"Serial" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0098.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0098.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..3287e6f0 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0098.txt.redacted.redaction.json.redacted @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0098.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0098.txt.redacted", + "totalRedactions": 4, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.988, + "snippet": "p for you.\nTaylor: Here is my <<>>: <<>>\nSofia: Let me look that up for you.\nTaylor: Can y" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.994, + "snippet": " can you also provide your <<>>: <<>>?\nTaylor: Here is my <<>>: <<>>" + }, + { + "type": "<<>>", + "value": "<<>>xample.com/jordan", + "confidence": 0.983, + "snippet": "ion: <<>>?\nTaylor: Here is my <<>>: <<>>xample.com/jordan\nSofia: Let me look that up for you.\nTaylor: Here " + }, + { + "type": "<<>>", + "value": "Hispanic", + "confidence": 0.963, + "snippet": "at up for you.\nTaylor: Here is my <<>>: Hispanic" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0098.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0098.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..a9ebcd23 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0098.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,453 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0098.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0098.txt.redacted.redaction.json.redacted", + "totalRedactions": 37, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.821, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.684, + "snippet": "cted\",\n \"totalRedactions\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.943, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.829, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Gender", + "value": "Biometric Identifier", + "confidence": 0.653, + "snippet": "ons\": 4,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.961, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.946, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.77, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.989, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Phone", + "value": "Date of Birth", + "confidence": 0.552, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.792, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.669, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.805, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.952, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.526, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.87, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "User Preferences", + "value": "Gender", + "confidence": 0.864, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Biometric Identifier", + "value": "Gender", + "confidence": 0.789, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.874, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.759, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.631, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.928, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.904, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.812, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.946, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.888, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.843, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.965, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.929, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.989, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.948, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.978, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Email", + "value": "Phone", + "confidence": 0.641, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Credit Card Number", + "value": "Phone", + "confidence": 0.592, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.935, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.636, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.689, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.672, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Industry", + "value": "Profile Picture URL", + "confidence": 0.59, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.526, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.655, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.538, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.753, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.7, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.696, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.61, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.706, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.94, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.955, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.924, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.851, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.954, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.878, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.91, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.725, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.697, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.94, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Gender", + "value": "User Preferences", + "confidence": 0.806, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.8, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.783, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.682, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"Sexual Orientation\",\n \"value\": \"Heterosexual\",\n \"confi" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.618, + "snippet": " \"type\": \"Sexual Orientation\",\n \"value\": \"Heterosexual\",\n \"confidence\": 0.988,\n \"snippet\":" + }, + { + "type": "Sexual Orientation", + "value": "Heterosexual", + "confidence": 0.774, + "snippet": " for you.\\nTaylor: Here is my Sexual Orientation: Heterosexual\\nSofia: Let me look that up for you.\\nTaylor: Can" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.914, + "snippet": "nTaylor: Can y\"\n },\n {\n \"type\": \"Political Affiliation\",\n \"value\": \"Independent\",\n \"confid" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.903, + "snippet": "4,\n \"snippet\": \" can you also provide your Political Affiliation: Independent?\\nTaylor: Here is my Profile URL: ht" + }, + { + "type": "Region", + "value": "Political Affiliation", + "confidence": 0.517, + "snippet": "4,\n \"snippet\": \" can you also provide your Political Affiliation: Independent?\\nTaylor: Here is my Profile URL: ht" + }, + { + "type": "Political Affiliation", + "value": "Independent", + "confidence": 0.673, + "snippet": " can you also provide your Political Affiliation: Independent?\\nTaylor: Here is my Profile URL: https://social." + }, + { + "type": "Union Membership", + "value": "https://social.e", + "confidence": 0.679, + "snippet": "on: Independent?\\nTaylor: Here is my Profile URL: https://social.e\"\n },\n {\n \"type\": \"Profile URL\",\n" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0099.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0099.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..aaf9e49c --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0099.txt.redacted.redaction.json.redacted @@ -0,0 +1,81 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0099.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0099.txt.redacted", + "totalRedactions": 5, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "marketing_emails=false", + "confidence": 0.915, + "snippet": "anks, can you also provide your <<>>: marketing_emails=false?\nTaylor: Can you check on that?\nMaya: Let me look" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.999, + "snippet": "t?\nMaya: Thanks, can you also provide your <<>>: <<>>?\nTaylor: Can you check on that?\nMaya: Let me look" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.58, + "snippet": ": Let me look that up for you.\nTaylor: Here is my <<>>: U123456\nMaya: Let me look that up for you.\nTaylo" + }, + { + "type": "<<>>", + "value": "U123456", + "confidence": 0.99, + "snippet": "look that up for you.\nTaylor: Here is my <<>>: U123456\nMaya: Let me look that up for you.\nTaylor: Here i" + }, + { + "type": "<<>>", + "value": "U123456", + "confidence": 0.991, + "snippet": "look that up for you.\nTaylor: Here is my <<>>: U123456\nMaya: Let me look that up for you.\nTaylor: Here i" + }, + { + "type": "<<>>", + "value": "Non-binary", + "confidence": 0.945, + "snippet": " look that up for you.\nTaylor: Here is my <<>>: Non-binary" + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0099.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0099.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..4e1a3626 --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0099.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,411 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0099.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0099.txt.redacted.redaction.json.redacted", + "totalRedactions": 35, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.857, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.802, + "snippet": "cted\",\n \"totalRedactions\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.892, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.887, + "snippet": "ons\": 5,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.967, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.913, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Phone", + "value": "Credit Card Number", + "confidence": 0.613, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.991, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.743, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Sexual Orientation", + "value": "Declared Interests", + "confidence": 0.736, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Interaction Events", + "value": "Declared Interests", + "confidence": 0.571, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.824, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.84, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.907, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.823, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.533, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.955, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.836, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Declared Interests", + "value": "Interaction Events", + "confidence": 0.808, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.888, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.938, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.656, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "IP Address", + "value": "MAC Address", + "confidence": 0.517, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.824, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.954, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.926, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.808, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.805, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.98, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.526, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Race/Ethnicity", + "value": "Profile Picture Image", + "confidence": 0.543, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.59, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.567, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Region", + "value": "Profile Picture URL", + "confidence": 0.558, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.726, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Region", + "value": "Profile URL", + "confidence": 0.545, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.629, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.618, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.523, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.594, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.944, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.949, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Tax Information", + "value": "Serial Number", + "confidence": 0.522, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.882, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.501, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.836, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.962, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.815, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "Account Number", + "value": "Union Membership", + "confidence": 0.632, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.895, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.672, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.668, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.956, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.771, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.815, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User Preferences\",\n \"value\": \"marketing_emails=false\",\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.838, + "snippet": " \"snippet\": \"anks, can you also provide your User Preferences: marketing_emails=false?\\nTaylor: Can you check o" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.605, + "snippet": "a: Let me look\"\n },\n {\n \"type\": \"Email\",\n \"value\": \"jordan@example.com\",\n " + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.858, + "snippet": " {\n \"type\": \"Email\",\n \"value\": \"jordan@example.com\",\n \"confidence\": 0.999,\n \"snippet\":" + }, + { + "type": "Email", + "value": "jordan@example.com", + "confidence": 0.86, + "snippet": "?\\nMaya: Thanks, can you also provide your Email: jordan@example.com?\\nTaylor: Can you check on that?\\nMaya: Let me lo" + } + ] + ] +} diff --git a/examples/chats/support_chats/redacted/redacted/chat_0100.txt.redacted.redaction.json.redacted b/examples/chats/support_chats/redacted/redacted/chat_0100.txt.redacted.redaction.json.redacted new file mode 100644 index 00000000..2c90e13f --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0100.txt.redacted.redaction.json.redacted @@ -0,0 +1,69 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/chat_0100.txt", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0100.txt.redacted", + "totalRedactions": 3, + "labels": [ + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>", + "<<>>" + ], + "matches": [ + [ + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.987, + "snippet": " look that up for you.\nCasey: Here is my <<>>: <<>>\nNoah: Thanks, can you also provide your Declared " + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.608, + "snippet": ": <<>>\nNoah: Thanks, can you also provide your <<>>: <<>>?\nCasey: Can you check on that?\nN" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.579, + "snippet": ": <<>>\nNoah: Thanks, can you also provide your <<>>: <<>>?\nCasey: Can you check on that?\nN" + }, + { + "type": "<<>>", + "value": "<<>>", + "confidence": 0.885, + "snippet": "ks, can you also provide your <<>>: <<>>?\nCasey: Can you check on that?\nNoah: Let me look " + } + ] + ] +} \ No newline at end of file diff --git a/examples/chats/support_chats/redacted/redacted/chat_0100.txt.redacted.redaction.json.redacted.redaction.json b/examples/chats/support_chats/redacted/redacted/chat_0100.txt.redacted.redaction.json.redacted.redaction.json new file mode 100644 index 00000000..58f9b8ee --- /dev/null +++ b/examples/chats/support_chats/redacted/redacted/chat_0100.txt.redacted.redaction.json.redacted.redaction.json @@ -0,0 +1,495 @@ +{ + "file": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/chat_0100.txt.redacted.redaction.json", + "outFile": "/Users/michaelfarrell/transcend/cli/examples/chats/support_chats/redacted/redacted/chat_0100.txt.redacted.redaction.json.redacted", + "totalRedactions": 36, + "labels": [ + "Account Number", + "Biometric Identifier", + "Credit Card Number", + "Date of Birth", + "Declared Interests", + "Driver's License", + "Email", + "Gender", + "Income", + "Industry", + "Interaction Events", + "IP Address", + "MAC Address", + "Name", + "Page views", + "Passport Number", + "Phone", + "Political Affiliation", + "Profile Picture Image", + "Profile Picture URL", + "Profile URL", + "Race/Ethnicity", + "Region", + "Religion", + "Routing Number", + "Serial Number", + "Sexual Orientation", + "Social Security Number", + "Tax Information", + "Union Membership", + "User Agent", + "User ID", + "User Preferences", + "Username" + ], + "matches": [ + [ + { + "type": "IP Address", + "value": "Account Number", + "confidence": 0.878, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "Account Number", + "value": "Account Number", + "confidence": 0.842, + "snippet": "cted\",\n \"totalRedactions\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Nu" + }, + { + "type": "User Preferences", + "value": "Biometric Identifier", + "confidence": 0.948, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Biometric Identifier", + "value": "Biometric Identifier", + "confidence": 0.866, + "snippet": "ons\": 3,\n \"labels\": [\n \"Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n" + }, + { + "type": "Credit Card Number", + "value": "Credit Card Number", + "confidence": 0.93, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Credit Card Number", + "confidence": 0.872, + "snippet": "Account Number\",\n \"Biometric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n" + }, + { + "type": "Date of Birth", + "value": "Date of Birth", + "confidence": 0.986, + "snippet": "etric Identifier\",\n \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.881, + "snippet": " \"Credit Card Number\",\n \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gende" + }, + { + "type": "Driver's License", + "value": "Driver's License", + "confidence": 0.85, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Religion", + "value": "Driver's License", + "confidence": 0.565, + "snippet": " \"Date of Birth\",\n \"Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"I" + }, + { + "type": "Email", + "value": "Email", + "confidence": 0.959, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Phone", + "value": "Email", + "confidence": 0.542, + "snippet": "Declared Interests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n " + }, + { + "type": "Gender", + "value": "Gender", + "confidence": 0.952, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Page views", + "value": "Gender", + "confidence": 0.566, + "snippet": "rests\",\n \"Driver's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction " + }, + { + "type": "Income", + "value": "Income", + "confidence": 0.888, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Interaction Events", + "value": "Income", + "confidence": 0.732, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Page views", + "value": "Income", + "confidence": 0.618, + "snippet": "river's License\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"" + }, + { + "type": "Industry", + "value": "Industry", + "confidence": 0.967, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "User ID", + "value": "Industry", + "confidence": 0.501, + "snippet": "e\",\n \"Email\",\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n " + }, + { + "type": "Interaction Events", + "value": "Interaction Events", + "confidence": 0.914, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "Income", + "value": "Interaction Events", + "confidence": 0.625, + "snippet": "\n \"Gender\",\n \"Income\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\"" + }, + { + "type": "IP Address", + "value": "IP Address", + "confidence": 0.871, + "snippet": "\",\n \"Industry\",\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\"" + }, + { + "type": "MAC Address", + "value": "MAC Address", + "confidence": 0.938, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Passport Number", + "value": "MAC Address", + "confidence": 0.512, + "snippet": "\n \"Interaction Events\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Num" + }, + { + "type": "Name", + "value": "Name", + "confidence": 0.836, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Username", + "value": "Name", + "confidence": 0.536, + "snippet": "vents\",\n \"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"P" + }, + { + "type": "Page views", + "value": "Page views", + "confidence": 0.972, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Income", + "value": "Page views", + "confidence": 0.943, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Interaction Events", + "value": "Page views", + "confidence": 0.697, + "snippet": "\"IP Address\",\n \"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Politi" + }, + { + "type": "Passport Number", + "value": "Passport Number", + "confidence": 0.979, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Political Affiliation", + "value": "Passport Number", + "confidence": 0.868, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "MAC Address", + "value": "Passport Number", + "confidence": 0.618, + "snippet": "\"MAC Address\",\n \"Name\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"" + }, + { + "type": "Phone", + "value": "Phone", + "confidence": 0.978, + "snippet": "e\",\n \"Page views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Pictu" + }, + { + "type": "Political Affiliation", + "value": "Political Affiliation", + "confidence": 0.954, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Passport Number", + "value": "Political Affiliation", + "confidence": 0.64, + "snippet": " views\",\n \"Passport Number\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Pictu" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture Image", + "confidence": 0.502, + "snippet": "\",\n \"Phone\",\n \"Political Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n " + }, + { + "type": "Profile Picture URL", + "value": "Profile Picture URL", + "confidence": 0.709, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile Picture URL", + "confidence": 0.702, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile Picture Image", + "value": "Profile Picture URL", + "confidence": 0.629, + "snippet": "l Affiliation\",\n \"Profile Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"R" + }, + { + "type": "Profile URL", + "value": "Profile URL", + "confidence": 0.751, + "snippet": "e Picture Image\",\n \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religi" + }, + { + "type": "User ID", + "value": "Race/Ethnicity", + "confidence": 0.914, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Race/Ethnicity", + "value": "Race/Ethnicity", + "confidence": 0.738, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "User Agent", + "value": "Race/Ethnicity", + "confidence": 0.708, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Industry", + "value": "Race/Ethnicity", + "confidence": 0.644, + "snippet": " \"Profile Picture URL\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Numb" + }, + { + "type": "Region", + "value": "Region", + "confidence": 0.641, + "snippet": "L\",\n \"Profile URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Seri" + }, + { + "type": "Religion", + "value": "Religion", + "confidence": 0.733, + "snippet": "le URL\",\n \"Race/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n " + }, + { + "type": "Routing Number", + "value": "Routing Number", + "confidence": 0.942, + "snippet": "ce/Ethnicity\",\n \"Region\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n" + }, + { + "type": "Serial Number", + "value": "Serial Number", + "confidence": 0.952, + "snippet": "gion\",\n \"Religion\",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security " + }, + { + "type": "Declared Interests", + "value": "Sexual Orientation", + "confidence": 0.615, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Sexual Orientation", + "value": "Sexual Orientation", + "confidence": 0.559, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Religion", + "value": "Sexual Orientation", + "confidence": 0.534, + "snippet": ",\n \"Routing Number\",\n \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Informat" + }, + { + "type": "Social Security Number", + "value": "Social Security Number", + "confidence": 0.828, + "snippet": " \"Serial Number\",\n \"Sexual Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n" + }, + { + "type": "Tax Information", + "value": "Tax Information", + "confidence": 0.961, + "snippet": " Orientation\",\n \"Social Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"" + }, + { + "type": "Union Membership", + "value": "Union Membership", + "confidence": 0.805, + "snippet": "ial Security Number\",\n \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Pref" + }, + { + "type": "User Agent", + "value": "User Agent", + "confidence": 0.88, + "snippet": " \"Tax Information\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Use" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.791, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Agent", + "value": "User ID", + "confidence": 0.708, + "snippet": "\",\n \"Union Membership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"" + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.931, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.883, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Biometric Identifier", + "value": "User Preferences", + "confidence": 0.645, + "snippet": "embership\",\n \"User Agent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n " + }, + { + "type": "Username", + "value": "Username", + "confidence": 0.84, + "snippet": "ent\",\n \"User ID\",\n \"User Preferences\",\n \"Username\"\n ],\n \"matches\": [\n [\n {\n \"type\"" + }, + { + "type": "User ID", + "value": "User ID", + "confidence": 0.505, + "snippet": " ],\n \"matches\": [\n [\n {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.791, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence\": 0.987,\n \"snippet\":" + }, + { + "type": "Race/Ethnicity", + "value": "U123456", + "confidence": 0.717, + "snippet": " {\n \"type\": \"User ID\",\n \"value\": \"U123456\",\n \"confidence\": 0.987,\n \"snippet\":" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.793, + "snippet": "look that up for you.\\nCasey: Here is my User ID: U123456\\nNoah: Thanks, can you also provide your Declared" + }, + { + "type": "Race/Ethnicity", + "value": "U123456", + "confidence": 0.736, + "snippet": "look that up for you.\\nCasey: Here is my User ID: U123456\\nNoah: Thanks, can you also provide your Declared" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.795, + "snippet": " \"type\": \"Declared Interests\",\n \"value\": \"Declared Interests\",\n \"confidence\": 0.608,\n \"snippet\":" + }, + { + "type": "User Preferences", + "value": "U123456", + "confidence": 0.679, + "snippet": " \"confidence\": 0.608,\n \"snippet\": \": U123456\\nNoah: Thanks, can you also provide your Declared" + }, + { + "type": "User ID", + "value": "U123456", + "confidence": 0.597, + "snippet": " \"confidence\": 0.608,\n \"snippet\": \": U123456\\nNoah: Thanks, can you also provide your Declared" + }, + { + "type": "Declared Interests", + "value": "Declared Interests", + "confidence": 0.742, + "snippet": " U123456\\nNoah: Thanks, can you also provide your Declared Interests: hiking, cooking?\\nCasey: Can you check on that?\\" + }, + { + "type": "Industry", + "value": "hiking, cooking", + "confidence": 0.924, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\\nCasey: Can you check on that?\\nN\"\n },\n " + }, + { + "type": "User ID", + "value": "hiking, cooking", + "confidence": 0.525, + "snippet": "ks, can you also provide your Declared Interests: hiking, cooking?\\nCasey: Can you check on that?\\nN\"\n },\n " + }, + { + "type": "User Preferences", + "value": "User Preferences", + "confidence": 0.86, + "snippet": "ck on that?\\nN\"\n },\n {\n \"type\": \"User Preferences\",\n \"value\": \"Declared Interests\",\n " + }, + { + "type": "User ID", + "value": "User Preferences", + "confidence": 0.713, + "snippet": "ck on that?\\nN\"\n },\n {\n \"type\": \"User Preferences\",\n \"value\": \"Declared Interests\",\n " + }, + { + "type": "Declared Interests", + "value": "User Preferences", + "confidence": 0.655, + "snippet": "ck on that?\\nN\"\n },\n {\n \"type\": \"User Preferences\",\n \"value\": \"Declared Interests\",\n " + } + ] + ] +} diff --git a/package.json b/package.json index 37f6de51..7a27a613 100644 --- a/package.json +++ b/package.json @@ -107,8 +107,8 @@ "@transcend-io/secret-value": "^1.2.0", "@transcend-io/type-utils": "^1.8.0", "JSONStream": "^1.3.5", - "cli-progress": "^3.11.2", "bluebird": "^3.7.2", + "cli-progress": "^3.11.2", "colors": "^1.4.0", "csv-parse": "^5.6.0", "fast-csv": "^4.3.6", @@ -129,12 +129,14 @@ "monocle-ts": "^2.3.13", "newtype-ts": "^0.3.5", "query-string": "=7.0.0", + "rimraf": "^6.0.1", "semver": "^7.6.0", "undici": "^5.22.1", "yargs-parser": "^21.1.1" }, "devDependencies": { "@types/JSONStream": "npm:@types/jsonstream@^0.8.33", + "@types/bluebird": "^3.5.38", "@types/cli-progress": "^3.11.0", "@types/colors": "^1.2.1", "@types/fuzzysearch": "^1.0.0", @@ -166,8 +168,7 @@ "tsx": "^4.20.3", "typescript": "^5.0.4", "vite-tsconfig-paths": "^5.1.4", - "vitest": "^3.2.4", - "@types/bluebird": "^3.5.38" + "vitest": "^3.2.4" }, "packageManager": "pnpm@10.12.4+sha512.5ea8b0deed94ed68691c9bad4c955492705c5eeb8a87ef86bc62c74a26b037b08ff9570f108b2e4dbd1dd1a9186fea925e527f141c648e85af45631074680184" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c1f7a79c..61080c81 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -104,6 +104,9 @@ importers: query-string: specifier: '=7.0.0' version: 7.0.0 + rimraf: + specifier: ^6.0.1 + version: 6.0.1 semver: specifier: ^7.6.0 version: 7.7.2 @@ -471,6 +474,14 @@ packages: resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==} deprecated: Use @eslint/object-schema instead + '@isaacs/balanced-match@4.0.1': + resolution: {integrity: sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==} + engines: {node: 20 || >=22} + + '@isaacs/brace-expansion@5.0.0': + resolution: {integrity: sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==} + engines: {node: 20 || >=22} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -1777,6 +1788,11 @@ packages: resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} hasBin: true + glob@11.0.3: + resolution: {integrity: sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA==} + engines: {node: 20 || >=22} + hasBin: true + glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} deprecated: Glob versions prior to v9 are no longer supported @@ -2097,6 +2113,10 @@ packages: jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jackspeak@4.1.1: + resolution: {integrity: sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==} + engines: {node: 20 || >=22} + joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} @@ -2255,6 +2275,10 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lru-cache@11.1.0: + resolution: {integrity: sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==} + engines: {node: 20 || >=22} + magic-string@0.30.17: resolution: {integrity: sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==} @@ -2361,6 +2385,10 @@ packages: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} + minimatch@10.0.3: + resolution: {integrity: sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==} + engines: {node: 20 || >=22} + minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -2554,6 +2582,10 @@ packages: resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} engines: {node: '>=16 || 14 >=14.18'} + path-scurry@2.0.0: + resolution: {integrity: sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==} + engines: {node: 20 || >=22} + path-type@4.0.0: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} @@ -2751,6 +2783,11 @@ packages: deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true + rimraf@6.0.1: + resolution: {integrity: sha512-9dkvaxAsk/xNXSJzMgFqqMCuFgt2+KsOFek3TMLfo8NCPfWpBmqwyNn5Y+NX56QUYfCtsyhF3ayiboEoUmJk/A==} + engines: {node: 20 || >=22} + hasBin: true + roarr@2.15.4: resolution: {integrity: sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==} engines: {node: '>=8.0'} @@ -3589,6 +3626,12 @@ snapshots: '@humanwhocodes/object-schema@2.0.3': {} + '@isaacs/balanced-match@4.0.1': {} + + '@isaacs/brace-expansion@5.0.0': + dependencies: + '@isaacs/balanced-match': 4.0.1 + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -5099,6 +5142,15 @@ snapshots: package-json-from-dist: 1.0.1 path-scurry: 1.11.1 + glob@11.0.3: + dependencies: + foreground-child: 3.3.1 + jackspeak: 4.1.1 + minimatch: 10.0.3 + minipass: 7.1.2 + package-json-from-dist: 1.0.1 + path-scurry: 2.0.0 + glob@7.2.3: dependencies: fs.realpath: 1.0.0 @@ -5455,6 +5507,10 @@ snapshots: optionalDependencies: '@pkgjs/parseargs': 0.11.0 + jackspeak@4.1.1: + dependencies: + '@isaacs/cliui': 8.0.2 + joycon@3.1.1: {} js-tokens@4.0.0: {} @@ -5583,6 +5639,8 @@ snapshots: lru-cache@10.4.3: {} + lru-cache@11.1.0: {} + magic-string@0.30.17: dependencies: '@jridgewell/sourcemap-codec': 1.5.4 @@ -5743,6 +5801,10 @@ snapshots: mimic-response@3.1.0: {} + minimatch@10.0.3: + dependencies: + '@isaacs/brace-expansion': 5.0.0 + minimatch@3.1.2: dependencies: brace-expansion: 1.1.12 @@ -5936,6 +5998,11 @@ snapshots: lru-cache: 10.4.3 minipass: 7.1.2 + path-scurry@2.0.0: + dependencies: + lru-cache: 11.1.0 + minipass: 7.1.2 + path-type@4.0.0: {} pathe@2.0.3: {} @@ -6121,6 +6188,11 @@ snapshots: dependencies: glob: 7.2.3 + rimraf@6.0.1: + dependencies: + glob: 11.0.3 + package-json-from-dist: 1.0.1 + roarr@2.15.4: dependencies: boolean: 3.2.0 diff --git a/src/commands/inventory/redact-unstructured-data/artifacts/collectMatchingFileOrExit.ts b/src/commands/inventory/redact-unstructured-data/artifacts/collectMatchingFileOrExit.ts new file mode 100644 index 00000000..4060ecce --- /dev/null +++ b/src/commands/inventory/redact-unstructured-data/artifacts/collectMatchingFileOrExit.ts @@ -0,0 +1,84 @@ +import { logger } from '../../../../logger'; +import type { LocalContext } from '../../../../context'; +import { join } from 'node:path'; +import { readdirSync, statSync } from 'node:fs'; + +const DEFAULT_GLOBS = [ + '**/*.txt', + '**/*.log', + '**/*.md', + '**/*.json', + '**/*.html', + '**/*.xml', + '**/*.csv', +]; + +/** + * Convert a glob pattern to a regex. + * + * @param glob - The glob pattern to convert. + * @returns - The regex that matches the glob pattern. + */ +function globToRegex(glob: string): RegExp { + // very small glob -> regex (supports **/, *, ?) + const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + const withStars = escaped + .replace(/\*\*\/?/g, '(?:.+/)?') + .replace(/\*/g, '[^/]*') + .replace(/\?/g, '.'); + return new RegExp(`^${withStars}$`); +} + +/** + * Walk a directory and return all files. + * + * @param dir - The directory to walk. + * @param files - The array to store files in. + * @param base - The base directory for relative paths. + * @returns - An array of file paths. + */ +function walk(dir: string, files: string[] = [], base = dir): string[] { + for (const ent of readdirSync(dir)) { + const p = join(dir, ent); + const s = statSync(p); + if (s.isDirectory()) walk(p, files, base); + else files.push(p); + } + return files; +} + +/** + * Collect matching files from a directory or exit. + * + * @param directory - The directory to search. + * @param ctx - The local context. + * @param globs - The glob patterns to match. + * @returns - An array of matching file paths. + */ +export function collectMatchingFilesOrExit( + directory: string, + ctx: LocalContext, + globs?: string[], +): string[] { + try { + const all = walk(directory); + const patterns = (globs && globs.length ? globs : DEFAULT_GLOBS).map( + globToRegex, + ); + const matches = all.filter((p) => { + const rel = p + .replace(directory.replace(/\/+$/, ''), '') + .replace(/^\/+/, ''); + return patterns.some((re) => re.test(rel)); + }); + if (!matches.length) { + logger.error(`No matching files found in ${directory}`); + ctx.process.exit(1); + } + return matches; + } catch (e) { + logger.error(`Failed reading directory ${directory}: ${e}`); + ctx.process.exit(1); + return []; + } +} diff --git a/src/commands/inventory/redact-unstructured-data/artifacts/index.ts b/src/commands/inventory/redact-unstructured-data/artifacts/index.ts new file mode 100644 index 00000000..61433a43 --- /dev/null +++ b/src/commands/inventory/redact-unstructured-data/artifacts/index.ts @@ -0,0 +1 @@ +export * from './collectMatchingFileOrExit'; diff --git a/src/commands/inventory/redact-unstructured-data/command.ts b/src/commands/inventory/redact-unstructured-data/command.ts new file mode 100644 index 00000000..fb73ebae --- /dev/null +++ b/src/commands/inventory/redact-unstructured-data/command.ts @@ -0,0 +1,82 @@ +// src/commands/classify/unstructured/command.ts +import { buildCommand } from '@stricli/core'; +import { + createAuthParameter, + createSombraAuthParameter, + createTranscendUrlParameter, +} from '../../../lib/cli/common-parameters'; +import { ScopeName } from '@transcend-io/privacy-types'; + +export const redactUnstructuredDataCommand = buildCommand({ + loader: async () => { + // impl will call a helper you provide to fetch data-category labels from Transcend + // e.g. getNerLabelsOrExit({ transcendUrl, auth, sombraAuth? }) + const { classifyUnstructured } = await import('./impl'); + return classifyUnstructured; + }, + parameters: { + flags: { + directory: { + kind: 'parsed', + parse: String, + brief: + 'Directory with unstructured text files to classify/redact (required)', + }, + outputDir: { + kind: 'parsed', + parse: String, + brief: + 'Directory to write redacted outputs (defaults to /redacted)', + optional: true, + }, + clearOutputDir: { + kind: 'boolean', + brief: 'Clear the output directory before writing', + default: false, + }, + + // ---- Transcend / Sombra auth & base URL ---- + transcendUrl: createTranscendUrlParameter(), + sombraAuth: createSombraAuthParameter(), // optional; only for self-hosted Sombra + auth: createAuthParameter({ + // Pick a conservative scope for read of classifications/config; + // adjust after you wire the helper. Silo scope likely required. + scopes: [ScopeName.ViewDataMap], // FIXME: narrow/expand per your labels API + requiresSiloScope: true, + }), + + // ---- Request shaping ---- + batchSize: { + kind: 'parsed', + parse: (v: string) => Math.max(1, Number(v) || 0), + brief: 'Number of text chunks to send per request (default 50)', + default: '50', + }, + + // ---- Concurrency / UI ---- + concurrency: { + kind: 'parsed', + parse: (v: string) => Math.max(1, Number(v) || 0), + brief: 'Max number of worker processes', + optional: true, + }, + viewerMode: { + kind: 'boolean', + brief: 'Non-interactive viewer mode', + default: false, + }, + writeSidecar: { + kind: 'boolean', + brief: 'Write a .redaction.json sidecar per output with match details', + default: true, + }, + }, + }, + docs: { + brief: + 'Classify & redact personal data in unstructured text files using Transcend', + fullDescription: + "Reads all text files in --directory, fetches data-category labels from Transcend, classifies content via Transcend's Named Entity Recognition endpoint, " + + 'replaces detected values with stable tokens, and writes redacted outputs to --outputDir. Files are processed concurrently across worker processes.', + }, +}); diff --git a/src/commands/inventory/redact-unstructured-data/impl.ts b/src/commands/inventory/redact-unstructured-data/impl.ts new file mode 100644 index 00000000..38891a89 --- /dev/null +++ b/src/commands/inventory/redact-unstructured-data/impl.ts @@ -0,0 +1,202 @@ +// src/commands/classify/unstructured/impl.ts +import type { LocalContext } from '../../../context'; +import colors from 'colors'; +import { logger } from '../../../logger'; +import { rimrafSync } from 'rimraf'; +import { join, resolve } from 'node:path'; +import { readdirSync, statSync } from 'node:fs'; + +import { + computePoolSize, + createExtraKeyHandler, + CHILD_FLAG, + type PoolHooks, + runPool, + dashboardPlugin, +} from '../../../lib/pooling'; + +import { + runChild, + type UnstructuredTask, + type UnstructuredProgress, + type UnstructuredResult, +} from './worker'; +import { classifyUnstructuredPlugin } from './ui'; +import { + buildTranscendGraphQLClient, + fetchAllDataCategories, +} from '../../../lib/graphql'; + +export type ClassifyUnstructuredFlags = { + directory: string; + outputDir?: string; + clearOutputDir: boolean; + + // Provided by common-parameters in command.ts + transcendUrl: string; + sombraAuth?: string; + auth: string; + + batchSize: number; + + concurrency?: number; + viewerMode: boolean; + writeSidecar: boolean; +}; + +type Totals = { + files: number; + redactions: number; + errors: number; +}; + +function getCurrentModulePath(): string { + // __filename is undefined in ESM; fall back to argv[1] + return typeof __filename !== 'undefined' + ? (__filename as unknown as string) + : process.argv[1]; +} + +const DEFAULT_EXTS = [ + '.txt', + '.log', + '.md', + '.markdown', + '.json', + '.html', + '.xml', + '.csv', + '.tsv', +]; + +function walkFiles(rootDir: string): string[] { + const out: string[] = []; + const stack = [resolve(rootDir)]; + while (stack.length) { + const dir = stack.pop()!; + for (const ent of readdirSync(dir)) { + const p = resolve(dir, ent); + const s = statSync(p); + if (s.isDirectory()) stack.push(p); + else out.push(p); + } + } + return out; +} + +function filterByExt(paths: string[]): string[] { + return paths.filter((p) => + DEFAULT_EXTS.some((ext) => p.toLowerCase().endsWith(ext)), + ); +} + +export async function classifyUnstructured( + this: LocalContext, + flags: ClassifyUnstructuredFlags, +): Promise { + const baseInputDir = resolve(flags.directory); + + // Discover files + const all = walkFiles(baseInputDir); + const files = filterByExt(all); + if (!files.length) { + logger.error( + colors.red(`No matching text-like files found in ${baseInputDir}`), + ); + this.process.exit(1); + } + + // Compute outputDir default: /redacted + const outputDir = resolve(flags.outputDir ?? join(baseInputDir, 'redacted')); + + // Optionally clear output directory ONCE (avoid per-worker races) + if (flags.clearOutputDir) { + try { + rimrafSync(outputDir); + } catch (e) { + logger.warn(`Failed to clear output directory ${outputDir}: ${e}`); + } + } + + // Resolve labels from Transcend + const client = buildTranscendGraphQLClient(flags.transcendUrl, flags.auth); + const dataCategories = await fetchAllDataCategories(client); + + // Size the pool + const { poolSize, cpuCount } = computePoolSize( + flags.concurrency, + files.length, + ); + + logger.info( + colors.green( + `Classifying & redacting ${files.length} file(s) with pool size ${poolSize} (CPU=${cpuCount})`, + ), + ); + + // Build FIFO queue + const queue: UnstructuredTask[] = files.map((filePath) => ({ + filePath, + options: { + baseInputDir, + outputDir, + auth: flags.auth, + sombraAuth: flags.sombraAuth, + transcendUrl: flags.transcendUrl, + dataCategories, + batchSize: flags.batchSize, + writeSidecar: flags.writeSidecar, + }, + })); + + // Pool hooks + const hooks: PoolHooks< + UnstructuredTask, + UnstructuredProgress, + UnstructuredResult, + Totals + > = { + nextTask: () => queue.shift(), + taskLabel: (t) => t.filePath, + initTotals: () => ({ files: files.length, redactions: 0, errors: 0 }), + initSlotProgress: () => undefined, + onProgress: (totals, prog) => { + if (prog?.redactionsDelta) totals.redactions += prog.redactionsDelta; + return totals; + }, + onResult: (totals, res) => { + if (!res.ok) totals.errors += 1; + return { totals, ok: !!res.ok }; + }, + postProcess: async () => { + // no-op + }, + }; + + // Launch pool + await runPool({ + title: 'Classify & Redact Unstructured', + baseDir: baseInputDir, + childFlag: CHILD_FLAG, + childModulePath: getCurrentModulePath(), + poolSize, + cpuCount, + filesTotal: files.length, + hooks, + viewerMode: flags.viewerMode, + render: (input) => + dashboardPlugin(input, classifyUnstructuredPlugin, flags.viewerMode), + extraKeyHandler: ({ logsBySlot, repaint, setPaused }) => + createExtraKeyHandler({ logsBySlot, repaint, setPaused }), + }); +} + +/* ------------------------------------------------------------------------------------------------- + * If invoked directly as a child process, enter worker loop + * ------------------------------------------------------------------------------------------------- */ +if (process.argv.includes(CHILD_FLAG)) { + runChild().catch((err) => { + logger.error(err); + process.exit(1); + }); +} diff --git a/src/commands/inventory/redact-unstructured-data/readme.ts b/src/commands/inventory/redact-unstructured-data/readme.ts new file mode 100644 index 00000000..0ae8b184 --- /dev/null +++ b/src/commands/inventory/redact-unstructured-data/readme.ts @@ -0,0 +1,67 @@ +// src/commands/classify/unstructured/readme.ts +import { buildExamples } from '../../../lib/docgen/buildExamples'; +import type { ClassifyUnstructuredFlags } from './impl'; + +const examples = buildExamples( + ['admin', 'classify-unstructured'], + [ + { + description: 'Redact a folder with defaults (output to /redacted)', + flags: { + directory: './working/unstructured', + auth: 'transcend-xxxx', + }, + }, + { + description: + 'Self-hosted Sombra: include Sombra auth and your gateway URL', + flags: { + directory: './working/unstructured', + outputDir: './working/redacted', + auth: 'transcend-xxxx', + sombraAuth: 'sombra-xxxx', + }, + }, + { + description: 'Use the US backend', + flags: { + directory: './working/unstructured', + transcendUrl: 'https://api.us.transcend.io', + auth: 'transcend-xxxx', + }, + }, + { + description: 'Increase batching and worker pool size', + flags: { + directory: './working/unstructured', + outputDir: './working/redacted', + auth: 'transcend-xxxx', + batchSize: 100, + concurrency: 4, + }, + }, + { + description: 'Write sidecar off (only redacted files)', + flags: { + directory: './working/unstructured', + transcendUrl: 'https://api.transcend.io', + auth: 'transcend-xxxx', + writeSidecar: false, + }, + }, + { + description: 'Viewer mode (non-interactive dashboard)', + flags: { + directory: './working/unstructured', + transcendUrl: 'https://api.transcend.io', + auth: 'transcend-xxxx', + viewerMode: true, + }, + }, + ], +); + +export default `#### Examples + +${examples} +`; diff --git a/src/commands/inventory/redact-unstructured-data/ui/index.ts b/src/commands/inventory/redact-unstructured-data/ui/index.ts new file mode 100644 index 00000000..1110b645 --- /dev/null +++ b/src/commands/inventory/redact-unstructured-data/ui/index.ts @@ -0,0 +1 @@ +export * from './plugin'; diff --git a/src/commands/inventory/redact-unstructured-data/ui/plugin.ts b/src/commands/inventory/redact-unstructured-data/ui/plugin.ts new file mode 100644 index 00000000..ea35d55e --- /dev/null +++ b/src/commands/inventory/redact-unstructured-data/ui/plugin.ts @@ -0,0 +1,41 @@ +import { + makeHeader, + makeWorkerRows, + type CommonCtx, + type DashboardPlugin, +} from '../../../../lib/pooling'; + +/** + * No progress + */ +export type UnstructuredSlotProgress = { + /** The file currently being processed. */ + filePath: string; + /** Bytes processed so far (approximate; based on chunk lengths). */ + processedBytes?: number; + /** Incremental number of redactions detected since previous tick. */ + redactionsDelta?: number; + /** Total bytes in the input file. */ + totalBytes?: number; +}; + +function renderHeader( + ctx: CommonCtx, +): string[] { + // Reuse shared header + return makeHeader(ctx); +} + +function renderWorkers( + ctx: CommonCtx, +): string[] { + return makeWorkerRows(ctx); +} + +export const classifyUnstructuredPlugin: DashboardPlugin< + unknown, + UnstructuredSlotProgress +> = { + renderHeader, + renderWorkers, +}; diff --git a/src/commands/inventory/redact-unstructured-data/worker.ts b/src/commands/inventory/redact-unstructured-data/worker.ts new file mode 100644 index 00000000..2cad0988 --- /dev/null +++ b/src/commands/inventory/redact-unstructured-data/worker.ts @@ -0,0 +1,422 @@ +import type { ToWorker } from '../../../lib/pooling'; +import { logger } from '../../../logger'; +import { readFile, writeFile } from 'node:fs/promises'; +import { mkdirSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { dirname, join, relative, resolve } from 'node:path'; +import { type Got, type HTTPError } from 'got'; +import { extractErrorMessage } from '../../../lib/helpers'; +import { + createSombraGotInstance, + type DataSubCategory, +} from '../../../lib/graphql'; + +/** + * A single file classification/redaction task sent to this worker. + */ +export type UnstructuredTask = { + /** Absolute or relative path of the UTF-8 text file to classify/redact. */ + filePath: string; + options: { + /** Root of input tree; used to mirror folder structure into outputDir. */ + baseInputDir: string; + /** Root of output tree; file paths are mirrored under this directory. */ + outputDir: string; + + /** Transcend/Sombra base URL, e.g. https://api.transcend.io or https://sombra… */ + transcendUrl: string; + /** Bearer token for Authorization header. */ + auth: string; + /** Optional Sombra auth header (x-sombra-authorization). */ + sombraAuth?: string; + + /** Data categories to classify for (used as labels). */ + dataCategories: DataSubCategory[]; + + /** How many paragraphs to classify per HTTP request. */ + batchSize: number; + + /** If true, also write a compact .redaction.json sidecar. */ + writeSidecar: boolean; + }; +}; + +/** + * Periodic progress snapshot emitted by the worker. + */ +export type UnstructuredProgress = { + /** The file currently being processed. */ + filePath: string; + /** Bytes processed so far (approximate; based on chunk lengths). */ + processedBytes?: number; + /** Incremental number of redactions detected since previous tick. */ + redactionsDelta?: number; + /** Total bytes in the input file. */ + totalBytes?: number; +}; + +/** + * Final result payload emitted by the worker when a task completes or fails. + */ +export type UnstructuredResult = { + ok: boolean; + filePath: string; + error?: string; +}; + +/** + * One Named-Entity-Recognition hit from the classifier. + */ +type NerHit = { + /** Category label (e.g., "Contact: Email"). */ + type: string; + /** Classifier confidence (0..1). */ + confidence: number; + /** Optional classifier metadata. */ + classificationMethod?: string; + /** Optional classifier version. */ + classifierVersion?: string; + /** Matched literal value in the text (exact substring). */ + value: string; + /** Optional short snippet for context. */ + snippet?: string; +}; + +/** API returns an array of hits for each input string. */ +type NerResponse = { + guesses: NerHit[][]; +}; + +/** + * Split a UTF-8 text buffer into paragraphs using blank lines as boundaries. + * + * We classify/redact by paragraph to improve precision and control payload size. + * + * @param text - The input text to split into paragraphs. + * @returns An array of paragraphs (strings). + */ +function splitIntoParagraphs(text: string): string[] { + const parts = text.split(/\r?\n\s*\r?\n/); + return parts.length ? parts : [text]; +} + +/** + * Build a deterministic replacement token for a (type, value) pair. + * Example: "__PD_EMAIL_1a2b3c4d__" + * + * @param type - The data category type (e.g., "Contact: Email"). + * @param value - The matched literal value (e.g., "example@example.com"). + * @returns A stable token string. + */ +function stableToken(type: string, value: string): string { + const typeSlug = type.replace(/\W+/g, '_').toUpperCase(); + const hash8 = createHash('sha1').update(value).digest('hex').slice(0, 8); + return `<<<${typeSlug}_${hash8}>>>`; +} + +/** + * Escape a literal string for use in a RegExp. + * + * @param value - The string to escape + * @returns The escaped string suitable for RegExp use + */ +function escapeForRegex(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * Join redacted paragraphs back together with the original spacing. + * + * @param chunks - The array of redacted paragraphs + * @returns The joined string with double newlines between paragraphs + */ +function joinParagraphs(chunks: string[]): string { + return chunks.join('\n\n'); +} + +/** + * POST a batch of inputs to Sombra's unstructured classifier endpoint with retries. + * + * @param opts - The options for the batch classification + * @returns The classification results + */ +async function classifyBatch(opts: { + sombra: Got; + inputs: string[]; + labels: string[]; +}): Promise { + const { sombra, inputs, labels } = opts; + + const res = await sombra + .post('classify/unstructured-text', { + json: { inputList: inputs, labels }, + retry: { + limit: 3, + methods: ['POST'], + statusCodes: [408, 413, 429, 500, 502, 503, 504], + calculateDelay: ({ attemptCount, error, computedValue }) => { + // Respect Retry-After if present + const ra = Number( + (error as HTTPError | undefined)?.response?.headers?.[ + 'retry-after' + ], + ); + if (Number.isFinite(ra) && ra > 0) return ra * 1000; + return computedValue ?? Math.min(2000 * attemptCount, 10_000); + }, + }, + timeout: { request: 60_000 }, + }) + .json(); + return res; +} + +/** + * Convert hrtime bigint to a friendly milliseconds number. + * + * @param startNs - The start time in hrtime bigint format + * @returns The elapsed time in milliseconds + */ +function msSince(startNs: bigint): number { + const deltaNs = process.hrtime.bigint() - startNs; + return Number(deltaNs / 1_000_000n); +} + +/** + * The worker entrypoint. Listens for "task" messages, performs classification + * and redaction, mirrors output paths, and emits progress + final result. + */ +export async function runChild(): Promise { + const workerId = Number(process.env.WORKER_ID || '0'); + + logger.info(`[unstructured:w${workerId}] worker online (pid=${process.pid})`); + process.send?.({ type: 'ready' }); + + process.on('message', async (msg: ToWorker) => { + if (!msg || typeof msg !== 'object') return; + if (msg.type === 'shutdown') process.exit(0); + if (msg.type !== 'task') return; + + const startedNs = process.hrtime.bigint(); + + const { filePath, options } = msg.payload; + const { + baseInputDir, + outputDir, + transcendUrl, + auth, + sombraAuth, + dataCategories, + batchSize, + writeSidecar, + } = options; + + // Prepare client (this may validate tokens and normalize the base URL) + const sombra = await createSombraGotInstance( + transcendUrl, + auth, + sombraAuth, + ); + + const labelStrings = dataCategories.map((cat) => cat.name).filter((x) => x); + + logger.info( + `[unstructured:w${workerId}] START file="${filePath}" labels=${labelStrings.length} batchSize=${batchSize}`, + ); + + try { + const absPath = resolve(filePath); + const raw = await readFile(absPath, 'utf8'); + const totalBytes = Buffer.byteLength(raw, 'utf8'); + + // Initial progress snapshot + process.send?.({ + type: 'progress', + payload: { filePath, totalBytes } as UnstructuredProgress, + }); + + // Split to improve precision & keep requests bounded + const chunks = splitIntoParagraphs(raw); + logger.info( + `[unstructured:w${workerId}] Read ${totalBytes.toLocaleString()}B; ${ + chunks.length + } paragraph(s)`, + ); + + // Running tallies for granular progress logging + const redactedChunks: string[] = []; + const matchesPerChunk: NerHit[][] = []; + let totalRedactions = 0; + let processedBytes = 0; + + // Classify in batches + let batchIndex = 0; + const classifyStartedNs = process.hrtime.bigint(); + + while (batchIndex < chunks.length) { + const sliceStart = batchIndex; + const batch = chunks.slice(batchIndex, batchIndex + batchSize); + + const httpStartNs = process.hrtime.bigint(); + let results: NerHit[][] = []; + try { + const raw = await classifyBatch({ + sombra, + inputs: batch, + labels: labelStrings, + }); + results = raw.guesses; + } catch (e) { + // Expand HTTP error detail when available + const httpErr = e as HTTPError; + const bodySnippet = + (httpErr?.response?.body && + String(httpErr.response.body).slice(0, 300)) || + ''; + throw new Error( + `classifyBatch failed (batch ${sliceStart}..${ + sliceStart + batch.length - 1 + }; ` + + `http=${httpErr?.response?.statusCode ?? 'n/a'}; ${msSince( + httpStartNs, + )}ms): ${extractErrorMessage(e)} ${ + bodySnippet ? `| body: ${bodySnippet}…` : '' + }`, + ); + } + + if (!Array.isArray(results) || results.length !== batch.length) { + logger.warn( + `[unstructured:w${workerId}] WARN: result size mismatch for batch starting at ${sliceStart}. ` + + `got=${ + Array.isArray(results) ? results.length : 'non-array' + } expected=${batch.length}`, + ); + } + + // Process each paragraph in the batch + for (let j = 0; j < batch.length; j += 1) { + const original = batch[j] ?? ''; + const byteLen = Buffer.byteLength(original, 'utf8'); + const hits = Array.isArray(results) ? results[j] ?? [] : []; + + matchesPerChunk.push(hits); + + // Replace literal matches with stable tokens + let redacted = original; + let redactionsThisChunk = 0; + + for (const hit of hits) { + // eslint-disable-next-line no-continue + if (!hit?.value || !hit?.type) continue; + const token = stableToken(hit.type, hit.value); + const safe = escapeForRegex(hit.value); + const before = redacted; + redacted = redacted.replace(new RegExp(safe, 'g'), token); + if (redacted !== before) redactionsThisChunk += 1; + } + + totalRedactions += redactionsThisChunk; + redactedChunks.push(redacted); + + processedBytes += byteLen; + // Emit a lightweight progress tick per paragraph (cheap) + process.send?.({ + type: 'progress', + payload: { + filePath, + processedBytes, + redactionsDelta: redactionsThisChunk, + } as UnstructuredProgress, + }); + } + + logger.info( + `[unstructured:w${workerId}] Batch ${sliceStart}-${ + sliceStart + batch.length - 1 + } ` + + `OK (${msSince( + httpStartNs, + )}ms). redactions+=${totalRedactions.toLocaleString()}`, + ); + + batchIndex += batch.length; + } + + logger.info( + `[unstructured:w${workerId}] Classified ${ + chunks.length + } paragraph(s) in ${msSince(classifyStartedNs)}ms. ` + + `Total redactions=${totalRedactions.toLocaleString()}`, + ); + + const redacted = joinParagraphs(redactedChunks); + + // Mirror directory under outputDir + const relDir = relative(baseInputDir, dirname(absPath)); + const outDir = resolve(join(outputDir, relDir || '')); + mkdirSync(outDir, { recursive: true }); + + const fileName = absPath.split('/').pop()!; + const outPath = resolve(join(outDir, `${fileName}.redacted`)); + await writeFile(outPath, redacted, 'utf8'); + + logger.info( + `[unstructured:w${workerId}] Wrote redacted file: ${outPath} (${processedBytes.toLocaleString()}/${totalBytes.toLocaleString()}B)`, + ); + + if (writeSidecar) { + const sidecarPath = resolve(`${outPath}.redaction.json`); + const sidecar = { + file: absPath, + outFile: outPath, + totalRedactions, + labels: labelStrings, + matches: matchesPerChunk.map((arr) => + arr.map((m) => ({ + type: m.type, + value: m.value, + confidence: m.confidence, + snippet: m.snippet, + })), + ), + }; + await writeFile(sidecarPath, JSON.stringify(sidecar, null, 2), 'utf8'); + logger.info( + `[unstructured:w${workerId}] Wrote sidecar: ${sidecarPath} (labels=${labelStrings.length})`, + ); + } + + const totalMs = msSince(startedNs); + logger.info( + `[unstructured:w${workerId}] DONE file="${filePath}" ` + + `bytes=${totalBytes.toLocaleString()} paragraphs=${ + chunks.length + } redactions=${totalRedactions.toLocaleString()} ` + + `in ${totalMs}ms`, + ); + + process.send?.({ + type: 'result', + payload: { ok: true, filePath } as UnstructuredResult, + }); + } catch (err) { + // Expand error message with context + const message = extractErrorMessage(err); + logger.error( + `[unstructured:w${workerId}] ERROR file="${filePath}": ${ + message || (err as Error)?.stack || err + }`, + ); + process.send?.({ + type: 'result', + payload: { ok: false, filePath, error: message } as UnstructuredResult, + }); + } + }); + + // Keep the worker alive until the parent kills it. + await new Promise(() => { + // This is a no-op; we just need to keep the event loop running. + }); +} diff --git a/src/commands/inventory/routes.ts b/src/commands/inventory/routes.ts index 39613782..de0c791d 100644 --- a/src/commands/inventory/routes.ts +++ b/src/commands/inventory/routes.ts @@ -7,6 +7,7 @@ import { pullUnstructuredDiscoveryFilesCommand } from './pull-unstructured-disco import { pullCommand } from './pull/command'; import { pushCommand } from './push/command'; import { scanPackagesCommand } from './scan-packages/command'; +import { redactUnstructuredDataCommand } from './redact-unstructured-data/command'; import { consentManagerServiceJsonToYmlCommand } from './consent-manager-service-json-to-yml/command'; import { consentManagersToBusinessEntitiesCommand } from './consent-managers-to-business-entities/command'; @@ -25,6 +26,7 @@ export const inventoryRoutes = buildRouteMap({ consentManagerServiceJsonToYmlCommand, 'consent-managers-to-business-entities': consentManagersToBusinessEntitiesCommand, + 'redact-unstructured-data': redactUnstructuredDataCommand, }, docs: { brief: 'Inventory commands',