Merge pull request #149 from 2Toad/jp-issue-92

Fixes #92: Add profane words for additional languages

Author: JasonPierce

.npmignore

@@ -11,4 +11,5 @@ tests/
 .nvmrc
 .prettierignore
 .prettierrc.json
+docker-compose.yml
 tsconfig.json

README.md

@@ -58,17 +58,17 @@ By default, this is set to `['en']` (English). You can change the default to any
 
 ```JavaScript
 const profanity = new Profanity({
-    languages: ["en", "de"],
+    languages: ['en', 'de'],
 });
 ```
 
 You can override this option by specifying the languages in `exists` or `censor`:
 
 ```JavaScript
-profanity.exists('Je suis un connard', ["fr"]);
+profanity.exists('Je suis un connard', ['fr']);
 // true
 
-profanity.censor('I like big butts and je suis un connard', CensorType.Word, ["en", "de", "fr"]);
+profanity.censor('I like big butts and je suis un connard', CensorType.Word, ['en', 'de', 'fr']);
 // I like big @#$%&! and je suis un @#$%&!
 ```
 

contribute.md

@@ -69,6 +69,40 @@ The Profanity project includes Husky for running Git Hooks. Running `git commit`
 
 - `chai`: we must use v4.x because v5.x is pure ESM, and we require CommonJS modules
 
+### Translations
+
+We utilize a self-hosted instance of the Open Source [LibreTranslate](https://github.com/LibreTranslate/LibreTranslate) lib to translate the core English list of profane words.
+
+#### Steps to Run Translations
+
+1. Open a terminal.
+2. Start Docker: `docker-compose up`.
+3. Run the translation script: `npm run translate`.
+
+#### Available Languages
+
+By default, the LibreTranslate service is configured to include all available target languages from [argos-translate](https://github.com/argosopentech/argos-translate). This configuration affects:
+
+- **Startup Time**: Initial service startup. Depending on your system, this can take ~5 minutes.
+- **Translation Time**: Translating across all languages increases processing time.
+- **Library Size**: The final size of the Profanity library.
+
+To optimize performance, we limit the target languages by configuring the `LT_LOAD_ONLY` environment variable:
+
+##### Configure Target Languages
+1. Open the `./docker-compose.yml` file.
+2. Add a comma-separated list of the [supported language codes](https://github.com/argosopentech/argos-translate/blob/master/argostranslate/languages.csv) you wish to include. Ensure English (`en`) is included, as it serves as the source language.
+
+**Example Configuration:**
+```yaml
+environment:
+  LT_LOAD_ONLY: "en,es,fr,de"
+```
+
+> **Note:** 
+> - To add a new language, remove existing language codes from `LT_LOAD_ONLY`
+> - To update existing languages after changes to the core English list, include their language codes in `LT_LOAD_ONLY`
+
 ### Deployment
 
 Deployments to Prod consist of building and publishing the Profanity lib to NPM, and are automated through our Continuous Deployment workflow.

docker-compose.yml

@@ -0,0 +1,24 @@
+version: "3"
+services:
+  libretranslate:
+    image: libretranslate/libretranslate:latest-cuda
+    ports:
+      - "5000:5000"
+    environment:
+      - LT_LOAD_ONLY=en,ar,zh,fr,de,hi,ja,ko,pt,ru,es
+      - NVIDIA_VISIBLE_DEVICES=all
+      - CUDA_LAUNCH_BLOCKING=0
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+        limits:
+          memory: 5G
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:5000/languages"]
+      interval: 30s
+      timeout: 10s
+      retries: 3

package-lock.json

@@ -22,7 +22,8 @@
     "lint:fix": "eslint . --fix",
     "format": "prettier . --write",
     "prepublishOnly": "npm run lint && npm test",
-    "prepare": "husky"
+    "prepare": "husky",
+    "translate": "ts-node src/scripts/translate.ts"
   },
   "repository": {
     "type": "git",
@@ -60,6 +61,7 @@
     "@types/node": "^22.5.2",
     "@typescript-eslint/eslint-plugin": "^8.4.0",
     "@typescript-eslint/parser": "^8.4.0",
+    "axios": "^1.7.9",
     "benchmark": "^2.1.4",
     "chai": "^4.5.0",
     "eslint": "^9.9.1",