ICU syntax parser

inc/validation/schema/project_template.json

@@ -87,6 +87,9 @@
     },
     "subfiltering_handlers": {
       "$ref": "subfiltering_handlers.json"
+    },
+    "icu_enabled": {
+      "type": "boolean"
     }
   },
   "required": [

lib/Controller/API/App/ConvertFileController.php

@@ -52,6 +52,7 @@ public function handle(): void
             $uploadDir,
             $errDir,
             $uploadTokenValue,
+            $data['icu_enabled'],
             $data['segmentation_rule'],
             $this->featureSet,
             $data['filters_extraction_parameters'],

lib/Controller/API/App/CreateProjectController.php

@@ -266,6 +266,8 @@ private function validateTheRequest(): array
         $deepl_formality = filter_var($this->request->param('deepl_formality'), FILTER_SANITIZE_SPECIAL_CHARS, ['flags' => FILTER_FLAG_STRIP_LOW]);
         $deepl_engine_type = filter_var($this->request->param('deepl_engine_type'), FILTER_SANITIZE_SPECIAL_CHARS, ['flags' => FILTER_FLAG_STRIP_LOW]);
 
+        $icu_enabled = filter_var($this->request->param('icu_enabled'), FILTER_VALIDATE_BOOLEAN);
+
         $array_keys = json_decode($private_keys_list, true);
         $array_keys = array_values(array_merge($array_keys['ownergroup'], $array_keys['mine'], $array_keys['anonymous']));
 
@@ -393,7 +395,7 @@ private function validateTheRequest(): array
             JobsMetadataDao::SUBFILTERING_HANDLERS => json_encode(
                 $this->validateSubfilteringOptions($this->request->param(JobsMetadataDao::SUBFILTERING_HANDLERS, '[]'))
             ),
-
+            'icu_enabled' => $icu_enabled,
         ];
 
         if ($disable_tms_engine_flag) {
@@ -520,6 +522,8 @@ private function setMetadataFromPostInput(array $data = []): void
             $options[MetadataDao::MT_QUALITY_VALUE_IN_EDITOR] = $data['mt_quality_value_in_editor'];
         }
 
+        $options[MetadataDao::ICU_ENABLED] = $data['icu_enabled'];
+
         $this->metadata = $options;
     }
 

lib/Controller/API/V1/NewController.php

@@ -111,6 +111,7 @@ public function create(): void
             $uploadDir,
             $errDir,
             $uploadTokenValue,
+            $request['icu_enabled'],
             $request['segmentation_rule'],
             $this->featureSet,
             $request['filters_extraction_parameters'],
@@ -376,6 +377,7 @@ private function validateTheRequest(): array
         $deepl_id_glossary = filter_var($this->request->param('deepl_id_glossary'), FILTER_SANITIZE_SPECIAL_CHARS, ['flags' => FILTER_FLAG_STRIP_LOW]);
         $deepl_formality = filter_var($this->request->param('deepl_formality'), FILTER_SANITIZE_SPECIAL_CHARS, ['flags' => FILTER_FLAG_STRIP_LOW]);
         $deepl_engine_type = filter_var($this->request->param('deepl_engine_type'), FILTER_SANITIZE_SPECIAL_CHARS, ['flags' => FILTER_FLAG_STRIP_LOW]);
+        $icu_enabled = filter_var($this->request->param('icu_enabled'), FILTER_VALIDATE_BOOLEAN);
 
         // Strip tags from instructions
         $instructions = [];
@@ -518,6 +520,8 @@ private function validateTheRequest(): array
             $metadata[MetadataDao::MT_EVALUATION] = true;
         }
 
+        $metadata[MetadataDao::ICU_ENABLED] = $icu_enabled;
+
         return [
             'project_info' => $project_info,
             'project_name' => $project_name,
@@ -571,7 +575,8 @@ private function validateTheRequest(): array
             'target_language_mt_engine_association' => $target_language_mt_engine_association,
             'mt_qe_workflow_payable_rate' => $mt_qe_PayableRate ?? null,
             'legacy_icu' => $legacy_icu,
-            JobsMetadataDao::SUBFILTERING_HANDLERS => json_encode($subfiltering_handlers)
+            JobsMetadataDao::SUBFILTERING_HANDLERS => json_encode($subfiltering_handlers),
+            'icu_enabled' => $icu_enabled,
         ];
     }
 

lib/Model/Conversion/ConversionHandler.php

@@ -48,6 +48,8 @@ class ConversionHandler
      */
     protected ?bool $legacy_icu;
 
+    protected bool $icu_enabled = false;
+
     /**
      * ConversionHandler constructor.
      */
@@ -148,7 +150,8 @@ public function processConversion(): void
             $single_language,
             $this->segmentation_rule,
             $extraction_parameters,
-            $this->legacy_icu
+            $this->icu_enabled,
+            $this->legacy_icu,
         );
         Filters::logConversionToXliff($convertResult, $file_path, $this->source_lang, $this->target_lang, $this->segmentation_rule, $extraction_parameters);
 
@@ -534,4 +537,10 @@ public function setFiltersLegacyIcu(?bool $legacy_icu = false): void
     {
         $this->legacy_icu = $legacy_icu;
     }
+
+    public function setIcuEnabled(bool $icu_enabled): void
+    {
+        $this->icu_enabled = $icu_enabled;
+    }
+
 }

lib/Model/Conversion/FilesConverter.php

@@ -49,6 +49,7 @@ class FilesConverter
      * @var bool|null
      */
     private ?bool $legacy_icu;
+    private bool $icu_enabled;
 
     /**
      * FilesConverter constructor.
@@ -71,6 +72,7 @@ public function __construct(
         string $intDir,
         string $errDir,
         string $uploadTokenValue,
+        bool $icu_enabled,
         ?string $segmentation_rule,
         FeatureSet $featureSet,
         ?FiltersConfigTemplateStruct $filters_extraction_parameters = null,
@@ -87,6 +89,7 @@ public function __construct(
         $this->featureSet = $featureSet;
         $this->filters_extraction_parameters = $filters_extraction_parameters;
         $this->legacy_icu = $legacy_icu;
+        $this->icu_enabled = $icu_enabled;
         $this->resultStack = new ConvertedFileList();
     }
 
@@ -187,6 +190,7 @@ private function getConversionHandlerInstance(string $fileName): ConversionHandl
         $conversionHandler->setFeatures($this->featureSet);
         $conversionHandler->setFiltersExtractionParameters($this->filters_extraction_parameters);
         $conversionHandler->setFiltersLegacyIcu($this->legacy_icu);
+        $conversionHandler->setIcuEnabled($this->icu_enabled);
 
         return $conversionHandler;
     }

lib/Model/Conversion/Filters.php

@@ -152,11 +152,12 @@ private static function extractInstanceInfoFromHeaders(array $headers): ?array
      * @param string $targetLang
      * @param string|null $segmentation
      * @param IDto|null $extractionParams
+     * @param bool $icu_enabled
      * @param bool|null $legacy_icu
      *
      * @return mixed
      */
-    public static function sourceToXliff(string $filePath, string $sourceLang, string $targetLang, ?string $segmentation = null, IDto $extractionParams = null, ?bool $legacy_icu = false): mixed
+    public static function sourceToXliff(string $filePath, string $sourceLang, string $targetLang, ?string $segmentation = null, IDto $extractionParams = null, bool $icu_enabled = false, ?bool $legacy_icu = false): mixed
     {
         $basename = AbstractFilesStorage::pathinfo_fix($filePath, PATHINFO_FILENAME);
         $extension = AbstractFilesStorage::pathinfo_fix($filePath, PATHINFO_EXTENSION);
@@ -176,6 +177,12 @@ public static function sourceToXliff(string $filePath, string $sourceLang, strin
             $extractionParams['escape_icu'] = true;
         }
 
+        /*
+         * icu_enabled = true => segment_icu = false
+         * icu_enabled = false => segment_icu = true (default)
+         */
+        $extractionParams['segment_icu'] = !$icu_enabled;
+
         if ($extractionParams !== null) {
             $data['extractionParams'] = json_encode($extractionParams);
         }

lib/Model/Projects/MetadataDao.php

@@ -29,6 +29,7 @@ class MetadataDao extends AbstractDao
     const string MT_QE_WORKFLOW_ENABLED = 'mt_qe_workflow_enabled';
     const string MT_QE_WORKFLOW_PARAMETERS = 'mt_qe_workflow_parameters';
     const string SUBFILTERING_HANDLERS = 'subfiltering_handlers';
+    const string ICU_ENABLED = 'icu_enabled';
 
     protected static string $_query_get_metadata = "SELECT * FROM project_metadata WHERE id_project = :id_project ";
 

lib/Model/Projects/ProjectTemplateDao.php

@@ -80,6 +80,7 @@ public static function getDefaultTemplate($uid): ProjectTemplateStruct
         $default->created_at = date("Y-m-d H:i:s");
         $default->modified_at = date("Y-m-d H:i:s");
         $default->subfiltering_handlers = json_encode(["markup", "twig", "double_snail", "double_square", "double_percent"]);
+        $default->icu_enabled = false;
 
         return $default;
     }
@@ -390,10 +391,62 @@ public static function getByIdAndUser(int $id, int $uid, int $ttl = 60): ?Projec
      */
     public static function save(ProjectTemplateStruct $projectTemplateStruct): ProjectTemplateStruct
     {
-        $sql = "INSERT INTO " . self::TABLE .
-            " ( `name`, `is_default`, `uid`, `id_team`, `segmentation_rule`, `tm`, `mt`, `payable_rate_template_id`,`qa_model_template_id`, `filters_template_id`, `xliff_config_template_id`, `pretranslate_100`, `pretranslate_101`, `tm_prioritization`, `dialect_strict`, `get_public_matches`, `public_tm_penalty`, `subject`, `source_language`, `target_language`, `character_counter_count_tags`, `character_counter_mode`, `mt_quality_value_in_editor`, `subfiltering_handlers`, `created_at` ) " .
-            " VALUES " .
-            " ( :name, :is_default, :uid, :id_team, :segmentation_rule, :tm, :mt, :payable_rate_template_id, :qa_model_template_id, :filters_template_id, :xliff_config_template_id, :pretranslate_100, :pretranslate_101, :tm_prioritization, :dialect_strict, :get_public_matches, :public_tm_penalty, :subject, :source_language, :target_language, :character_counter_count_tags, :character_counter_mode, :mt_quality_value_in_editor, :subfiltering_handlers, :now ); ";
+
+        $sql = "INSERT INTO " . self::TABLE . " (
+                    `name`,
+                    `is_default`,
+                    `uid`,
+                    `id_team`,
+                    `segmentation_rule`,
+                    `tm`,
+                    `mt`,
+                    `payable_rate_template_id`,
+                    `qa_model_template_id`,
+                    `filters_template_id`,
+                    `xliff_config_template_id`,
+                    `pretranslate_100`,
+                    `pretranslate_101`,
+                    `tm_prioritization`,
+                    `dialect_strict`,
+                    `get_public_matches`,
+                    `public_tm_penalty`,
+                    `subject`,
+                    `source_language`,
+                    `target_language`,
+                    `character_counter_count_tags`,
+                    `character_counter_mode`,
+                    `mt_quality_value_in_editor`,
+                    `subfiltering_handlers`,
+                    `created_at`,
+                    `icu_enabled`
+                ) VALUES (
+                    :name,
+                    :is_default,
+                    :uid,
+                    :id_team,
+                    :segmentation_rule,
+                    :tm,
+                    :mt,
+                    :payable_rate_template_id,
+                    :qa_model_template_id,
+                    :filters_template_id,
+                    :xliff_config_template_id,
+                    :pretranslate_100,
+                    :pretranslate_101,
+                    :tm_prioritization,
+                    :dialect_strict,
+                    :get_public_matches,
+                    :public_tm_penalty,
+                    :subject,
+                    :source_language,
+                    :target_language,
+                    :character_counter_count_tags,
+                    :character_counter_mode,
+                    :mt_quality_value_in_editor,
+                    :subfiltering_handlers,
+                    :now,
+                    :icu_enabled
+                ); ";
 
         $now = (new DateTime())->format('Y-m-d H:i:s');
 
@@ -425,6 +478,7 @@ public static function save(ProjectTemplateStruct $projectTemplateStruct): Proje
             "character_counter_count_tags" => $projectTemplateStruct->character_counter_count_tags,
             "character_counter_mode" => $projectTemplateStruct->character_counter_mode,
             'now' => (new DateTime())->format('Y-m-d H:i:s'),
+            'icu_enabled' => $projectTemplateStruct->icu_enabled
         ]);
 
         $projectTemplateStruct->id = $conn->lastInsertId();
@@ -475,7 +529,8 @@ public static function update(ProjectTemplateStruct $projectTemplateStruct): Pro
             `character_counter_count_tags` = :character_counter_count_tags,
             `character_counter_mode` = :character_counter_mode,
             `mt_quality_value_in_editor` = :mt_quality_value_in_editor,
-            `modified_at` = :now 
+            `modified_at` = :now,
+            `icu_enabled` = :icu_enabled
          WHERE id = :id;";
 
         $conn = Database::obtain()->getConnection();
@@ -507,6 +562,7 @@ public static function update(ProjectTemplateStruct $projectTemplateStruct): Pro
             "source_language" => $projectTemplateStruct->source_language,
             "target_language" => $projectTemplateStruct->target_language,
             'now' => (new DateTime())->format('Y-m-d H:i:s'),
+            'icu_enabled' => $projectTemplateStruct->icu_enabled
         ]);
 
         self::destroyQueryByIdCache($conn, $projectTemplateStruct->id);

lib/Model/Projects/ProjectTemplateStruct.php

@@ -38,6 +38,7 @@ class ProjectTemplateStruct extends AbstractDaoSilentStruct implements IDaoStruc
     public ?string $character_counter_mode = null;
     public ?string $subfiltering_handlers = null;
     public ?int $mt_quality_value_in_editor = null;
+    public bool $icu_enabled = false;
 
     /**
      * @param object $decodedObject
@@ -73,6 +74,7 @@ public function hydrateFromJSON(object $decodedObject, int $uid, ?int $id = null
         $this->source_language = $decodedObject->source_language;
         $this->target_language = (!empty($decodedObject->target_language)) ? serialize($decodedObject->target_language) : null;
         $this->mt_quality_value_in_editor = (!empty($decodedObject->mt_quality_value_in_editor)) ? (int)$decodedObject->mt_quality_value_in_editor : null;
+        $this->icu_enabled = $decodedObject->icu_enabled ?? false;
 
         return $this;
     }
@@ -167,6 +169,7 @@ public function jsonSerialize(): array
             'target_language' => $this->getTargetLanguage(),
             'created_at' => date_create($this->created_at)->format(DATE_RFC822),
             'modified_at' => date_create($this->modified_at)->format(DATE_RFC822),
+            'icu_enabled' => $this->icu_enabled
         ];
     }
 }

migrations/20260119173800_add_icu_enabled_to_project_templates.php

@@ -0,0 +1,16 @@
+<?php
+
+use migrations\AbstractMatecatMigration;
+
+class AddIcuEnabledToProjectTemplates extends AbstractMatecatMigration {
+
+    public $sql_up = [
+        "ALTER TABLE `project_templates` add column `icu_enabled` tinyint(1) DEFAULT 0, ALGORITHM=INPLACE, LOCK=NONE;",
+    ];
+
+
+    public $sql_down = [
+        'ALTER TABLE `project_templates` DROP COLUMN `icu_enabled`;',
+    ];
+
+}

package.json

@@ -25,14 +25,17 @@
     "diff-match-patch": "^1.0.5",
     "draft-js": "^0.11.4",
     "file-saver": "^2.0.5",
-    "flux": "^4.0.1",
+    "flux": "^4.0.4",
+    "format-message": "^6.2.4",
+    "format-message-parse": "^6.2.4",
     "immutability-helper": "^3.1.1",
     "immutable": "^5.0.0",
     "jquery": "^3.7.1",
     "js-base64": "^3.6.1",
     "js-cookie": "^3.0.0",
     "keymirror": "^0.1.1",
     "lodash": "^4.17.21",
+    "make-plural": "^8.1.0",
     "moment": "^2.29.4",
     "object-assign": "^4.1.1",
     "platform": "1.3.6",
@@ -64,6 +67,7 @@
     "@testing-library/react": "^16.0.0",
     "@testing-library/user-event": "^14.5.2",
     "babel-loader": "10.0.0",
+    "baseline-browser-mapping": "^2.9.15",
     "broadcast-channel": "^7.1.0",
     "css-loader": "7.1.2",
     "eslint": "^8.0.0",

public/css/sass/cattool.scss

@@ -1,5 +1,6 @@
 @use 'commons/mixins';
 @use 'commons/_colors';
+@use 'components/segment/IcuHighlight';
 body.cattool {
   //display: flex;
   height: 100vh;

public/css/sass/components/pages/CattoolPage.scss

@@ -29,6 +29,7 @@
 @use '../header/FilesMenu';
 @use '../header/ActionMenu';
 @use '../segment/segmentFooterTabAiAssistant';
+@use '../segment/segmentFooterTabIcu';
 @use '../CattolFooter';
 @use '../signin/OnBoarding';
 

public/css/sass/components/segment/IcuHighlight.scss

@@ -0,0 +1,27 @@
+@use '../../commons/colors';
+
+
+.icuItem {
+  display: inline-block;
+  position: relative;
+  color: colors.$linkBlue;
+  &.icuItem-error {
+    color: colors.$redDefault;
+    cursor: pointer;
+  }
+}
+
+.icu-tooltip {
+  padding: 6px;
+  max-width: 400px;
+  h3 {
+    font-size: 16px;
+    margin: 0;
+    margin-bottom: 8px;
+  }
+  p {
+    white-space: normal;
+    margin-bottom: 4px !important;
+  }
+}
+

public/css/sass/components/segment/segmentFooterTabIcu.scss

@@ -0,0 +1,6 @@
+.segment-footer-icu-container {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    height: 100%;
+}