From bf885c4c9520fe9d20119ca3db121a2f48d3956a Mon Sep 17 00:00:00 2001 From: Oleh Martsokha Date: Mon, 6 Jul 2026 11:57:36 +0200 Subject: [PATCH 1/3] Add custom patterns/dictionaries with configurable guardrails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Wire**: `PatternRecognizerParams` gains `custom` + `customDictionaries` slots for caller-inlined regex rules and literal-term dictionaries. New wire types in `nvisy-schema`: - `CustomPatternRule`, `CustomPatternVariant`, `CustomPatternContext` — regex-based rules mirroring `elide_pattern::Regex`. - `CustomDictionary`, `CustomDictionaryTerm` — literal-term rules mirroring `elide_pattern::Dictionary`. Types reuse elide's `LabelRef` and `Confidence` (both derive `JsonSchema` under elide-core's `schema` feature) — no lossy string/f32 mirror types. Re-exports `LabelRef` from `nvisy_schema::entity` and `Confidence` from `nvisy_schema::primitive` so SDK callers don't need `elide-core`. **Engine**: new `Engine::with_pattern_guardrails` sets a `PatternGuardrails` config that bounds runaway compile cost. Four knobs with conservative defaults: - `max_regex_source_len` (default 512) — enforced twice: at deserialize in `nvisy-schema` (`MAX_REGEX_SOURCE_LEN`) as a hard ceiling, and at engine compile time against the potentially-tightened deployment value. `with_pattern_guardrails` clamps to the wire ceiling on the way in. - `max_custom_rules` (default 32) — cap on total `custom.len() + customDictionaries.len()` per request. - `max_dictionary_term_count` (default 100 000) — aggregate across every dictionary in the recognizer, enforced by elide's `PatternRecognizerBuilder::with_term_count_limit`. - `max_dictionary_term_bytes` (default 8 MiB) — same aggregate, enforced by `with_term_bytes_limit`. Regex NFA/DFA size limits are deliberately NOT wired: elide's single-budget API can't separate per-regex from shared `RegexSet` union, and the two want very different bounds. See #317 for the split-budget follow-up. **Analyzer module reorg**: `analyzer/common.rs` split into three sibling subfolders that mirror the wire schema's shape: - `analyzer/recognizer/` — pattern, guardrails, ner, llm. - `analyzer/enricher/` — language (was in `common.rs`), ocr (was inline in `image.rs`), stt (was inline in `audio.rs`). - `analyzer/layer/` — dedup (was in `common.rs`). Per-modality entry files (text/tabular/image/audio) shrink to just orchestration. `PatternGuardrails` sits next to the pattern module in `recognizer/`. **Tests** (`custom_patterns.rs`): 7 covering the happy path (regex + dictionary matching the docx fixture), wire guardrails (deserialize rejects overlong source, analyze rejects too many rules), and configurability (tightened rule-count / source-len take effect, over-ceiling source-len clamps). --- Cargo.lock | 26 +- crates/nvisy-engine/src/analyzer/audio.rs | 50 +--- crates/nvisy-engine/src/analyzer/common.rs | 112 -------- .../src/analyzer/enricher/language.rs | 29 ++ .../nvisy-engine/src/analyzer/enricher/mod.rs | 19 ++ .../nvisy-engine/src/analyzer/enricher/ocr.rs | 40 +++ .../nvisy-engine/src/analyzer/enricher/stt.rs | 40 +++ crates/nvisy-engine/src/analyzer/image.rs | 40 +-- .../nvisy-engine/src/analyzer/layer/dedup.rs | 75 +++++ crates/nvisy-engine/src/analyzer/layer/mod.rs | 10 + crates/nvisy-engine/src/analyzer/mod.rs | 69 ++++- .../src/analyzer/recognizer/guardrails.rs | 94 +++++++ .../src/analyzer/{ => recognizer}/llm.rs | 0 .../src/analyzer/recognizer/mod.rs | 28 ++ .../src/analyzer/{ => recognizer}/ner.rs | 2 +- .../src/analyzer/recognizer/pattern.rs | 198 ++++++++++++++ crates/nvisy-engine/src/analyzer/tabular.rs | 13 +- crates/nvisy-engine/src/analyzer/text.rs | 10 +- crates/nvisy-engine/src/lib.rs | 1 + crates/nvisy-engine/src/pipeline/mod.rs | 21 +- .../nvisy-engine/src/pipeline/orchestrator.rs | 12 +- crates/nvisy-engine/tests/custom_patterns.rs | 256 ++++++++++++++++++ crates/nvisy-engine/tests/multimodal.rs | 1 + crates/nvisy-schema/src/entity.rs | 6 +- crates/nvisy-schema/src/plan/mod.rs | 5 + crates/nvisy-schema/src/plan/pattern.rs | 230 ++++++++++++++++ crates/nvisy-schema/src/plan/recognizer.rs | 26 +- crates/nvisy-schema/src/primitive.rs | 4 +- 28 files changed, 1185 insertions(+), 232 deletions(-) delete mode 100644 crates/nvisy-engine/src/analyzer/common.rs create mode 100644 crates/nvisy-engine/src/analyzer/enricher/language.rs create mode 100644 crates/nvisy-engine/src/analyzer/enricher/mod.rs create mode 100644 crates/nvisy-engine/src/analyzer/enricher/ocr.rs create mode 100644 crates/nvisy-engine/src/analyzer/enricher/stt.rs create mode 100644 crates/nvisy-engine/src/analyzer/layer/dedup.rs create mode 100644 crates/nvisy-engine/src/analyzer/layer/mod.rs create mode 100644 crates/nvisy-engine/src/analyzer/recognizer/guardrails.rs rename crates/nvisy-engine/src/analyzer/{ => recognizer}/llm.rs (100%) create mode 100644 crates/nvisy-engine/src/analyzer/recognizer/mod.rs rename crates/nvisy-engine/src/analyzer/{ => recognizer}/ner.rs (98%) create mode 100644 crates/nvisy-engine/src/analyzer/recognizer/pattern.rs create mode 100644 crates/nvisy-engine/tests/custom_patterns.rs create mode 100644 crates/nvisy-schema/src/plan/pattern.rs diff --git a/Cargo.lock b/Cargo.lock index 6969b37d..0bc5d9c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -839,7 +839,7 @@ checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "elide" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "async-trait", "elide-codec", @@ -875,7 +875,7 @@ dependencies = [ [[package]] name = "elide-codec" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "async-trait", "bytes", @@ -899,7 +899,7 @@ dependencies = [ [[package]] name = "elide-context" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "async-trait", "elide-core", @@ -910,7 +910,7 @@ dependencies = [ [[package]] name = "elide-core" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "async-trait", "bytes", @@ -928,7 +928,7 @@ dependencies = [ [[package]] name = "elide-detection" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "async-trait", "elide-core", @@ -941,7 +941,7 @@ dependencies = [ [[package]] name = "elide-lingua" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "async-trait", "elide-core", @@ -952,7 +952,7 @@ dependencies = [ [[package]] name = "elide-llm" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "async-trait", "derive_builder", @@ -972,7 +972,7 @@ dependencies = [ [[package]] name = "elide-ner" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "async-trait", "derive_builder", @@ -986,7 +986,7 @@ dependencies = [ [[package]] name = "elide-ocr" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "async-trait", "elide-core", @@ -996,7 +996,7 @@ dependencies = [ [[package]] name = "elide-orchestration" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "bytes", "elide-codec", @@ -1010,7 +1010,7 @@ dependencies = [ [[package]] name = "elide-pattern" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "aho-corasick", "async-trait", @@ -1030,7 +1030,7 @@ dependencies = [ [[package]] name = "elide-redaction" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "async-trait", "bytes", @@ -1045,7 +1045,7 @@ dependencies = [ [[package]] name = "elide-stt" version = "0.1.0" -source = "git+https://github.com/nvisycom/elide?branch=main#5bc8557ff10c0d9a28413a11a932981582f16314" +source = "git+https://github.com/nvisycom/elide?branch=main#b0fbfbade1f51610ec31e8a200a6fc1abc3ac76e" dependencies = [ "async-trait", "elide-core", diff --git a/crates/nvisy-engine/src/analyzer/audio.rs b/crates/nvisy-engine/src/analyzer/audio.rs index 67e2cc8b..584812c9 100644 --- a/crates/nvisy-engine/src/analyzer/audio.rs +++ b/crates/nvisy-engine/src/analyzer/audio.rs @@ -16,18 +16,22 @@ //! [`AnalyzerParams`]: nvisy_schema::plan::AnalyzerParams use elide::detection::Analyzer; +use elide_core::Error; use elide_core::modality::audio::Audio; -use elide_core::{Error, ErrorKind}; -#[cfg(feature = "test-utils")] -use elide_stt::{MockBackend as MockSttBackend, SttEnricher}; use nvisy_core::ner::NerConfig; -use nvisy_schema::plan::{AnalyzerParams, SttBackendParams, SttEnricherParams}; +use nvisy_schema::plan::AnalyzerParams; -use super::common::{attach_dedup, attach_pattern}; -use super::ner::attach_ner_lineup; +use super::PatternGuardrails; +use super::enricher::attach_stt; +use super::layer::attach_dedup; +use super::recognizer::{attach_ner_lineup, attach_pattern}; /// Compile `spec` into an audio-modality [`Analyzer`]. -pub(super) fn compile(spec: &AnalyzerParams, ner: &NerConfig) -> Result, Error> { +pub(super) fn compile( + spec: &AnalyzerParams, + ner: &NerConfig, + guardrails: &PatternGuardrails, +) -> Result, Error> { let mut analyzer = Analyzer::