diff --git a/apps/pii/engines.py b/apps/pii/engines.py deleted file mode 100644 index c8edf982ff1..00000000000 --- a/apps/pii/engines.py +++ /dev/null @@ -1,267 +0,0 @@ -"""Analyzer engine builders for the PII service. - -Two NER engines share one recognizer surface: - -- spacy (default): the 5 large spaCy models do NER (PERSON/LOCATION/NRP/ - DATE_TIME) and tokenization. -- gliner (opt-in): one multilingual GLiNER model does NER on CPU or GPU; - small spaCy models remain only for tokenization + lemmas. - -Both engines register the identical regex/checksum recognizer set (Presidio -defaults, EXTRA_RECOGNIZERS, VIN) — only the source of the 4 NER entity types -differs. Side-effect free: importing this module loads no models. -""" - -import importlib.util - -import spacy.util -from presidio_analyzer import AnalyzerEngine, Pattern, PatternRecognizer -from presidio_analyzer.nlp_engine import NlpEngineProvider -from presidio_analyzer.predefined_recognizers import ( - AuAbnRecognizer, - AuAcnRecognizer, - AuMedicareRecognizer, - AuTfnRecognizer, - EsNieRecognizer, - EsNifRecognizer, - FiPersonalIdentityCodeRecognizer, - GLiNERRecognizer, - InAadhaarRecognizer, - InPanRecognizer, - InPassportRecognizer, - InVehicleRegistrationRecognizer, - InVoterRecognizer, - ItDriverLicenseRecognizer, - ItFiscalCodeRecognizer, - ItIdentityCardRecognizer, - ItPassportRecognizer, - ItVatCodeRecognizer, - PlPeselRecognizer, - SgFinRecognizer, - SgUenRecognizer, - UkNinoRecognizer, -) - -# Languages served. Each needs its spaCy model installed in the image; the -# es/it/pl/fi predefined recognizers (ES_NIF, IT_FISCAL_CODE, PL_PESEL, ...) -# auto-load once their NLP engine is present. -NLP_CONFIGURATION = { - "nlp_engine_name": "spacy", - "models": [ - {"lang_code": "en", "model_name": "en_core_web_lg"}, - {"lang_code": "es", "model_name": "es_core_news_lg"}, - {"lang_code": "it", "model_name": "it_core_news_lg"}, - {"lang_code": "pl", "model_name": "pl_core_news_lg"}, - {"lang_code": "fi", "model_name": "fi_core_news_lg"}, - ], -} -SUPPORTED_LANGUAGES = [m["lang_code"] for m in NLP_CONFIGURATION["models"]] - -# The gliner engine still needs a spaCy pipeline per language: the regex -# recognizers consume NlpArtifacts and the LemmaContextAwareEnhancer boosts -# scores from surrounding lemmas. The small models (~12-40MB each vs ~400MB -# large) keep tokenization + lemmas intact while GLiNER owns NER. Blank -# pipelines ("blank:xx") are not an option: Presidio's SpacyNlpEngine treats -# unknown model names as pip packages and tries to download them. -# labels_to_ignore strips the small models' NER output from NlpArtifacts — -# correctness comes from removing SpacyRecognizer in build_gliner_analyzer; -# this only silences unmapped-label noise. -GLINER_NLP_CONFIGURATION = { - "nlp_engine_name": "spacy", - "models": [ - {"lang_code": "en", "model_name": "en_core_web_sm"}, - {"lang_code": "es", "model_name": "es_core_news_sm"}, - {"lang_code": "it", "model_name": "it_core_news_sm"}, - {"lang_code": "pl", "model_name": "pl_core_news_sm"}, - {"lang_code": "fi", "model_name": "fi_core_news_sm"}, - ], - "ner_model_configuration": { - "labels_to_ignore": [ - "CARDINAL", "DATE", "EVENT", "FAC", "GPE", "LANGUAGE", "LAW", - "LOC", "MISC", "MONEY", "NORP", "ORDINAL", "ORG", "PER", - "PERCENT", "PERSON", "PRODUCT", "QUANTITY", "TIME", "WORK_OF_ART", - ], - }, -} - -# Zero-shot label prompts -> the 4 Presidio NER entities GLiNER owns. Multiple -# prompts per entity trade a little inference cost for recall; tune against -# scripts/bench_engines.py output. -GLINER_ENTITY_MAPPING = { - "person": "PERSON", - "name": "PERSON", - "location": "LOCATION", - "address": "LOCATION", - "date": "DATE_TIME", - "time": "DATE_TIME", - "nationality": "NRP", - "religious group": "NRP", - "political group": "NRP", - "ethnic group": "NRP", -} - -# Predefined recognizers Presidio ships but does NOT load into the default -# registry — they must be added explicitly. Each carries its own -# supported_language, so it fires under that language once its NLP model is -# loaded. en: UK/AU/IN/SG locale ids; es/it/pl/fi: national ids. -EXTRA_RECOGNIZERS = [ - UkNinoRecognizer, - AuAbnRecognizer, - AuAcnRecognizer, - AuTfnRecognizer, - AuMedicareRecognizer, - InPanRecognizer, - InAadhaarRecognizer, - InVehicleRegistrationRecognizer, - InVoterRecognizer, - InPassportRecognizer, - SgFinRecognizer, - SgUenRecognizer, - EsNifRecognizer, - EsNieRecognizer, - ItFiscalCodeRecognizer, - ItDriverLicenseRecognizer, - ItVatCodeRecognizer, - ItPassportRecognizer, - ItIdentityCardRecognizer, - PlPeselRecognizer, - FiPersonalIdentityCodeRecognizer, -] - - -class VinRecognizer(PatternRecognizer): - """VIN (17 chars, A-Z/0-9 excluding I/O/Q) with ISO 3779 check-digit - validation (position 9). Validation makes accidental matches on arbitrary - 17-char codes (request ids, SKUs, tokens) extremely unlikely. Some - non-North-American VINs omit the check digit and are skipped — an - intentional bias toward precision. - """ - - _TRANSLIT = { - **{str(d): d for d in range(10)}, - "A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "H": 8, - "J": 1, "K": 2, "L": 3, "M": 4, "N": 5, "P": 7, "R": 9, - "S": 2, "T": 3, "U": 4, "V": 5, "W": 6, "X": 7, "Y": 8, "Z": 9, - } - _WEIGHTS = [8, 7, 6, 5, 4, 3, 2, 10, 0, 9, 8, 7, 6, 5, 4, 3, 2] - - def validate_result(self, pattern_text: str): - vin = pattern_text.upper() - if len(vin) != 17: - return False - try: - total = sum(self._TRANSLIT[c] * w for c, w in zip(vin, self._WEIGHTS)) - except KeyError: - return False - check = total % 11 - expected = "X" if check == 10 else str(check) - return vin[8] == expected - - -class SharedModelGLiNERRecognizer(GLiNERRecognizer): - """Per-language GLiNER recognizer sharing ONE loaded model. - - Presidio routes recognizers by supported_language, so the registry holds - one instance per served language — but each instance's load() would pull - its own ~1.2GB model copy. The first instance loads (an ImportError from - a missing gliner package propagates — fail fast in the lean image); the - rest reuse the cached model. - """ - - _shared_models: dict = {} - - def load(self) -> None: - key = (self.model_name, self.map_location) - cached = self._shared_models.get(key) - if cached is None: - super().load() - self._shared_models[key] = self.gliner - else: - self.gliner = cached - - def analyze(self, text, entities, nlp_artifacts=None): - """GLiNERRecognizer appends any requested entity it doesn't know as an - ad-hoc zero-shot label and returns its hits. The analyzer passes ALL - supported entities (~40) when a request doesn't narrow them, which - would prompt GLiNER for CREDIT_CARD/VIN/ES_NIF/... — wrong scope, and - inference cost scales with label count. Restrict to the NER entities - this recognizer owns.""" - requested = [e for e in (entities or self.supported_entities) if e in self.supported_entities] - if not requested: - return [] - return super().analyze(text, requested, nlp_artifacts) - - -def _register_common_recognizers(analyzer: AnalyzerEngine) -> None: - """Regex/checksum recognizers shared by both engines.""" - # VIN is language-agnostic, so register it under every served language — - # a recognizer only fires for the language the caller routes to. - vin_pattern = Pattern(name="vin", regex=r"\b[A-HJ-NPR-Z0-9]{17}\b", score=0.7) - for language in SUPPORTED_LANGUAGES: - analyzer.registry.add_recognizer( - VinRecognizer( - supported_entity="VIN", - patterns=[vin_pattern], - context=["vin", "vehicle", "chassis"], - supported_language=language, - ) - ) - for recognizer_cls in EXTRA_RECOGNIZERS: - analyzer.registry.add_recognizer(recognizer_cls()) - - -def build_spacy_analyzer() -> AnalyzerEngine: - nlp_engine = NlpEngineProvider(nlp_configuration=NLP_CONFIGURATION).create_engine() - analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=SUPPORTED_LANGUAGES) - _register_common_recognizers(analyzer) - return analyzer - - -def build_gliner_analyzer(model_name: str, device: str | None) -> AnalyzerEngine: - """GLiNER engine: one multilingual zero-shot model replaces spaCy NER for - PERSON/LOCATION/NRP/DATE_TIME; everything else is unchanged. - - :param model_name: HuggingFace id of the GLiNER model. - :param device: torch device ("cpu", "cuda", "cuda:0"); None auto-detects - via Presidio's device_detector (cuda when available, else cpu). - """ - # Fail fast with an actionable message when gliner deps are missing (e.g. - # a custom-built image without them). Without these checks Presidio would - # try to pip-download the missing spaCy models at startup (a silent - # network fallback that dies with an unrelated pip permission error), and - # the gliner ImportError would surface only later. - if importlib.util.find_spec("gliner") is None: - raise RuntimeError( - "PII_ENGINE=gliner but the gliner package is not installed; " - "use the stock pii image (docker/pii.Dockerfile ships torch + gliner)" - ) - missing = [ - m["model_name"] - for m in GLINER_NLP_CONFIGURATION["models"] - if not spacy.util.is_package(m["model_name"]) - ] - if missing: - raise RuntimeError( - f"PII_ENGINE=gliner needs spaCy models {missing}; " - "use the stock pii image (docker/pii.Dockerfile ships them)" - ) - nlp_engine = NlpEngineProvider(nlp_configuration=GLINER_NLP_CONFIGURATION).create_engine() - analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=SUPPORTED_LANGUAGES) - # The default registry wires SpacyRecognizer per language; with GLiNER - # owning the NER entities it would emit duplicate/competing spans from the - # small models' ner pipe. remove_recognizer only logs when nothing matched, - # so assert the removal actually happened. - analyzer.registry.remove_recognizer("SpacyRecognizer") - if any(r.name == "SpacyRecognizer" for r in analyzer.registry.recognizers): - raise RuntimeError("SpacyRecognizer removal failed; Presidio registry layout changed") - for language in SUPPORTED_LANGUAGES: - analyzer.registry.add_recognizer( - SharedModelGLiNERRecognizer( - entity_mapping=GLINER_ENTITY_MAPPING, - model_name=model_name, - map_location=device, - supported_language=language, - ) - ) - _register_common_recognizers(analyzer) - return analyzer diff --git a/apps/pii/requirements-dev.txt b/apps/pii/requirements-dev.txt deleted file mode 100644 index 68a538f5f1d..00000000000 --- a/apps/pii/requirements-dev.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Test-only deps. Unit tests need requirements.txt + this file (no models); -# integration tests additionally need the models baked into the docker images -# (see tests/test_integration.py). -pytest==9.0.3 -httpx==0.28.1 diff --git a/apps/pii/requirements-gliner.txt b/apps/pii/requirements-gliner.txt deleted file mode 100644 index a3d002f3850..00000000000 --- a/apps/pii/requirements-gliner.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Extras for the opt-in GLiNER engine — installed ONLY in the `gliner` -# Dockerfile target, on top of requirements.txt. Pinned for reproducible image -# builds; bump deliberately. presidio-analyzer 2.2.362 requires -# gliner >=0.2.13,<1.0.0. -# -# torch is pinned in the Dockerfile instead: the CPU and CUDA targets install -# the same version from different wheel indexes. -gliner==0.2.27 -transformers==5.3.0 -huggingface_hub==1.3.0 diff --git a/apps/pii/scripts/bench_engines.py b/apps/pii/scripts/bench_engines.py deleted file mode 100644 index b7ea70ca764..00000000000 --- a/apps/pii/scripts/bench_engines.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Benchmark + parity harness for the spacy vs gliner NER engines. - -Runs the same payload through both engines and reports per-engine throughput -(batch analyze, the production /redact_batch path) and per-text latency, plus -an accuracy diff over the 4 NER entity types (PERSON/LOCATION/NRP/DATE_TIME). -Non-NER (regex/checksum) results must be identical between engines — both -register the same recognizers — so any mismatch there is a wiring bug and the -script exits non-zero. - -Meant to run inside the pii image (both engines ship in it): - - docker run --rm python scripts/bench_engines.py - docker run --rm -v $PWD/texts.json:/data.json \\ - python scripts/bench_engines.py --payload /data.json - -Payload format: JSON list of {"text": str, "language": str} objects. -This doubles as the tuning harness for GLINER_ENTITY_MAPPING label prompts. -""" - -import argparse -import json -import statistics -import sys -import time -from collections import defaultdict -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -import engines # noqa: E402 - -# Entities sourced from the NER models rather than regex/checksum patterns. -# ORGANIZATION is emitted by the spacy engine's NER on unfiltered requests but -# is not in the app's supported set and has no GLiNER mapping — it shows up in -# the NER diff (spacy-only) rather than failing the regex-parity gate. -NER_ENTITIES = {"PERSON", "LOCATION", "NRP", "DATE_TIME", "ORGANIZATION"} -DEFAULT_PAYLOAD = Path(__file__).resolve().parent / "bench_payload.json" - - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--payload", type=Path, default=DEFAULT_PAYLOAD) - parser.add_argument("--engines", default="spacy,gliner") - parser.add_argument("--runs", type=int, default=3) - parser.add_argument("--warmup", type=int, default=1) - parser.add_argument("--device", default=None, help="torch device for gliner (default: auto)") - parser.add_argument("--gliner-model", default="urchade/gliner_multi_pii-v1") - parser.add_argument("--max-examples", type=int, default=10) - parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") - return parser.parse_args() - - -def build(engine: str, args) -> tuple: - started = time.perf_counter() - if engine == "spacy": - analyzer = engines.build_spacy_analyzer() - elif engine == "gliner": - analyzer = engines.build_gliner_analyzer(model_name=args.gliner_model, device=args.device) - else: - raise ValueError(f"Unknown engine {engine!r}") - return analyzer, time.perf_counter() - started - - -def analyze_all(analyzer, items) -> list[list]: - """One analyze() call per text, in payload order.""" - return [analyzer.analyze(text=item["text"], language=item["language"]) for item in items] - - -def bench(analyzer, items, runs: int, warmup: int) -> dict: - for _ in range(warmup): - analyze_all(analyzer, items) - run_times = [] - latencies = [] - for _ in range(runs): - run_started = time.perf_counter() - for item in items: - text_started = time.perf_counter() - analyzer.analyze(text=item["text"], language=item["language"]) - latencies.append(time.perf_counter() - text_started) - run_times.append(time.perf_counter() - run_started) - total_chars = sum(len(item["text"]) for item in items) - avg_run = statistics.mean(run_times) - return { - "texts_per_sec": len(items) / avg_run, - "chars_per_sec": total_chars / avg_run, - "latency_p50_ms": statistics.median(latencies) * 1000, - "latency_p95_ms": statistics.quantiles(latencies, n=20)[18] * 1000, - } - - -def spans(results, keep_ner: bool) -> set: - return { - (r.entity_type, r.start, r.end) - for r in results - if (r.entity_type in NER_ENTITIES) == keep_ner - } - - -def iou(a: tuple, b: tuple) -> float: - inter = max(0, min(a[2], b[2]) - max(a[1], b[1])) - union = max(a[2], b[2]) - min(a[1], b[1]) - return inter / union if union else 0.0 - - -def diff_ner(items, results_a, results_b, max_examples: int) -> dict: - """Per-entity-type agreement between two engines (span IoU >= 0.5).""" - per_type = defaultdict(lambda: {"a_total": 0, "b_total": 0, "matched": 0}) - examples = [] - for item, res_a, res_b in zip(items, results_a, results_b): - a = sorted(spans(res_a, keep_ner=True)) - b = sorted(spans(res_b, keep_ner=True)) - unmatched_b = set(b) - for span_a in a: - per_type[span_a[0]]["a_total"] += 1 - match = next( - (s for s in unmatched_b if s[0] == span_a[0] and iou(span_a, s) >= 0.5), None - ) - if match: - per_type[span_a[0]]["matched"] += 1 - unmatched_b.discard(match) - for span_b in b: - per_type[span_b[0]]["b_total"] += 1 - only_a = [s for s in a if not any(s[0] == t[0] and iou(s, t) >= 0.5 for t in b)] - only_b = sorted(unmatched_b) - if (only_a or only_b) and len(examples) < max_examples: - examples.append( - { - "text": item["text"], - "language": item["language"], - "only_a": [f"{t}[{s}:{e}]={item['text'][s:e]!r}" for t, s, e in only_a], - "only_b": [f"{t}[{s}:{e}]={item['text'][s:e]!r}" for t, s, e in only_b], - } - ) - return {"per_type": dict(per_type), "examples": examples} - - -def diff_regex(items, results_a, results_b) -> list: - """Non-NER results must be identical: same recognizers on both engines.""" - mismatches = [] - for item, res_a, res_b in zip(items, results_a, results_b): - a = spans(res_a, keep_ner=False) - b = spans(res_b, keep_ner=False) - if a != b: - mismatches.append({"text": item["text"], "only_a": sorted(a - b), "only_b": sorted(b - a)}) - return mismatches - - -def main() -> int: - args = parse_args() - items = json.loads(args.payload.read_text()) - engine_names = [e.strip() for e in args.engines.split(",") if e.strip()] - - report = {"payload": str(args.payload), "texts": len(items), "engines": {}} - results_by_engine = {} - for name in engine_names: - analyzer, build_secs = build(name, args) - stats = bench(analyzer, items, runs=args.runs, warmup=args.warmup) - stats["build_secs"] = build_secs - report["engines"][name] = stats - results_by_engine[name] = analyze_all(analyzer, items) - - exit_code = 0 - if set(engine_names) >= {"spacy", "gliner"}: - report["ner_diff"] = diff_ner( - items, results_by_engine["spacy"], results_by_engine["gliner"], args.max_examples - ) - regex_mismatches = diff_regex( - items, results_by_engine["spacy"], results_by_engine["gliner"] - ) - report["regex_mismatches"] = regex_mismatches - if regex_mismatches: - exit_code = 1 - - if args.json: - print(json.dumps(report, indent=2, default=str)) - return exit_code - - for name, stats in report["engines"].items(): - print(f"\n== {name} ==") - print(f" build: {stats['build_secs']:.1f}s") - print(f" throughput: {stats['texts_per_sec']:.2f} texts/s ({stats['chars_per_sec']:.0f} chars/s)") - print(f" latency: p50 {stats['latency_p50_ms']:.1f}ms p95 {stats['latency_p95_ms']:.1f}ms") - if "ner_diff" in report: - print("\n== NER parity (spacy=a vs gliner=b, span IoU>=0.5) ==") - for entity, counts in sorted(report["ner_diff"]["per_type"].items()): - print( - f" {entity:<10} spacy={counts['a_total']:<4} gliner={counts['b_total']:<4} " - f"matched={counts['matched']}" - ) - for example in report["ner_diff"]["examples"]: - print(f"\n [{example['language']}] {example['text']}") - if example["only_a"]: - print(f" spacy only: {', '.join(example['only_a'])}") - if example["only_b"]: - print(f" gliner only: {', '.join(example['only_b'])}") - if report["regex_mismatches"]: - print("\n!! REGEX MISMATCHES (wiring bug — engines must agree on non-NER):") - for mismatch in report["regex_mismatches"]: - print(f" {mismatch}") - else: - print("\n regex/checksum entities: identical across engines ✓") - return exit_code - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/apps/pii/scripts/bench_payload.json b/apps/pii/scripts/bench_payload.json deleted file mode 100644 index fd0f207164d..00000000000 --- a/apps/pii/scripts/bench_payload.json +++ /dev/null @@ -1,97 +0,0 @@ -[ - { "text": "My name is John Smith and I live in Paris with my wife Marie.", "language": "en" }, - { - "text": "Dr. Angela Rodriguez will see you on March 14, 2026 at 3:30 PM in the Boston clinic.", - "language": "en" - }, - { - "text": "Contact Sarah O'Connor at sarah.oconnor@example.com or call (212) 555-0123.", - "language": "en" - }, - { "text": "The package ships to 45 Queen Street, Toronto, next Tuesday.", "language": "en" }, - { - "text": "Ahmed is a practicing Muslim from Egypt who moved to Berlin in 2019.", - "language": "en" - }, - { "text": "She is a Catholic Norwegian citizen born on 12/05/1988.", "language": "en" }, - { - "text": "Payment with card 4111111111111111 was declined yesterday at 10am.", - "language": "en" - }, - { - "text": "The vehicle VIN 1HGCM82633A004352 was registered to James Wilson in Ohio.", - "language": "en" - }, - { - "text": "His National Insurance number is AB123456C and he lives in Manchester.", - "language": "en" - }, - { - "text": "Meeting rescheduled: Friday, 9 January 2026, 14:00, with Priya Natarajan from Mumbai.", - "language": "en" - }, - { "text": "Send the invoice to accounts@acme.io before the end of Q1 2026.", "language": "en" }, - { - "text": "Klaus, a German engineer, and his Buddhist colleague Mei flew from Munich to Osaka.", - "language": "en" - }, - { - "text": "The Democrats and Republicans debated in Washington on election night.", - "language": "en" - }, - { - "text": "No PII here: the quarterly revenue grew 14% and margins held steady.", - "language": "en" - }, - { - "text": "Server request id a7f3k2m9x1q8w5z2b is not a VIN and not a person.", - "language": "en" - }, - { - "text": "Me llamo María García y vivo en Madrid desde el 3 de mayo de 2020.", - "language": "es" - }, - { "text": "Mi NIF es 12345678Z y mi correo es maria.garcia@ejemplo.es.", "language": "es" }, - { - "text": "El señor Javier Morales, ciudadano mexicano, llegó a Barcelona el lunes.", - "language": "es" - }, - { "text": "La reunión con Carmen será el 15 de junio de 2026 en Sevilla.", "language": "es" }, - { - "text": "Los musulmanes y los católicos convivieron durante siglos en Córdoba.", - "language": "es" - }, - { "text": "Mi chiamo Marco Rossi e abito a Roma vicino al Colosseo.", "language": "it" }, - { - "text": "Il codice fiscale di Maria Rossi è RSSMRA85T10A562S, nata il 10 dicembre 1985.", - "language": "it" - }, - { - "text": "Giulia Bianchi, cittadina italiana, si trasferì a Milano nel gennaio 2021.", - "language": "it" - }, - { - "text": "L'appuntamento con il dottor Ferrari è fissato per il 20 marzo 2026 a Torino.", - "language": "it" - }, - { "text": "Nazywam się Jan Kowalski i mieszkam w Warszawie od 2015 roku.", "language": "pl" }, - { - "text": "Mój numer PESEL to 44051401359, urodziłem się 14 maja 1944 w Krakowie.", - "language": "pl" - }, - { - "text": "Anna Nowak, obywatelka polska, spotka się z nami we wtorek 12 maja 2026.", - "language": "pl" - }, - { "text": "Katolicy i protestanci wspólnie świętowali w Gdańsku.", "language": "pl" }, - { - "text": "Nimeni on Matti Virtanen ja asun Helsingissä Töölön kaupunginosassa.", - "language": "fi" - }, - { - "text": "Henkilötunnukseni on 131052-308T ja synnyin Tampereella lokakuussa 1952.", - "language": "fi" - }, - { "text": "Liisa Korhonen muutti Ouluun maanantaina 5. tammikuuta 2026.", "language": "fi" }, - { "text": "Suomalaiset ja ruotsalaiset kilpailevat jääkiekossa joka vuosi.", "language": "fi" } -] diff --git a/apps/pii/server.py b/apps/pii/server.py index 3f59cc540aa..b60368eb58c 100644 --- a/apps/pii/server.py +++ b/apps/pii/server.py @@ -1,54 +1,270 @@ """Combined Presidio REST service: analyzer + anonymizer on one port. -Constructs one warm AnalyzerEngine (multi-language NLP + a native check-digit -VIN recognizer) and one AnonymizerEngine at startup, exposing stock-compatible -endpoints so a single PRESIDIO_URL serves both. - -NER engine selection (see engines.py): -- PII_ENGINE=spacy (default): the 5 large spaCy models, unchanged behavior. -- PII_ENGINE=gliner: one multilingual GLiNER model for PERSON/LOCATION/NRP/ - DATE_TIME. The stock image ships both engines, so this is a pure env flip. - PII_DEVICE picks cpu/cuda (unset = auto-detect), PII_GLINER_MODEL overrides - the model id. The same code runs on CPU and GPU. Each uvicorn worker - (PII_WORKERS) loads its own GLiNER model copy — into GPU memory when on - cuda — so GPU deployments generally want PII_WORKERS=1 per GPU, unlike the - CPU/spacy path where workers scale with vCPUs. +Constructs one warm AnalyzerEngine (5 large spaCy models for NER + +regex/checksum pattern recognizers, incl. a native check-digit VIN recognizer) +and one AnonymizerEngine at startup, exposing stock-compatible endpoints so a +single PII_URL serves both. """ import logging -import os import time from typing import Any -from engines import build_gliner_analyzer, build_spacy_analyzer from fastapi import FastAPI -from presidio_analyzer import AnalyzerEngine, BatchAnalyzerEngine, RecognizerResult +from presidio_analyzer import ( + AnalyzerEngine, + BatchAnalyzerEngine, + Pattern, + PatternRecognizer, + RecognizerResult, +) +from presidio_analyzer.nlp_engine import NlpEngineProvider +from presidio_analyzer.predefined_recognizers import ( + AuAbnRecognizer, + AuAcnRecognizer, + AuMedicareRecognizer, + AuTfnRecognizer, + EsNieRecognizer, + EsNifRecognizer, + FiPersonalIdentityCodeRecognizer, + InAadhaarRecognizer, + InPanRecognizer, + InPassportRecognizer, + InVehicleRegistrationRecognizer, + InVoterRecognizer, + ItDriverLicenseRecognizer, + ItFiscalCodeRecognizer, + ItIdentityCardRecognizer, + ItPassportRecognizer, + ItVatCodeRecognizer, + PlPeselRecognizer, + SgFinRecognizer, + SgUenRecognizer, + SpacyRecognizer, + UkNinoRecognizer, +) from presidio_anonymizer import AnonymizerEngine from presidio_anonymizer.entities import OperatorConfig from pydantic import BaseModel -PII_ENGINE = os.environ.get("PII_ENGINE", "spacy") -if PII_ENGINE not in ("spacy", "gliner"): - raise ValueError(f"Invalid PII_ENGINE={PII_ENGINE!r}; expected 'spacy' or 'gliner'") -# Empty/unset -> None -> auto-detect (cuda when torch sees a GPU, else cpu). -PII_DEVICE = os.environ.get("PII_DEVICE") or None -PII_GLINER_MODEL = os.environ.get("PII_GLINER_MODEL", "urchade/gliner_multi_pii-v1") - -# Propagates to uvicorn's root handler, so timing lands in the container log stream. -logger = logging.getLogger("sim.pii") +# Languages served. Each needs its spaCy model installed in the image; the +# es/it/pl/fi predefined recognizers (ES_NIF, IT_FISCAL_CODE, PL_PESEL, ...) +# auto-load once their NLP engine is present. +NLP_CONFIGURATION = { + "nlp_engine_name": "spacy", + "models": [ + {"lang_code": "en", "model_name": "en_core_web_lg"}, + {"lang_code": "es", "model_name": "es_core_news_lg"}, + {"lang_code": "it", "model_name": "it_core_news_lg"}, + {"lang_code": "pl", "model_name": "pl_core_news_lg"}, + {"lang_code": "fi", "model_name": "fi_core_news_lg"}, + ], +} +SUPPORTED_LANGUAGES = [m["lang_code"] for m in NLP_CONFIGURATION["models"]] + +# spaCy pipeline components PII detection does not use. NER depends only on +# `tok2vec` + `ner`; the parser (the most expensive stage), tagger, morphologizer, +# attribute_ruler and lemmatizer are dead weight here. Disabling them is a ~2x +# throughput win with NO change to the detected entities. The only side effect is +# that Presidio's lemma-based context score-boosting weakens slightly — an +# acceptable trade for the speed on this CPU-bound service. +NER_ONLY_DISABLE = ("parser", "tagger", "morphologizer", "attribute_ruler", "lemmatizer") + +# Predefined recognizers Presidio ships but does NOT load into the default +# registry — they must be added explicitly. Each carries its own +# supported_language, so it fires under that language once its NLP model is +# loaded. en: UK/AU/IN/SG locale ids; es/it/pl/fi: national ids. +EXTRA_RECOGNIZERS = [ + UkNinoRecognizer, + AuAbnRecognizer, + AuAcnRecognizer, + AuTfnRecognizer, + AuMedicareRecognizer, + InPanRecognizer, + InAadhaarRecognizer, + InVehicleRegistrationRecognizer, + InVoterRecognizer, + InPassportRecognizer, + SgFinRecognizer, + SgUenRecognizer, + EsNifRecognizer, + EsNieRecognizer, + ItFiscalCodeRecognizer, + ItDriverLicenseRecognizer, + ItVatCodeRecognizer, + ItPassportRecognizer, + ItIdentityCardRecognizer, + PlPeselRecognizer, + FiPersonalIdentityCodeRecognizer, +] + + +class VinRecognizer(PatternRecognizer): + """VIN (17 chars, A-Z/0-9 excluding I/O/Q) with ISO 3779 check-digit + validation (position 9). Validation makes accidental matches on arbitrary + 17-char codes (request ids, SKUs, tokens) extremely unlikely. Some + non-North-American VINs omit the check digit and are skipped — an + intentional bias toward precision. + """ + + _TRANSLIT = { + **{str(d): d for d in range(10)}, + "A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "H": 8, + "J": 1, "K": 2, "L": 3, "M": 4, "N": 5, "P": 7, "R": 9, + "S": 2, "T": 3, "U": 4, "V": 5, "W": 6, "X": 7, "Y": 8, "Z": 9, + } + _WEIGHTS = [8, 7, 6, 5, 4, 3, 2, 10, 0, 9, 8, 7, 6, 5, 4, 3, 2] + + def validate_result(self, pattern_text: str): + vin = pattern_text.upper() + if len(vin) != 17: + return False + try: + total = sum(self._TRANSLIT[c] * w for c, w in zip(vin, self._WEIGHTS)) + except KeyError: + return False + check = total % 11 + expected = "X" if check == 10 else str(check) + return vin[8] == expected + + +def _register_common_recognizers(analyzer: AnalyzerEngine) -> None: + """Regex/checksum recognizers on top of spaCy NER + the Presidio defaults.""" + # VIN is language-agnostic, so register it under every served language — + # a recognizer only fires for the language the caller routes to. + vin_pattern = Pattern(name="vin", regex=r"\b[A-HJ-NPR-Z0-9]{17}\b", score=0.7) + for language in SUPPORTED_LANGUAGES: + analyzer.registry.add_recognizer( + VinRecognizer( + supported_entity="VIN", + patterns=[vin_pattern], + context=["vin", "vehicle", "chassis"], + supported_language=language, + ) + ) + for recognizer_cls in EXTRA_RECOGNIZERS: + analyzer.registry.add_recognizer(recognizer_cls()) def build_analyzer() -> AnalyzerEngine: - if PII_ENGINE == "gliner": - return build_gliner_analyzer(model_name=PII_GLINER_MODEL, device=PII_DEVICE) - return build_spacy_analyzer() - - -logger.info("building analyzer engine=%s device=%s", PII_ENGINE, PII_DEVICE or "auto") + nlp_engine = NlpEngineProvider(nlp_configuration=NLP_CONFIGURATION).create_engine() + for nlp in getattr(nlp_engine, "nlp", {}).values(): + for pipe in NER_ONLY_DISABLE: + if pipe in nlp.pipe_names: + nlp.disable_pipe(pipe) + analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=SUPPORTED_LANGUAGES) + _register_common_recognizers(analyzer) + return analyzer + + +# Own handler at INFO. uvicorn configures only its own loggers, not the root, so a +# bare getLogger propagates to a handler-less root at the default WARNING level and +# every info() (the per-request timing lines) is silently dropped. Attach a stream +# handler directly and stop propagation so timing lands in the container log stream +# regardless of uvicorn's config or worker count. +logger = logging.getLogger("sim.pii") +logger.setLevel(logging.INFO) +if not logger.handlers: + _log_handler = logging.StreamHandler() + _log_handler.setFormatter(logging.Formatter("%(levelname)s: %(name)s: %(message)s")) + logger.addHandler(_log_handler) + logger.propagate = False + +logger.info("building analyzer (spacy)") analyzer = build_analyzer() batch_analyzer = BatchAnalyzerEngine(analyzer_engine=analyzer) anonymizer = AnonymizerEngine() +# Every entity the spaCy NER recognizers can produce. A request touching any of +# these must run spaCy; a request naming only non-NER (regex/checksum) entities can +# skip it. Derived from the live registry so it stays authoritative if Presidio's +# default entity set changes (e.g. ORGANIZATION), unioned with a known floor so an +# unexpectedly empty derivation can never let an NER request skip the NLP pass. +_SPACY_NER_FLOOR = frozenset({"PERSON", "LOCATION", "NRP", "DATE_TIME", "ORGANIZATION"}) +NER_ENTITIES = _SPACY_NER_FLOOR | frozenset( + entity + for recognizer in analyzer.registry.recognizers + if isinstance(recognizer, SpacyRecognizer) + for entity in recognizer.supported_entities +) + +# One blank NlpArtifacts per language, built once at startup. Passing these to +# analyze() skips nlp_engine.process_text (the spaCy tok2vec+ner pass) entirely: +# the pattern recognizers still match on the raw text, SpacyRecognizer is excluded +# by the entity filter, and score_threshold is unset so detection is identical. +# Only context-based score boosting (which needs real tokens) is unavailable — an +# accepted trade for skipping NER on the hot block-output path. Read-only, so it +# is safe to share across requests and workers. +_BLANK_ARTIFACTS = { + language: analyzer.nlp_engine.process_text("", language) + for language in SUPPORTED_LANGUAGES +} + + +def _regex_only(entities: list[str] | None, score_threshold: float | None) -> bool: + """True when the spaCy NLP pass can be skipped: the request names entities, none + require spaCy NER, and no positive score_threshold is set. The blank-artifacts + fast path drops context-based score boosting, which can only change what is + returned when a threshold gates a match between its base and context-boosted + score — so fall back to the full path whenever a threshold is in play.""" + return ( + bool(entities) + and NER_ENTITIES.isdisjoint(entities) + and (score_threshold is None or score_threshold <= 0) + ) + + +def _analyze_one( + text: str, + language: str, + entities: list[str] | None, + score_threshold: float | None, + return_decision_process: bool = False, +): + # Regex-only requests reuse a blank NlpArtifacts to skip the spaCy NLP pass; + # otherwise analyze() computes artifacts (runs spaCy) as usual. + nlp_artifacts = ( + _BLANK_ARTIFACTS.get(language) if _regex_only(entities, score_threshold) else None + ) + return analyzer.analyze( + text=text, + language=language, + entities=entities or None, + score_threshold=score_threshold, + return_decision_process=return_decision_process, + nlp_artifacts=nlp_artifacts, + ) + + +def _analyze_many( + texts: list[str], + language: str, + entities: list[str] | None, + score_threshold: float | None, +): + """Analyze many texts, skipping the spaCy pass for regex-only requests.""" + if _regex_only(entities, score_threshold): + blank = _BLANK_ARTIFACTS.get(language) + return [ + analyzer.analyze( + text=text, + language=language, + entities=entities, + score_threshold=score_threshold, + nlp_artifacts=blank, + ) + for text in texts + ] + return list( + batch_analyzer.analyze_iterator( + texts=texts, + language=language, + entities=entities or None, + score_threshold=score_threshold, + ) + ) + + app = FastAPI(title="Sim Presidio", docs_url=None, redoc_url=None) @@ -150,12 +366,12 @@ def supported_entities(language: str = "en") -> list[str]: @app.post("/analyze") def analyze(req: AnalyzeRequest) -> list[dict[str, Any]]: started = time.perf_counter() - results = analyzer.analyze( - text=req.text, - language=req.language, - entities=req.entities or None, - score_threshold=req.score_threshold, - return_decision_process=req.return_decision_process, + results = _analyze_one( + req.text, + req.language, + req.entities, + req.score_threshold, + req.return_decision_process, ) logger.info( "analyze lang=%s chars=%d entities=%d duration_ms=%.1f", @@ -171,12 +387,7 @@ def analyze(req: AnalyzeRequest) -> list[dict[str, Any]]: def analyze_batch(req: AnalyzeBatchRequest) -> list[list[dict[str, Any]]]: """Analyze many texts in one pass (spaCy nlp.pipe), returning one span list per input in request order — the batched counterpart to /analyze.""" - results = batch_analyzer.analyze_iterator( - texts=req.texts, - language=req.language, - entities=req.entities or None, - score_threshold=req.score_threshold, - ) + results = _analyze_many(req.texts, req.language, req.entities, req.score_threshold) return [[r.to_dict() for r in per_text] for per_text in results] @@ -228,12 +439,7 @@ def redact(req: RedactRequest) -> dict[str, str]: anonymizer directly (no dict round-trip).""" started = time.perf_counter() operators = build_operators(req.anonymizers or req.operators) - results = analyzer.analyze( - text=req.text, - language=req.language, - entities=req.entities or None, - score_threshold=req.score_threshold, - ) + results = _analyze_one(req.text, req.language, req.entities, req.score_threshold) text = ( req.text if not results @@ -261,14 +467,7 @@ def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]: texts that actually matched.""" started = time.perf_counter() operators = build_operators(req.anonymizers or req.operators) - analyzed = list( - batch_analyzer.analyze_iterator( - texts=req.texts, - language=req.language, - entities=req.entities or None, - score_threshold=req.score_threshold, - ) - ) + analyzed = _analyze_many(req.texts, req.language, req.entities, req.score_threshold) masked: list[str] = [] total_spans = 0 for text, per_text in zip(req.texts, analyzed): @@ -282,9 +481,11 @@ def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]: ).text ) logger.info( - "redact_batch lang=%s texts=%d spans=%d duration_ms=%.1f", + "redact_batch lang=%s texts=%d entities=%s nlp=%s spans=%d duration_ms=%.1f", req.language, len(req.texts), + len(req.entities) if req.entities else "all", + "skip" if _regex_only(req.entities, req.score_threshold) else "full", total_spans, (time.perf_counter() - started) * 1000, ) diff --git a/apps/pii/tests/conftest.py b/apps/pii/tests/conftest.py deleted file mode 100644 index c9787a309d9..00000000000 --- a/apps/pii/tests/conftest.py +++ /dev/null @@ -1,5 +0,0 @@ -import sys -from pathlib import Path - -# server.py / engines.py live one level up (repo: apps/pii, image: /app). -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) diff --git a/apps/pii/tests/test_engines.py b/apps/pii/tests/test_engines.py deleted file mode 100644 index 1e6c1b8840c..00000000000 --- a/apps/pii/tests/test_engines.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Unit tests for engines.py — no models, no downloads, no network. - -Run: pip install -r requirements.txt -r requirements-dev.txt && python -m pytest tests -""" - -import importlib.util -import os -import subprocess -import sys -from pathlib import Path - -import pytest -from presidio_analyzer.predefined_recognizers.ner import gliner_recognizer - -import engines - -PII_DIR = Path(__file__).resolve().parent.parent - - -class FakeModel: - def __init__(self): - self.seen_labels: list[list[str]] = [] - - def predict_entities(self, text, labels, flat_ner=True, threshold=0.3, multi_label=False): - self.seen_labels.append(list(labels)) - return [{"label": "person", "score": 0.92, "start": 0, "end": 4, "text": text[0:4]}] - - -class FakeGLiNER: - calls = 0 - - @classmethod - def from_pretrained(cls, model_name, **kwargs): - cls.calls += 1 - return FakeModel() - - -@pytest.fixture -def fake_gliner(monkeypatch): - monkeypatch.setattr(gliner_recognizer, "GLiNER", FakeGLiNER) - engines.SharedModelGLiNERRecognizer._shared_models.clear() - FakeGLiNER.calls = 0 - yield FakeGLiNER - engines.SharedModelGLiNERRecognizer._shared_models.clear() - - -def make_recognizer(language: str): - return engines.SharedModelGLiNERRecognizer( - entity_mapping=engines.GLINER_ENTITY_MAPPING, - model_name="fake/model", - map_location="cpu", - supported_language=language, - ) - - -def test_invalid_pii_engine_fails_import(): - result = subprocess.run( - [sys.executable, "-c", "import server"], - cwd=PII_DIR, - env={**os.environ, "PII_ENGINE": "bogus"}, - capture_output=True, - text=True, - ) - assert result.returncode != 0 - assert "Invalid PII_ENGINE" in result.stderr - - -@pytest.mark.skipif( - importlib.util.find_spec("gliner") is not None, - reason="fail-fast path only exists when gliner is not installed", -) -def test_build_gliner_analyzer_fails_fast_without_gliner(): - with pytest.raises(RuntimeError, match="gliner package is not installed"): - engines.build_gliner_analyzer(model_name="fake/model", device="cpu") - - -def test_shared_model_loads_once_across_languages(fake_gliner): - first = make_recognizer("en") - second = make_recognizer("es") - assert fake_gliner.calls == 1 - assert first.gliner is second.gliner - - -def test_analyze_never_prompts_gliner_with_foreign_entities(fake_gliner): - recognizer = make_recognizer("en") - all_supported = ["PERSON", "LOCATION", "NRP", "DATE_TIME", "CREDIT_CARD", "VIN", "ES_NIF"] - results = recognizer.analyze("John went home", entities=all_supported) - for labels in recognizer.gliner.seen_labels: - assert set(labels) <= set(engines.GLINER_ENTITY_MAPPING) - assert results and results[0].entity_type == "PERSON" - - -def test_analyze_skips_inference_when_no_owned_entity_requested(fake_gliner): - recognizer = make_recognizer("en") - assert recognizer.analyze("4111111111111111", entities=["CREDIT_CARD"]) == [] - assert recognizer.gliner.seen_labels == [] - - -def test_entity_mapping_targets_exactly_the_ner_entities(): - assert set(engines.GLINER_ENTITY_MAPPING.values()) == { - "PERSON", - "LOCATION", - "NRP", - "DATE_TIME", - } diff --git a/apps/pii/tests/test_integration.py b/apps/pii/tests/test_integration.py deleted file mode 100644 index 40c97975754..00000000000 --- a/apps/pii/tests/test_integration.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Integration tests — exercise the real engines end-to-end via the FastAPI app. - -Requires the models present, so run inside the built images (gated behind -RUN_PII_INTEGRATION to keep plain `pytest` runs model-free): - - # spacy regression (default engine) - docker run --rm -e RUN_PII_INTEGRATION=1 python -m pytest tests - - # gliner engine - docker run --rm -e RUN_PII_INTEGRATION=1 -e PII_ENGINE=gliner \ - python -m pytest tests/test_integration.py - -The suite adapts to PII_ENGINE: shared assertions always run, engine-specific -ones only for the active engine. -""" - -import os - -import pytest - -if not os.environ.get("RUN_PII_INTEGRATION"): - pytest.skip( - "integration tests need the built image (RUN_PII_INTEGRATION=1)", - allow_module_level=True, - ) - -from fastapi.testclient import TestClient - -import server - -ENGINE = server.PII_ENGINE -client = TestClient(server.app) - - -def redact_batch(texts, language="en"): - response = client.post("/redact_batch", json={"texts": texts, "language": language}) - assert response.status_code == 200 - return response.json()["texts"] - - -def test_health(): - assert client.get("/health").json() == {"status": "ok"} - - -def test_masks_person_and_email(): - [masked] = redact_batch(["My name is John Smith, email john.smith@example.com."]) - assert "" in masked - assert "" in masked - assert "John Smith" not in masked - assert "john.smith@example.com" not in masked - - -def test_masks_location_and_phone(): - [masked] = redact_batch(["I live in Paris, call me at (212) 555-0123."]) - assert "" in masked - assert "" in masked - assert "Paris" not in masked - - -def test_regex_recognizers_fire_in_non_english_languages(): - [masked] = redact_batch(["Mi NIF es 12345678Z."], language="es") - assert "" in masked - # On the spacy engine the it_core_news_lg NER tags the fiscal code as - # ORGANIZATION and outscores the pattern recognizer, so only assert the - # value is masked; the exact label is checked on the gliner engine where - # spaCy NER can't compete. - [masked] = redact_batch(["Il codice fiscale è RSSMRA85T10A562S."], language="it") - assert "RSSMRA85T10A562S" not in masked - if ENGINE == "gliner": - assert "" in masked - - -def test_vin_checksum_recognizer_fires(): - [masked] = redact_batch(["The car VIN is 1HGCM82633A004352."]) - assert "" in masked - - -def test_no_pii_passes_through_unchanged(): - # NB: "Quarterly" would be tagged DATE_TIME by the spacy engine — keep - # this text free of anything either engine considers an entity. - text = "Revenue grew and margins held steady." - assert redact_batch([text]) == [text] - - -@pytest.mark.skipif(ENGINE != "gliner", reason="gliner-only wiring assertions") -def test_gliner_registry_has_no_spacy_recognizer(): - names = {r.name for r in server.analyzer.registry.recognizers} - assert "SpacyRecognizer" not in names - assert "GLiNERRecognizer" in names - - -@pytest.mark.skipif(ENGINE != "gliner", reason="gliner-only wiring assertions") -def test_gliner_supported_entities_keep_ner_types(): - supported = set(server.analyzer.get_supported_entities("en")) - assert {"PERSON", "LOCATION", "NRP", "DATE_TIME"} <= supported - - -@pytest.mark.skipif(ENGINE != "spacy", reason="spacy-only wiring assertions") -def test_spacy_registry_unchanged(): - names = {r.name for r in server.analyzer.registry.recognizers} - assert "SpacyRecognizer" in names - assert "GLiNERRecognizer" not in names diff --git a/apps/sim/ee/data-retention/components/data-retention-settings.tsx b/apps/sim/ee/data-retention/components/data-retention-settings.tsx index d115bfc33e0..99ca1070a16 100644 --- a/apps/sim/ee/data-retention/components/data-retention-settings.tsx +++ b/apps/sim/ee/data-retention/components/data-retention-settings.tsx @@ -35,6 +35,7 @@ import { type PiiStageKey, type PiiStagePolicy, type PiiStages, + stripNerEntities, } from '@/lib/guardrails/pii-entities' import { UnsavedChangesModal } from '@/app/workspace/[workspaceId]/components/credential-detail' import { saveDiscardActions } from '@/app/workspace/[workspaceId]/settings/components/save-discard-actions/save-discard-actions' @@ -172,7 +173,10 @@ function anyStageHasContent(stages: PiiStages): boolean { /** Persist-time guarantee that `enabled` mirrors "has entity types" for every stage. */ function withSyncedEnabled(stages: PiiStages): PiiStages { return PII_STAGES.reduce((acc, key) => { - acc[key] = { ...stages[key], enabled: stages[key].entityTypes.length > 0 } + // Block outputs are regex-only — strip any NER before persisting. + const entityTypes = + key === 'blockOutputs' ? stripNerEntities(stages[key].entityTypes) : stages[key].entityTypes + acc[key] = { ...stages[key], entityTypes, enabled: entityTypes.length > 0 } return acc }, {} as PiiStages) } @@ -321,6 +325,7 @@ function PiiLanguageSelect({ value, onChange }: PiiLanguageSelectProps) { } interface PiiStagePanelProps { + stageKey: PiiStageKey description: string value: PiiStagePolicy onChange: (next: PiiStagePolicy) => void @@ -331,8 +336,12 @@ interface PiiStagePanelProps { * stage is "on" purely by virtue of having entity types selected — `enabled` is * kept in sync with that, so there is no separate toggle. */ -function PiiStagePanel({ description, value, onChange }: PiiStagePanelProps) { - const groups = getEntityGroupsForLanguage(value.language) +function PiiStagePanel({ stageKey, description, value, onChange }: PiiStagePanelProps) { + // Block outputs run in-flight on large payloads, so they are restricted to the + // regex/checksum recognizers (no spaCy NER) — see the server fast path. + const groups = getEntityGroupsForLanguage(value.language, { + regexOnly: stageKey === 'blockOutputs', + }) function update(entityTypes: string[], language = value.language) { onChange({ ...value, language, entityTypes, enabled: entityTypes.length > 0 }) @@ -570,6 +579,7 @@ function PolicyDetail({ /> )} diff --git a/apps/sim/lib/api/contracts/data-retention.test.ts b/apps/sim/lib/api/contracts/data-retention.test.ts index 10fdcb9e720..d5969732426 100644 --- a/apps/sim/lib/api/contracts/data-retention.test.ts +++ b/apps/sim/lib/api/contracts/data-retention.test.ts @@ -145,6 +145,41 @@ describe('piiRedactionRuleSchema', () => { expect(result.success).toBe(false) }) + it('strips spaCy-NER entities from the blockOutputs stage only (regex-only)', () => { + const result = piiRedactionRuleSchema.safeParse({ + id: 'r-1', + workspaceId: null, + stages: { + input: stage(true, ['PERSON', 'EMAIL_ADDRESS']), + blockOutputs: stage(true, ['PERSON', 'EMAIL_ADDRESS']), + logs: stage(true, ['DATE_TIME']), + }, + }) + expect(result.success).toBe(true) + if (!result.success) return + expect(result.data.stages?.blockOutputs.entityTypes).toEqual(['EMAIL_ADDRESS']) + expect(result.data.stages?.blockOutputs.enabled).toBe(true) + // input and logs keep their NER entities. + expect(result.data.stages?.input.entityTypes).toEqual(['PERSON', 'EMAIL_ADDRESS']) + expect(result.data.stages?.logs.entityTypes).toEqual(['DATE_TIME']) + }) + + it('disables blockOutputs when the NER strip leaves it empty (migration-safe, no lockout)', () => { + const result = piiRedactionRuleSchema.safeParse({ + id: 'r-1', + workspaceId: null, + stages: { + input: stage(false, []), + blockOutputs: stage(true, ['PERSON', 'LOCATION']), + logs: stage(false, []), + }, + }) + expect(result.success).toBe(true) + if (!result.success) return + expect(result.data.stages?.blockOutputs.entityTypes).toEqual([]) + expect(result.data.stages?.blockOutputs.enabled).toBe(false) + }) + it('enforces one rule per scope (uniqueness refine still applies)', () => { const result = piiRedactionSettingsSchema.safeParse({ rules: [ diff --git a/apps/sim/lib/api/contracts/primitives.ts b/apps/sim/lib/api/contracts/primitives.ts index 6acf45a4238..1c6e1e28eb3 100644 --- a/apps/sim/lib/api/contracts/primitives.ts +++ b/apps/sim/lib/api/contracts/primitives.ts @@ -1,5 +1,5 @@ import { z } from 'zod' -import { PII_LANGUAGE_CODES } from '@/lib/guardrails/pii-entities' +import { PII_LANGUAGE_CODES, stripNerEntities } from '@/lib/guardrails/pii-entities' export const unknownRecordSchema = z.record(z.string(), z.unknown()) @@ -134,12 +134,31 @@ export const piiStagePolicySchema = z export type PiiStagePolicy = z.output -/** The three redaction stages, each independently configured. */ -export const piiStagesSchema = z.object({ - input: piiStagePolicySchema, - blockOutputs: piiStagePolicySchema, - logs: piiStagePolicySchema, -}) +/** + * The three redaction stages, each independently configured. + * + * Block outputs are regex-only: they run in-flight on Presidio's spaCy-free fast + * path, so the spaCy-NER entities (PERSON/LOCATION/NRP/DATE_TIME) are stripped + * here rather than rejected — a stored rule that still selects NER stays saveable + * (migration-safe), and a blockOutputs stage left empty by the strip is disabled. + */ +export const piiStagesSchema = z + .object({ + input: piiStagePolicySchema, + blockOutputs: piiStagePolicySchema, + logs: piiStagePolicySchema, + }) + .transform((stages) => { + const entityTypes = stripNerEntities(stages.blockOutputs.entityTypes) + return { + ...stages, + blockOutputs: { + ...stages.blockOutputs, + entityTypes, + enabled: stages.blockOutputs.enabled && entityTypes.length > 0, + }, + } + }) export type PiiStages = z.output diff --git a/apps/sim/lib/billing/retention.test.ts b/apps/sim/lib/billing/retention.test.ts index f14e343fdfc..506e4c118ec 100644 --- a/apps/sim/lib/billing/retention.test.ts +++ b/apps/sim/lib/billing/retention.test.ts @@ -128,6 +128,45 @@ describe('resolveEffectivePiiRedaction', () => { expect(result.logs).toEqual({ enabled: true, entityTypes: ['PERSON'], language: 'en' }) }) + it('strips spaCy-NER entities from blockOutputs at resolve time (regex-only)', () => { + const result = resolveEffectivePiiRedaction({ + orgSettings: settings([ + { + id: 'r-1', + workspaceId: 'ws-1', + stages: { + input: stage(true, ['PERSON', 'EMAIL_ADDRESS']), + blockOutputs: stage(true, ['PERSON', 'EMAIL_ADDRESS']), + logs: stage(true, ['DATE_TIME']), + }, + }, + ]), + workspaceId: 'ws-1', + }) + // input + logs keep NER; blockOutputs drops it (regex-only execution path). + expect(result.input.entityTypes).toEqual(['PERSON', 'EMAIL_ADDRESS']) + expect(result.blockOutputs.entityTypes).toEqual(['EMAIL_ADDRESS']) + expect(result.logs.entityTypes).toEqual(['DATE_TIME']) + }) + + it('disables blockOutputs when only NER was stored (un-migrated rule)', () => { + const result = resolveEffectivePiiRedaction({ + orgSettings: settings([ + { + id: 'r-1', + workspaceId: 'ws-1', + stages: { + input: stage(false, []), + blockOutputs: stage(true, ['PERSON']), + logs: stage(false, []), + }, + }, + ]), + workspaceId: 'ws-1', + }) + expect(result.blockOutputs).toEqual(DISABLED) + }) + it('selects the whole workspace rule over the all rule (no per-stage merge)', () => { const result = resolveEffectivePiiRedaction({ orgSettings: settings([ diff --git a/apps/sim/lib/billing/retention.ts b/apps/sim/lib/billing/retention.ts index afc8e0c0c76..08eecafe26a 100644 --- a/apps/sim/lib/billing/retention.ts +++ b/apps/sim/lib/billing/retention.ts @@ -1,5 +1,9 @@ import type { DataRetentionSettings, PiiStagePolicy } from '@sim/db/schema' -import { coercePiiLanguage, DEFAULT_PII_LANGUAGE } from '@/lib/guardrails/pii-entities' +import { + coercePiiLanguage, + DEFAULT_PII_LANGUAGE, + stripNerEntities, +} from '@/lib/guardrails/pii-entities' /** Resolved policy for one redaction stage. */ export interface EffectivePiiStage { @@ -44,8 +48,16 @@ function sanitizeEntityTypes(value: unknown): string[] { * rejects enabled-with-no-types), so an empty entity list always means "off" — * consistent across the UI, the contract, and the masking layer. */ -function toEffectiveStage(policy: PiiStagePolicy | undefined): EffectivePiiStage { - const types = sanitizeEntityTypes(policy?.entityTypes) +function toEffectiveStage( + policy: PiiStagePolicy | undefined, + opts?: { regexOnly?: boolean } +): EffectivePiiStage { + // Block outputs are regex-only. Strip any spaCy-NER entity defensively here so + // execution never masks block outputs with NER, even for rules stored before + // the restriction (the write path already strips via the API contract). + const types = opts?.regexOnly + ? stripNerEntities(sanitizeEntityTypes(policy?.entityTypes)) + : sanitizeEntityTypes(policy?.entityTypes) if (!policy?.enabled || types.length === 0) return DISABLED_STAGE return { enabled: true, @@ -93,7 +105,7 @@ export function resolveEffectivePiiRedaction(params: { return { input: toEffectiveStage(rule.stages.input), - blockOutputs: toEffectiveStage(rule.stages.blockOutputs), + blockOutputs: toEffectiveStage(rule.stages.blockOutputs, { regexOnly: true }), logs: toEffectiveStage(rule.stages.logs), } } diff --git a/apps/sim/lib/guardrails/pii-entities.test.ts b/apps/sim/lib/guardrails/pii-entities.test.ts new file mode 100644 index 00000000000..6c0d67debc2 --- /dev/null +++ b/apps/sim/lib/guardrails/pii-entities.test.ts @@ -0,0 +1,91 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + getEntityGroupsForLanguage, + NER_PII_ENTITIES, + normalizeRuleStages, + stripNerEntities, +} from '@/lib/guardrails/pii-entities' + +describe('NER_PII_ENTITIES', () => { + it('covers the spaCy-NER entities including ORGANIZATION', () => { + expect(new Set(NER_PII_ENTITIES)).toEqual( + new Set(['PERSON', 'LOCATION', 'NRP', 'DATE_TIME', 'ORGANIZATION']) + ) + }) +}) + +describe('stripNerEntities', () => { + it('drops NER entities and keeps regex/checksum ones (order preserved)', () => { + expect( + stripNerEntities([ + 'PERSON', + 'EMAIL_ADDRESS', + 'DATE_TIME', + 'US_SSN', + 'ORGANIZATION', + 'LOCATION', + 'PHONE_NUMBER', + ]) + ).toEqual(['EMAIL_ADDRESS', 'US_SSN', 'PHONE_NUMBER']) + }) + + it('returns an empty list when only NER was selected', () => { + expect(stripNerEntities(['PERSON', 'NRP'])).toEqual([]) + }) + + it('is a no-op for a regex-only list', () => { + expect(stripNerEntities(['EMAIL_ADDRESS', 'US_SSN'])).toEqual(['EMAIL_ADDRESS', 'US_SSN']) + }) +}) + +describe('getEntityGroupsForLanguage', () => { + const flatten = (groups: Array<{ entities: Array<{ value: string }> }>) => + groups.flatMap((g) => g.entities.map((e) => e.value)) + + it('includes NER entities by default', () => { + const values = flatten(getEntityGroupsForLanguage('en')) + expect(values).toContain('PERSON') + expect(values).toContain('EMAIL_ADDRESS') + }) + + it('excludes the spaCy-NER entities when regexOnly', () => { + const values = flatten(getEntityGroupsForLanguage('en', { regexOnly: true })) + for (const ner of ['PERSON', 'LOCATION', 'NRP', 'DATE_TIME']) { + expect(values).not.toContain(ner) + } + // Regex/checksum entities remain selectable. + expect(values).toContain('EMAIL_ADDRESS') + expect(values).toContain('US_SSN') + }) +}) + +describe('normalizeRuleStages', () => { + it('strips NER from a stored blockOutputs stage (input/logs keep it)', () => { + const stages = normalizeRuleStages({ + stages: { + input: { enabled: true, entityTypes: ['PERSON', 'EMAIL_ADDRESS'], language: 'en' }, + blockOutputs: { enabled: true, entityTypes: ['PERSON', 'EMAIL_ADDRESS'], language: 'en' }, + logs: { enabled: true, entityTypes: ['DATE_TIME'], language: 'en' }, + }, + }) + expect(stages.blockOutputs.entityTypes).toEqual(['EMAIL_ADDRESS']) + expect(stages.blockOutputs.enabled).toBe(true) + expect(stages.input.entityTypes).toEqual(['PERSON', 'EMAIL_ADDRESS']) + expect(stages.logs.entityTypes).toEqual(['DATE_TIME']) + }) + + it('disables blockOutputs when the NER strip empties it', () => { + const stages = normalizeRuleStages({ + stages: { + input: { enabled: false, entityTypes: [] }, + blockOutputs: { enabled: true, entityTypes: ['PERSON', 'LOCATION'] }, + logs: { enabled: false, entityTypes: [] }, + }, + }) + expect(stages.blockOutputs.entityTypes).toEqual([]) + expect(stages.blockOutputs.enabled).toBe(false) + }) +}) diff --git a/apps/sim/lib/guardrails/pii-entities.ts b/apps/sim/lib/guardrails/pii-entities.ts index 9fd57eee8a8..cb7f5504636 100644 --- a/apps/sim/lib/guardrails/pii-entities.ts +++ b/apps/sim/lib/guardrails/pii-entities.ts @@ -229,11 +229,43 @@ export function isEntitySupportedForLanguage( return PII_ENTITIES_BY_LANGUAGE[language].has(entity) } -/** {@link PII_ENTITY_GROUPS} filtered to entities the language recognizes (empty groups dropped). */ -export function getEntityGroupsForLanguage(language: PIILanguage) { +/** + * Entity types produced by the spaCy NER model (vs the regex/checksum pattern + * recognizers). The block-output redaction stage is restricted to the non-NER + * (regex) entities so it runs on the Presidio spaCy-free fast path without the + * per-leaf NER cost. Includes ORGANIZATION — which Presidio's spaCy recognizer + * emits but the user-facing catalog above does not list — so it too is stripped + * from block-output selections, keeping this in sync with the derived + * `NER_ENTITIES` in `apps/pii/server.py`. Typed as strings because ORGANIZATION + * isn't a catalog `PIIEntityType`. + */ +export const NER_PII_ENTITIES: ReadonlySet = new Set([ + 'PERSON', + 'LOCATION', + 'NRP', + 'DATE_TIME', + 'ORGANIZATION', +]) + +/** Drop the spaCy-NER entities ({@link NER_PII_ENTITIES}) from a selection. */ +export function stripNerEntities(entities: readonly string[]): string[] { + return entities.filter((e) => !NER_PII_ENTITIES.has(e)) +} + +/** + * {@link PII_ENTITY_GROUPS} filtered to entities the language recognizes (empty + * groups dropped). With `regexOnly`, the spaCy-NER entities are also excluded — + * used for the block-output stage. + */ +export function getEntityGroupsForLanguage(language: PIILanguage, opts?: { regexOnly?: boolean }) { + const regexOnly = opts?.regexOnly ?? false return PII_ENTITY_GROUPS.map((group) => ({ label: group.label, - entities: group.entities.filter((e) => isEntitySupportedForLanguage(e.value, language)), + entities: group.entities.filter( + (e) => + isEntitySupportedForLanguage(e.value, language) && + (!regexOnly || !NER_PII_ENTITIES.has(e.value)) + ), })).filter((group) => group.entities.length > 0) } @@ -319,9 +351,17 @@ export function normalizeRuleStages(rule: { }) if (rule.stages) { + // Block outputs are regex-only (no spaCy NER) — strip NER from any stored + // rule so hydrated drafts never carry it; a stage left empty becomes disabled. + const blockOutputs = sanitize(rule.stages.blockOutputs) + const blockOutputsEntities = stripNerEntities(blockOutputs.entityTypes) return { input: sanitize(rule.stages.input), - blockOutputs: sanitize(rule.stages.blockOutputs), + blockOutputs: { + ...blockOutputs, + entityTypes: blockOutputsEntities, + enabled: blockOutputs.enabled && blockOutputsEntities.length > 0, + }, logs: sanitize(rule.stages.logs), } } diff --git a/docker/pii.Dockerfile b/docker/pii.Dockerfile index 205462bad3f..4c688ac6e55 100644 --- a/docker/pii.Dockerfile +++ b/docker/pii.Dockerfile @@ -1,15 +1,6 @@ # ======================================== -# Combined Presidio service (analyzer + anonymizer) on a single port (5001) -# -# ONE image serves both NER engines — the engine is a pure runtime choice via -# PII_ENGINE (spacy default | gliner). spaCy large models, torch (CPU), the -# gliner package, and the baked GLiNER weights all ship in it, so flipping -# engines never requires an image swap. -# -# ONE image also serves both fleets: the amd64 build ships CUDA torch, which -# falls back to CPU when no GPU is present, so the Fargate CPU tasks and the -# EC2-GPU tasks pull the same tag. (torch CUDA wheels bundle their own CUDA -# libs; the host only needs the nvidia driver + container runtime.) +# Combined Presidio service (analyzer + anonymizer) on a single port (5001). +# CPU spaCy NER + regex/checksum pattern recognizers. # # Source files are COPY'd last so code edits never re-download deps or models. # ======================================== @@ -43,78 +34,11 @@ RUN --mount=type=cache,target=/root/.cache/pip \ pip install /tmp/*.whl && \ rm /tmp/*.whl -# --- GLiNER engine deps ------------------------------------------------------- -# torch is pinned here (not requirements-gliner.txt) because the CPU and CUDA -# builds install the same version from different wheel indexes. 2.11.0 is the -# newest release published on both the cpu and cu128 indexes for py312. -# -# cu128's arch list keeps sm_75, the compute capability of the GPU fleet's T4s. -# cu121 could not serve this pin anyway — that index stops at torch 2.5.1. -# CUDA 12.8 needs an NVIDIA driver >=525 via minor-version compatibility, which -# the ECS GPU AMI's nvidia-driver-latest-dkms satisfies. -# -# arm64 takes the cpu index: cu128 publishes no aarch64 wheel at 2.11.0, and no -# arm64 target has a GPU. -ARG TORCH_VERSION=2.11.0 -ARG TORCH_CUDA_INDEX_URL=https://download.pytorch.org/whl/cu128 -ARG TORCH_CPU_INDEX_URL=https://download.pytorch.org/whl/cpu -ARG TARGETARCH -RUN --mount=type=cache,target=/root/.cache/pip \ - case "${TARGETARCH}" in \ - amd64) torch_index="${TORCH_CUDA_INDEX_URL}" ;; \ - arm64) torch_index="${TORCH_CPU_INDEX_URL}" ;; \ - *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ - esac && \ - pip install torch==${TORCH_VERSION} --index-url "${torch_index}" - -COPY apps/pii/requirements-gliner.txt ./requirements-gliner.txt -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install -r requirements-gliner.txt - -# Small spaCy models (~60MB total) give the gliner engine tokenization + -# lemmas for the regex recognizers; GLiNER does the NER (see engines.py). -ARG SPACY_SM_MODELS="en_core_web_sm-3.8.0 es_core_news_sm-3.8.0 it_core_news_sm-3.8.0 pl_core_news_sm-3.8.0 fi_core_news_sm-3.8.0" -RUN --mount=type=cache,target=/root/.cache/pip \ - for model in ${SPACY_SM_MODELS}; do \ - whl="${model}-py3-none-any.whl"; \ - curl -fL --retry 5 --retry-delay 5 --retry-all-errors -C - \ - -o "/tmp/${whl}" \ - "https://github.com/explosion/spacy-models/releases/download/${model}/${whl}" || exit 1; \ - done && \ - pip install /tmp/*.whl && \ - rm /tmp/*.whl - -# Bake the GLiNER weights at build time (cached layer) so startup never -# touches the network. HF_HUB_OFFLINE makes a missing/overridden -# PII_GLINER_MODEL fail fast at startup instead of silently downloading. -ENV HF_HOME=/opt/hf-cache -ARG GLINER_MODEL=urchade/gliner_multi_pii-v1 -RUN python -c "from gliner import GLiNER; GLiNER.from_pretrained('${GLINER_MODEL}')" && \ - chmod -R a+rX /opt/hf-cache -ENV HF_HUB_OFFLINE=1 - -# pytest/httpx for the in-image test suites (tests/) — baked in because the -# runtime user has no writable HOME for pip install --user. -COPY apps/pii/requirements-dev.txt ./requirements-dev.txt -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install -r requirements-dev.txt - -# Runs after every pip install, because the requirements above resolve against -# PyPI and could swap the wheel torch_index chose. A cpu-only torch on amd64 -# otherwise surfaces only as "torch.cuda.is_available() is False" once GLiNER -# loads on a GPU host. -RUN python -c "import torch; \ -have = torch.version.cuda is not None; \ -want = '${TARGETARCH}' == 'amd64'; \ -assert have == want, f'{torch.__version__}: cuda build={have}, expected={want}'" - RUN groupadd -g 1001 pii && \ useradd -u 1001 -g pii pii && \ chown -R pii:pii /app -COPY --chown=pii:pii apps/pii/server.py apps/pii/engines.py ./ -COPY --chown=pii:pii apps/pii/scripts ./scripts -COPY --chown=pii:pii apps/pii/tests ./tests +COPY --chown=pii:pii apps/pii/server.py ./ USER pii @@ -134,6 +58,4 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=300s --retries=3 \ # `sh -c exec` expands the env var while keeping uvicorn as PID 1 for clean SIGTERM. # Quote the expansion so a malformed PII_WORKERS fails uvicorn arg-parsing rather # than being interpreted by the shell. -# NB for the gliner engine: EACH worker loads its own GLiNER model copy (into GPU -# memory when on cuda), so GPU deployments generally want PII_WORKERS=1 per GPU. CMD ["sh", "-c", "exec uvicorn server:app --host 0.0.0.0 --port 5001 --workers \"${PII_WORKERS:-1}\""] diff --git a/helm/sim/templates/deployment-pii.yaml b/helm/sim/templates/deployment-pii.yaml index 2dd4b8b22c6..033d87725a1 100644 --- a/helm/sim/templates/deployment-pii.yaml +++ b/helm/sim/templates/deployment-pii.yaml @@ -45,13 +45,7 @@ spec: containerPort: {{ .Values.pii.service.targetPort }} protocol: TCP env: - - name: PII_ENGINE - value: {{ .Values.pii.engine | quote }} - {{- if .Values.pii.device }} - - name: PII_DEVICE - value: {{ .Values.pii.device | quote }} - {{- end }} - {{- range $key, $value := omit (.Values.pii.env | default dict) "PII_ENGINE" "PII_DEVICE" }} + {{- range $key, $value := .Values.pii.env | default dict }} - name: {{ $key }} value: {{ $value | quote }} {{- end }} diff --git a/helm/sim/tests/pii_test.yaml b/helm/sim/tests/pii_test.yaml index f35c87a5759..8c03f193c30 100644 --- a/helm/sim/tests/pii_test.yaml +++ b/helm/sim/tests/pii_test.yaml @@ -30,64 +30,18 @@ tests: path: spec.template.spec.containers[0].ports[0].containerPort value: 5001 - - it: pii pod defaults to the spacy engine - template: deployment-pii.yaml - set: - pii.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: PII_ENGINE - value: "spacy" - - notContains: - path: spec.template.spec.containers[0].env - content: - name: PII_DEVICE - value: "cpu" - - - it: pii.engine and pii.device render through to the container env - template: deployment-pii.yaml - set: - pii.enabled: true - pii.engine: gliner - pii.device: cuda - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: PII_ENGINE - value: "gliner" - - contains: - path: spec.template.spec.containers[0].env - content: - name: PII_DEVICE - value: "cuda" - - - it: user-set pii.env cannot override the chart-owned engine keys + - it: pii.env renders through to the container env template: deployment-pii.yaml set: pii.enabled: true pii.env: - PII_ENGINE: evil - PII_DEVICE: evil - PII_GLINER_MODEL: acme/custom-model + PII_WORKERS: "4" asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: PII_ENGINE - value: "evil" - - notContains: - path: spec.template.spec.containers[0].env - content: - name: PII_DEVICE - value: "evil" - contains: path: spec.template.spec.containers[0].env content: - name: PII_GLINER_MODEL - value: "acme/custom-model" + name: PII_WORKERS + value: "4" - it: app pod gets chart-computed PII_URL pointing at the in-cluster service template: deployment-app.yaml diff --git a/helm/sim/values.schema.json b/helm/sim/values.schema.json index 60bdf101dcb..9ba8ac8f657 100644 --- a/helm/sim/values.schema.json +++ b/helm/sim/values.schema.json @@ -759,15 +759,6 @@ "type": "boolean", "description": "Enable the Presidio PII redaction service" }, - "engine": { - "type": "string", - "enum": ["spacy", "gliner"], - "description": "NER engine (spacy default, gliner opt-in; both ship in the image)" - }, - "device": { - "type": "string", - "description": "Torch device for the gliner engine (cpu, cuda, cuda:N); empty auto-detects" - }, "replicaCount": { "type": "integer", "minimum": 1, diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 8361b45e034..a3d4c94085e 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -836,18 +836,6 @@ pii: digest: "" # sha256: pin overrides tag pullPolicy: IfNotPresent - # NER engine: "spacy" (default) or "gliner" (opt-in zero-shot transformer NER - # for PERSON/LOCATION/NRP/DATE_TIME; regex/checksum recognizers are identical - # on both engines). The image ships both engines, so this is a pure env flip - # — no image change needed. Note: gliner on CPU is orders of magnitude slower - # than spacy; it is intended for GPU nodes. - engine: "spacy" - - # Torch device for the gliner engine: "cpu", "cuda", or "cuda:N". Empty - # auto-detects (cuda when a GPU is visible, else cpu). GPU resource requests - # (nvidia.com/gpu) are not wired yet — GPU scheduling is an infra follow-up. - device: "" - # Number of replicas replicaCount: 1