From 156554e0d15125d45dd887fe33c49068723b4d41 Mon Sep 17 00:00:00 2001
From: aqilaziz <gonzes7@gmail.com>
Date: Wed, 6 May 2026 07:06:24 +0700
Subject: [PATCH] Fix OCR double period artifacts

---
 scripts/parser/ocr_correct.py      |  3 +++
 scripts/parser/test_ocr_correct.py | 21 +++++++++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 scripts/parser/test_ocr_correct.py

diff --git a/scripts/parser/ocr_correct.py b/scripts/parser/ocr_correct.py
index 6ff3465..13f724e 100644
--- a/scripts/parser/ocr_correct.py
+++ b/scripts/parser/ocr_correct.py
@@ -34,6 +34,9 @@
     (re.compile(r'ff'), 'ff'),
     (re.compile(r'\u00a0'), ' '),  # Non-breaking space → regular space
 
+    # Double periods from OCR/tokenization artifacts, while preserving ellipsis
+    (re.compile(r'(?<![.])\.{2}(?![.])'), '.'),
+
     # Common scanner artifacts
     (re.compile(r'^[;,.]$', re.MULTILINE), ''),  # Lone punctuation on a line
     (re.compile(r'^\s*[-_]{3,}\s*$', re.MULTILINE), ''),  # Horizontal rules from scan lines
diff --git a/scripts/parser/test_ocr_correct.py b/scripts/parser/test_ocr_correct.py
new file mode 100644
index 0000000..46d0eb8
--- /dev/null
+++ b/scripts/parser/test_ocr_correct.py
@@ -0,0 +1,21 @@
+"""Tests for deterministic OCR correction patterns."""
+
+from parser.ocr_correct import correct_ocr_errors
+
+
+def test_double_periods_are_collapsed():
+    text = "Setiap orang wajib mematuhi ketentuan.."
+
+    assert correct_ocr_errors(text) == "Setiap orang wajib mematuhi ketentuan."
+
+
+def test_double_periods_after_list_markers_are_collapsed():
+    text = "i.. melakukan pengawasan;\na.. menyusun standar;"
+
+    assert correct_ocr_errors(text) == "i. melakukan pengawasan;\na. menyusun standar;"
+
+
+def test_ellipsis_is_preserved():
+    text = "Ketentuan ini berlaku..."
+
+    assert correct_ocr_errors(text) == "Ketentuan ini berlaku..."