⚡ Bolt: [performance improvement] Fast yEnc decoding

.jules/bolt.md

@@ -0,0 +1,3 @@
+## 2024-05-24 - [Optimize yEnc decoding with translate and find]
+**Learning:** Python's byte-by-byte iteration in a `while` loop is extremely slow for decoding binary formats like yEnc. When working with byte arrays, utilizing C-backed methods built into the Python standard library, such as `bytes.translate()` and `bytes.find()`, provides a massive performance speedup. The previous implementation checked for escape characters (61 / '=') iteratively per byte.
+**Action:** Always prefer C-backed built-in string/byte methods (`translate`, `find`, `split`) over manual iteration for processing byte strings. When optimizing decoding/encoding, define mapping tables statically at the module level (like `_YENC_TRANS_TABLE`) to avoid recreation overhead. Algebraically simplify translation logic where possible (e.g., `(char - 64 - 42) % 256` into `(char - 106) % 256`). Ensure escape checks correctly handle sequence boundaries instead of failing at line ends.

verify_nzb.py

@@ -115,19 +115,32 @@
     return attrs
 
 
+_YENC_TRANS_TABLE = bytes((i - 42) % 256 for i in range(256))
+
+
 def _decode_yenc_lines(lines: Iterable[bytes]) -> bytes:
     decoded = bytearray()
     for line in lines:
+        escape_pos = line.find(b"=")
+        if escape_pos == -1:
+            decoded.extend(line.translate(_YENC_TRANS_TABLE))
+            continue
+
         index = 0
-        while index < len(line):
-            byte = line[index]
-            if byte == 61:
-                index += 1
-                if index >= len(line):
-                    raise ValueError("dangling yEnc escape")
-                byte = (line[index] - 64) % 256
-            decoded.append((byte - 42) % 256)
-            index += 1
+        length = len(line)
+        while True:
+            if escape_pos == -1:
+                decoded.extend(line[index:].translate(_YENC_TRANS_TABLE))
+                break
+
+            decoded.extend(line[index:escape_pos].translate(_YENC_TRANS_TABLE))
+            if escape_pos + 1 >= length:
+                raise ValueError("dangling yEnc escape")
+
+            decoded.append((line[escape_pos + 1] - 106) % 256)
+            index = escape_pos + 2
+            escape_pos = line.find(b"=", index)
+
     return bytes(decoded)
 
 
@@ -138,7 +151,9 @@
     data_lines: list[bytes] = []
 
     for raw_line in lines:
-        line = raw_line.encode("latin-1") if isinstance(raw_line, str) else bytes(raw_line)
+        line = (
+            raw_line.encode("latin-1") if isinstance(raw_line, str) else bytes(raw_line)
+        )
         line = line.rstrip(b"\r\n")
         if line.startswith(b"=ybegin"):
             ybegin_attrs = _parse_yenc_attrs(line)
@@ -235,7 +250,9 @@
     unique_ids = list(dict.fromkeys(message_ids))
     if not unique_ids:
         return []
-    sample_size = min(len(unique_ids), max(1, math.ceil(len(unique_ids) * sample_percent / 100)))
+    sample_size = min(
+        len(unique_ids), max(1, math.ceil(len(unique_ids) * sample_percent / 100))
+    )
     if sample_size == len(unique_ids):
         return unique_ids
     return random.Random(sample_seed).sample(unique_ids, sample_size)
@@ -271,9 +288,13 @@
             if value is None
         ]
         if missing:
-            raise ValueError(f"missing required options in [{section}]: {', '.join(missing)}")
+            raise ValueError(
+                f"missing required options in [{section}]: {', '.join(missing)}"
+            )
         if (username is None) ^ (password is None):
-            raise ValueError(f"[{section}] must set both username and password or neither")
+            raise ValueError(
+                f"[{section}] must set both username and password or neither"
+            )
         if max_connections < 1:
             raise ValueError(f"[{section}] max_connections must be at least 1")
         if port < 1:
@@ -295,7 +316,9 @@
         )
 
     if not servers:
-        raise ValueError("configuration must contain at least one [server.<name>] section")
+        raise ValueError(
+            "configuration must contain at least one [server.<name>] section"
+        )
     return servers
 
 
@@ -413,13 +436,20 @@
             raise ProtocolNntpError("NNTP command is not ASCII encodable") from exc
         except asyncio.TimeoutError as exc:
             raise TransientNntpError("command timeout") from exc
-        except (ConnectionResetError, BrokenPipeError, OSError, asyncio.IncompleteReadError) as exc:
+        except (
+            ConnectionResetError,
+            BrokenPipeError,
+            OSError,
+            asyncio.IncompleteReadError,
+        ) as exc:
             raise TransientNntpError("connection lost") from exc
 
     async def _read_response(self) -> tuple[int, str]:
         assert self._reader is not None
         try:
-            line = await asyncio.wait_for(self._reader.readline(), timeout=self.config.timeout)
+            line = await asyncio.wait_for(
+                self._reader.readline(), timeout=self.config.timeout
+            )
         except asyncio.TimeoutError as exc:
             raise TransientNntpError("read timeout") from exc
         if not line:
@@ -434,7 +464,9 @@
         lines: list[bytes] = []
         while True:
             try:
-                line = await asyncio.wait_for(self._reader.readline(), timeout=self.config.timeout)
+                line = await asyncio.wait_for(
+                    self._reader.readline(), timeout=self.config.timeout
+                )
             except asyncio.TimeoutError as exc:
                 raise TransientNntpError("read timeout") from exc
             if not line:
@@ -494,7 +526,9 @@
         self._progress_was_written = False
         self._progress_failed = False
 
-    async def run(self, message_ids: Iterable[str], missing_output: str | Path | None = None) -> VerificationSummary:
+    async def run(
+        self, message_ids: Iterable[str], missing_output: str | Path | None = None
+    ) -> VerificationSummary:
         start = time.monotonic()
         try:
             await self._start_workers()
@@ -522,7 +556,9 @@
             present=self.present,
             missing=self.missing,
             error=self.error,
-            stat_requests=sum(conn.request_count for server in self.connections for conn in server),
+            stat_requests=sum(
+                conn.request_count for server in self.connections for conn in server
+            ),
             elapsed_seconds=elapsed,
         )
 
@@ -534,7 +570,11 @@
                 self.workers.append(task)
 
         await asyncio.gather(
-            *(connection.connect(self.retries) for server_connections in self.connections for connection in server_connections),
+            *(
+                connection.connect(self.retries)
+                for server_connections in self.connections
+                for connection in server_connections
+            ),
             return_exceptions=True,
         )
 
@@ -548,14 +588,21 @@
             for connection in server_connections:
                 await connection.close()
 
-    async def _worker_loop(self, server_index: int, connection: AsyncNntpConnection) -> None:
+    async def _worker_loop(
+        self, server_index: int, connection: AsyncNntpConnection
+    ) -> None:
         try:
             while True:
                 job = await self._take_job(server_index)
                 if job is None:
                     return
                 try:
-                    await self._handle_job(server_index, connection, job.message_id, job.target_server_index)
+                    await self._handle_job(
+                        server_index,
+                        connection,
+                        job.message_id,
+                        job.target_server_index,
+                    )
                 except Exception:
                     async with self.job_condition:
                         state = self.states.get(job.message_id)
@@ -581,7 +628,10 @@
 
     def _find_job_for_server(self, server_index: int) -> _Job | None:
         for job in self.jobs:
-            if job.target_server_index is None or job.target_server_index == server_index:
+            if (
+                job.target_server_index is None
+                or job.target_server_index == server_index
+            ):
                 return job
         return None
 
@@ -632,7 +682,9 @@
                 if next_index is not None:
                     self._defer_message_locked(message_id, next_index)
                     return
-                await self._finalize_locked(message_id, "error" if state.had_error else "missing")
+                await self._finalize_locked(
+                    message_id, "error" if state.had_error else "missing"
+                )
                 return
             state.had_error = True
             if next_index is not None:
@@ -660,7 +712,9 @@
         self.jobs.append(_Job(message_id=message_id, target_server_index=server_index))
         self.job_condition.notify_all()
 
-    def _next_server_index_locked(self, state: _MessageState, current_server_index: int) -> int | None:
+    def _next_server_index_locked(
+        self, state: _MessageState, current_server_index: int
+    ) -> int | None:
         total = len(self.servers)
         for offset in range(1, total + 1):
             candidate = (current_server_index + offset) % total
@@ -742,7 +796,10 @@
         try:
             await self._start()
             results = await asyncio.gather(
-                *(self._check_one(index, message_id) for index, message_id in enumerate(message_ids))
+                *(
+                    self._check_one(index, message_id)
+                    for index, message_id in enumerate(message_ids)
+                )
             )
         finally:
             await self._stop()
@@ -751,7 +808,9 @@
             with Path(deep_output).open("w", encoding="utf-8") as handle:
                 for result in results:
                     server = result.server or "-"
-                    handle.write(f"{result.message_id}\t{result.status}\t{result.detail}\t{server}\n")
+                    handle.write(
+                        f"{result.message_id}\t{result.status}\t{result.detail}\t{server}\n"
+                    )
 
         elapsed = time.monotonic() - start
         return DeepCheckSummary(
@@ -769,7 +828,11 @@
 
     async def _start(self) -> None:
         await asyncio.gather(
-            *(connection.connect(self.retries) for server_connections in self.connections for connection in server_connections),
+            *(
+                connection.connect(self.retries)
+                for server_connections in self.connections
+                for connection in server_connections
+            ),
             return_exceptions=True,
         )
         for server_index, server_connections in enumerate(self.connections):
@@ -814,7 +877,9 @@
             finally:
                 self.connection_queues[server_index].put_nowait(connection)
 
-        return DeepCheckResult(message_id=message_id, status="error", detail=last_error, server=None)
+        return DeepCheckResult(
+            message_id=message_id, status="error", detail=last_error, server=None
+        )
 
 
 async def verify_nzb(
@@ -827,11 +892,13 @@
     sample_percent: float = 1.0,
     sample_seed: int | None = None,
     deep_output: str | Path | None = None,
-    progress_stream = sys.stdout,
+    progress_stream=sys.stdout,
 ) -> VerificationSummary:
     servers = load_config(config_path)
     verifier = _Verifier(servers, retries=retries, progress_stream=progress_stream)
-    summary = await verifier.run(parse_nzb_message_ids(nzb_path), missing_output=missing_output)
+    summary = await verifier.run(
+        parse_nzb_message_ids(nzb_path), missing_output=missing_output
+    )
     if deep_check:
         sampled_ids = select_deep_sample(
             verifier.present_message_ids,
@@ -844,10 +911,19 @@
 
 
 def build_arg_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(description="Verify NZB message IDs against NNTP servers.")
+    parser = argparse.ArgumentParser(
+        description="Verify NZB message IDs against NNTP servers."
+    )
     parser.add_argument("nzb_path", type=Path, help="path to the NZB file")
-    parser.add_argument("--config", required=True, type=Path, help="path to the NNTP INI file")
-    parser.add_argument("--retries", type=int, default=1, help="retry count for transient network errors")
+    parser.add_argument(
+        "--config", required=True, type=Path, help="path to the NNTP INI file"
+    )
+    parser.add_argument(
+        "--retries",
+        type=int,
+        default=1,
+        help="retry count for transient network errors",
+    )
     parser.add_argument(
         "--missing-output",
         type=Path,