fix(cli): initialize DB for workspace run-step

chipcompiler/cli/app.py

@@ -5,9 +5,9 @@
 import click
 import typer
 
+from chipcompiler.cli.commands import workspace as workspace_commands
 from chipcompiler.cli.commands.param import param_app
 from chipcompiler.cli.commands.project import register_project_commands
-from chipcompiler.cli.commands.workspace import workspace_app
 from chipcompiler.cli.core.version_info import root_version_line, version_payload, version_text
 
 app = typer.Typer(
@@ -52,22 +52,29 @@ def version_cmd(
 
 register_project_commands(app)
 app.add_typer(param_app, name="param")
-app.add_typer(workspace_app, name="workspace")
+app.add_typer(workspace_commands.workspace_app, name="workspace")
 
 
-def invoke_typer_app(argv: Sequence[str]) -> int:
+def invoke_typer_app(
+    argv: Sequence[str],
+    *,
+    keep_workspace_json_stdio_redirect: bool = False,
+) -> int:
     if not argv:
         command = typer.main.get_command(app)
         click.echo(command.get_help(click.Context(command, info_name="ecc")), err=True)
         return 1
 
     command = typer.main.get_command(app)
     try:
-        result = command.main(
-            args=list(argv),
-            prog_name="ecc",
-            standalone_mode=False,
-        )
+        with workspace_commands.keep_json_stdio_redirect(
+            keep_workspace_json_stdio_redirect
+        ):
+            result = command.main(
+                args=list(argv),
+                prog_name="ecc",
+                standalone_mode=False,
+            )
     except click.exceptions.Exit as exc:
         return int(exc.exit_code or 0)
     except click.ClickException as exc:

chipcompiler/cli/commands/workspace.py

@@ -1,6 +1,8 @@
+import json
+import os
 import sys
 from collections.abc import Callable
-from contextlib import redirect_stdout
+from contextlib import contextmanager, redirect_stdout, suppress
 from typing import Annotated
 
 import typer
@@ -27,17 +29,144 @@
     help="Manage legacy runtime workspaces",
 )
 
+_KEEP_JSON_STDIO_REDIRECT = False
 
-def _finish(result: dict, json_output: bool) -> None:
-    render_workspace_response(result, json_output)
+
+@contextmanager
+def keep_json_stdio_redirect(enabled: bool):
+    global _KEEP_JSON_STDIO_REDIRECT
+
+    previous = _KEEP_JSON_STDIO_REDIRECT
+    _KEEP_JSON_STDIO_REDIRECT = enabled
+    try:
+        yield
+    finally:
+        _KEEP_JSON_STDIO_REDIRECT = previous
+
+
+def _finish(result: dict, json_output: bool, output_stream=None) -> None:
+    if json_output and output_stream is not None:
+        print(json.dumps(result, ensure_ascii=False), file=output_stream)
+    else:
+        render_workspace_response(result, json_output)
     raise typer.Exit(code=exit_code_for_response(result["response"]))
 
 
-def _call_runtime(callback: Callable[[], dict], json_output: bool) -> dict:
-    if not json_output:
-        return callback()
-    with redirect_stdout(sys.stderr):
-        return callback()
+@contextmanager
+def _preserve_cli_stdio():
+    saved_stdout = sys.stdout
+    saved_stderr = sys.stderr
+    saved_stdout_fd = None
+    saved_stderr_fd = None
+
+    for stream in (sys.stdout, sys.stderr):
+        with suppress(Exception):
+            stream.flush()
+
+    try:
+        saved_stdout_fd = os.dup(1)
+        saved_stderr_fd = os.dup(2)
+    except OSError:
+        if saved_stdout_fd is not None:
+            os.close(saved_stdout_fd)
+        try:
+            yield
+        finally:
+            sys.stdout = saved_stdout
+            sys.stderr = saved_stderr
+        return
+
+    try:
+        yield
+    finally:
+        for stream in (sys.stdout, sys.stderr):
+            with suppress(Exception):
+                stream.flush()
+
+        try:
+            os.dup2(saved_stdout_fd, 1)
+            os.dup2(saved_stderr_fd, 2)
+        finally:
+            os.close(saved_stdout_fd)
+            os.close(saved_stderr_fd)
+            sys.stdout = saved_stdout
+            sys.stderr = saved_stderr
+
+
+@contextmanager
+def _json_runtime_stdio():
+    saved_stdout = sys.stdout
+    saved_stderr = sys.stderr
+    saved_stdout_fd = None
+    saved_stderr_fd = None
+    output_stream = None
+    close_output_stream = False
+
+    for stream in (sys.stdout, sys.stderr):
+        with suppress(Exception):
+            stream.flush()
+
+    try:
+        saved_stdout_fd = os.dup(1)
+        saved_stderr_fd = os.dup(2)
+        try:
+            saved_stdout_file_fd = saved_stdout.fileno()
+        except (AttributeError, OSError, ValueError):
+            output_stream = saved_stdout
+        else:
+            if saved_stdout_file_fd == 1:
+                output_stream = os.fdopen(
+                    os.dup(saved_stdout_fd),
+                    "w",
+                    encoding=getattr(saved_stdout, "encoding", None) or "utf-8",
+                    buffering=1,
+                    closefd=True,
+                )
+                close_output_stream = True
+            else:
+                output_stream = saved_stdout
+        os.dup2(2, 1)
+        sys.stdout = sys.stderr
+    except OSError:
+        for fd in (saved_stdout_fd, saved_stderr_fd):
+            if fd is not None:
+                os.close(fd)
+        if close_output_stream and output_stream is not None:
+            output_stream.close()
+        with redirect_stdout(sys.stderr):
+            yield None
+        return
+
+    try:
+        yield output_stream
+    finally:
+        for stream in (sys.stdout, sys.stderr):
+            with suppress(Exception):
+                stream.flush()
+
+        try:
+            if not (close_output_stream and _KEEP_JSON_STDIO_REDIRECT):
+                os.dup2(saved_stdout_fd, 1)
+                os.dup2(saved_stderr_fd, 2)
+        finally:
+            os.close(saved_stdout_fd)
+            os.close(saved_stderr_fd)
+            if close_output_stream and output_stream is not None:
+                output_stream.close()
+            sys.stdout = saved_stdout
+            sys.stderr = saved_stderr
+
+
+def _dispatch_runtime(callback: Callable[[], dict], json_output: bool) -> None:
+    if json_output:
+        with _json_runtime_stdio() as output_stream:
+            result = callback()
+            _finish(result, json_output, output_stream=output_stream)
+        return
+
+    with _preserve_cli_stdio():
+        result = callback()
+    _finish(result, json_output)
 
 
 @workspace_app.command("create", help="Create a legacy runtime workspace")
@@ -76,7 +205,8 @@ def create_cmd(
     except InputError as exc:
         result = workspace_response("create_workspace", exc.response, message=[str(exc)])
     else:
-        result = _call_runtime(lambda: create_workspace_from_request(request), json_output)
+        _dispatch_runtime(lambda: create_workspace_from_request(request), json_output)
+        return
     _finish(result, json_output)
 
 
@@ -85,7 +215,7 @@ def load_cmd(
     directory: Annotated[str, typer.Option("--directory")] = "",
     json_output: Annotated[bool, typer.Option("--json")] = False,
 ) -> None:
-    _finish(_call_runtime(lambda: load_workspace(directory), json_output), json_output)
+    _dispatch_runtime(lambda: load_workspace(directory), json_output)
 
 
 @workspace_app.command("run-flow", help="Run the workspace flow")
@@ -94,7 +224,7 @@ def run_flow_cmd(
     rerun: Annotated[bool, typer.Option("--rerun")] = False,
     json_output: Annotated[bool, typer.Option("--json")] = False,
 ) -> None:
-    _finish(_call_runtime(lambda: run_workspace_flow(directory, rerun), json_output), json_output)
+    _dispatch_runtime(lambda: run_workspace_flow(directory, rerun), json_output)
 
 
 @workspace_app.command("run-step", help="Run one workspace step")
@@ -104,8 +234,7 @@ def run_step_cmd(
     rerun: Annotated[bool, typer.Option("--rerun")] = False,
     json_output: Annotated[bool, typer.Option("--json")] = False,
 ) -> None:
-    result = _call_runtime(lambda: run_workspace_step(directory, step, rerun), json_output)
-    _finish(result, json_output)
+    _dispatch_runtime(lambda: run_workspace_step(directory, step, rerun), json_output)
 
 
 @workspace_app.command("get-info", help="Show workspace or step runtime information")
@@ -115,13 +244,12 @@ def get_info_cmd(
     info_id: Annotated[str, typer.Option("--id")] = "",
     json_output: Annotated[bool, typer.Option("--json")] = False,
 ) -> None:
-    result = _call_runtime(lambda: get_workspace_info(directory, step, info_id), json_output)
-    _finish(result, json_output)
+    _dispatch_runtime(lambda: get_workspace_info(directory, step, info_id), json_output)
 
 
 @workspace_app.command("get-home", help="Show workspace home-page data")
 def get_home_cmd(
     directory: Annotated[str, typer.Option("--directory")] = "",
     json_output: Annotated[bool, typer.Option("--json")] = False,
 ) -> None:
-    _finish(_call_runtime(lambda: get_workspace_home(directory), json_output), json_output)
+    _dispatch_runtime(lambda: get_workspace_home(directory), json_output)

chipcompiler/cli/main.py

@@ -2,16 +2,23 @@
 from collections.abc import Sequence
 
 
-def run(argv: Sequence[str] | None = None) -> int:
+def run(
+    argv: Sequence[str] | None = None,
+    *,
+    _keep_workspace_json_stdio_redirect: bool = False,
+) -> int:
     raw = list(argv) if argv is not None else sys.argv[1:]
 
     from chipcompiler.cli.app import invoke_typer_app
 
-    return invoke_typer_app(raw)
+    return invoke_typer_app(
+        raw,
+        keep_workspace_json_stdio_redirect=_keep_workspace_json_stdio_redirect,
+    )
 
 
 def main() -> None:
-    sys.exit(run())
+    sys.exit(run(_keep_workspace_json_stdio_redirect=True))
 
 
 if __name__ == "__main__":

chipcompiler/cli/workspace/service.py

@@ -154,7 +154,21 @@ def run_workspace_step(directory: str, step: str, rerun: bool) -> dict:
 
     try:
         workspace, engine_flow = load_workspace_runtime(directory)
-        state = engine_flow.run_step(step, rerun)
+        workspace_step = engine_flow.get_workspace_step(step)
+        if workspace_step is None:
+            state = engine_flow.run_step(step, rerun)
+        else:
+            from chipcompiler.data.step import StateEnum
+
+            if not rerun and engine_flow.check_state(
+                name=workspace_step.name,
+                tool=workspace_step.tool,
+                state=StateEnum.Success,
+            ):
+                state = engine_flow.run_step(workspace_step, rerun)
+            else:
+                _init_db_engine_for_workspace_step(engine_flow, workspace_step)
+                state = engine_flow.run_step(workspace_step, rerun)
     except WorkspaceValidationError as exc:
         return workspace_response(cmd, "failed", data=response_data, message=[str(exc)])
     except Exception as exc:
@@ -303,6 +317,19 @@ def build_flow_for_workspace(workspace, create_step_workspaces: bool = True):
     return engine_flow
 
 
+def _init_db_engine_for_workspace_step(engine_flow, workspace_step):
+    engine_db = getattr(engine_flow, "engine_db", None)
+    if engine_db is None:
+        from chipcompiler.engine import EngineDB
+
+        engine_db = EngineDB(workspace=engine_flow.workspace)
+        engine_flow.engine_db = engine_db
+    elif engine_db.has_init():
+        return True
+
+    return engine_db.create_db_engine(step=workspace_step)
+
+
 def _workspace_step_from_flow(workspace, name: str):
     previous_step = None
     for flow_step in workspace.flow.data.get("steps", []):

nix/python/ecc-dreamplace/default.nix

@@ -22,7 +22,7 @@
   setuptools,
   shapely,
   torch,
-  uv-build,
+  hatchling,
   wheel,
 }:
 
@@ -32,7 +32,7 @@ let
   rootSrc = fetchFromGitHub {
     owner = "openecos-projects";
     repo = "ecc-dreamplace";
-    rev = "v${version}";
+    rev = "1e9d038870efa0141b6122e120456c7eb05c8fee";
     hash = "sha256-EoxYaAwSnr/PwRnUxeeZrrqJx4VNwEARjP8HapAfFlQ=";
     fetchSubmodules = true;
   };
@@ -94,15 +94,10 @@ buildPythonPackage {
 
   src = rootSrc;
 
-  build-system = [ uv-build ];
+  build-system = [ hatchling ];
 
   buildInputs = runtimeInputs;
 
-  postPatch = ''
-    substituteInPlace pyproject.toml \
-      --replace-fail 'uv_build>=0.10.9,<0.12' 'uv_build>=0.10.0,<0.12'
-  '';
-
   preBuild = ''
     cp -r ${runtime}/dreamplace/. dreamplace/
     rm -rf thirdparty

nix/python/ecc-tools/default.nix

@@ -30,19 +30,18 @@
 }:
 
 let
-  version = "0.1.0-alpha.3";
+  version = "0.1.0-alpha.4";
 
   src = fetchFromGitHub {
     owner = "openecos-projects";
     repo = "ecc-tools";
-    rev = "984c032e90f3fb2620b7c1ebeefc6fff583d385f";
-    hash = "sha256-j4cLgJqSaGHJPw80U9CdZuDh9uWkl1P1lmp2+xmtj2o=";
+    rev = "af793d33efe91c37f42bfe00eb737427de43715a";
+    hash = "sha256-wD3mGge2vhGoXDIpanyRZGl1tTRmQJodDOmp8Lw2aC0=";
   };
 
   installDeps =
     lib.pipe
       {
-        iir-rust = "src/operation/iIR/source/iir-rust/iir";
         sdf_parser = "src/database/manager/parser/sdf/sdf_parse";
         vcd_parser = "src/database/manager/parser/vcd/vcd_parser";
         verilog-parser = "src/database/manager/parser/verilog/verilog-rust/verilog-parser";