From 5deca55d276f62804baf80484e60b27ff013bab7 Mon Sep 17 00:00:00 2001 From: Wojtek Majewski Date: Wed, 10 Dec 2025 11:40:54 +0100 Subject: [PATCH] rewrite existing diagrams to d2 --- pkgs/website/bin/mmdc-compile | 30 ---------- pkgs/website/bin/mmdc-watch | 10 ---- pkgs/website/package.json | 3 +- pkgs/website/public/analyze_website.svg | 1 - pkgs/website/public/diagrams/schema.svg | 1 - .../public/diagrams/step-state-machine.svg | 1 - .../public/diagrams/task-execution.svg | 1 - pkgs/website/public/diagrams/three-layers.svg | 1 - .../src/assets/analyze_website.mermaid | 5 -- .../src/assets/architecture-diagram.mermaid | 24 -------- .../step-state-machine.mermaid | 4 -- .../task-execution.mermaid | 22 -------- .../three-layers.mermaid | 17 ------ .../src/assets/hero_content_pipeline.mermaid | 7 --- .../src/content/docs/concepts/data-model.mdx | 55 ++++++++++++++++++- .../concepts/three-layer-architecture.mdx | 22 +++++++- .../reference/queue-worker/how-it-works.mdx | 29 +++++++++- .../docs/tutorials/ai-web-scraper/backend.mdx | 16 +++++- .../docs/tutorials/ai-web-scraper/index.mdx | 16 +++++- 19 files changed, 132 insertions(+), 133 deletions(-) delete mode 100755 pkgs/website/bin/mmdc-compile delete mode 100755 pkgs/website/bin/mmdc-watch delete mode 100644 pkgs/website/public/analyze_website.svg delete mode 100644 pkgs/website/public/diagrams/schema.svg delete mode 100644 pkgs/website/public/diagrams/step-state-machine.svg delete mode 100644 pkgs/website/public/diagrams/task-execution.svg delete mode 100644 pkgs/website/public/diagrams/three-layers.svg delete mode 100644 pkgs/website/src/assets/analyze_website.mermaid delete mode 100644 pkgs/website/src/assets/architecture-diagram.mermaid delete mode 100644 pkgs/website/src/assets/architecture-diagrams/step-state-machine.mermaid delete mode 100644 pkgs/website/src/assets/architecture-diagrams/task-execution.mermaid delete mode 100644 pkgs/website/src/assets/architecture-diagrams/three-layers.mermaid delete mode 100644 pkgs/website/src/assets/hero_content_pipeline.mermaid diff --git a/pkgs/website/bin/mmdc-compile b/pkgs/website/bin/mmdc-compile deleted file mode 100755 index 9b7b3635e..000000000 --- a/pkgs/website/bin/mmdc-compile +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Check if an argument was provided -if [ $# -eq 0 ]; then - echo "Usage: $0 " - exit 1 -fi - -# Get the input file path -input_file="$1" - -# Check if the input file exists -if [ ! -f "$input_file" ]; then - echo "Error: File '$input_file' not found" - exit 1 -fi - -# Extract the basename without extension -basename=$(basename "$input_file" .mermaid) - -# Create output directory if it doesn't exist -mkdir -p public/diagrams - -# Output file path -output_file="public/diagrams/${basename}.svg" - -# Compile the mermaid file to SVG -mmdc --theme dark --output "$output_file" --backgroundColor '#121a19' < "$input_file" - -echo "Compiled $input_file to $output_file" diff --git a/pkgs/website/bin/mmdc-watch b/pkgs/website/bin/mmdc-watch deleted file mode 100755 index 83004d1ee..000000000 --- a/pkgs/website/bin/mmdc-watch +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -diagram_path=src/content/docs/reference/diagram.mermaid -diagram_dir=$(dirname $diagram_path) -diagram_file=$(basename $diagram_path) -diagram_base=$(basename $diagram_path .mermaid) -png_path=${diagram_dir}/${diagram_base}.png - -echo $diagram_path |\ - entr fish -c "mmdc --outputFormat png --output $png_path < $diagram_path ; kitten icat $png_path" diff --git a/pkgs/website/package.json b/pkgs/website/package.json index ecfb48ccc..b28b2d9f1 100644 --- a/pkgs/website/package.json +++ b/pkgs/website/package.json @@ -9,8 +9,7 @@ "start": "astro dev", "build": "astro check && astro build", "preview": "astro preview", - "astro": "astro", - "gen-diagram": "mmdc --backgroundColor='#121a19' --theme neutral --output public/analyze_website.svg < m.mermaid" + "astro": "astro" }, "dependencies": { "@astrojs/check": "^0.9.4", diff --git a/pkgs/website/public/analyze_website.svg b/pkgs/website/public/analyze_website.svg deleted file mode 100644 index 8208bf10b..000000000 --- a/pkgs/website/public/analyze_website.svg +++ /dev/null @@ -1 +0,0 @@ -

website

summary

tags

saveToDb

 \ No newline at end of file diff --git a/pkgs/website/public/diagrams/schema.svg b/pkgs/website/public/diagrams/schema.svg deleted file mode 100644 index 5b9a69a26..000000000 --- a/pkgs/website/public/diagrams/schema.svg +++ /dev/null @@ -1 +0,0 @@ -flowstextflow_slugPKintopt_max_attemptsintopt_base_delayintopt_timeoutstepstextflow_slugPK,FKtextstep_slugPKtextstep_typeintstep_indexintdeps_countintopt_max_attemptsintopt_base_delayintopt_timeoutintopt_start_delayrunsuuidrun_idPKtextflow_slugFKtextstatusjsonbinputjsonboutputintremaining_stepsdepstextflow_slugPK,FKtextdep_slugPK,FKtextstep_slugPK,FKstep_statestextflow_slugFKuuidrun_idPK,FKtextstep_slugPK,FKtextstatusintremaining_tasksintinitial_tasksintremaining_depsstep_taskstextflow_slugFKuuidrun_idPK,FKtextstep_slugPK,FKinttask_indexPKbigintmessage_idtextstatusintattempts_countjsonboutputhasinstantiatesdepends_onstate_ofcontainsspawns \ No newline at end of file diff --git a/pkgs/website/public/diagrams/step-state-machine.svg b/pkgs/website/public/diagrams/step-state-machine.svg deleted file mode 100644 index 638320371..000000000 --- a/pkgs/website/public/diagrams/step-state-machine.svg +++ /dev/null @@ -1 +0,0 @@ -

dependencies met

success

error

created

started

completed

failed

 \ No newline at end of file diff --git a/pkgs/website/public/diagrams/task-execution.svg b/pkgs/website/public/diagrams/task-execution.svg deleted file mode 100644 index c7e96bc08..000000000 --- a/pkgs/website/public/diagrams/task-execution.svg +++ /dev/null @@ -1 +0,0 @@ -

Worker Process

Task Execution (inside Postgres)

Step Lifecycle

creates

enqueues

Unsupported markdown: list
Unsupported markdown: list
Unsupported markdown: list

updates

advances

Step: created → started → completed

pgmq Queue

SQL Core

step_tasks table

Edge Worker

 \ No newline at end of file diff --git a/pkgs/website/public/diagrams/three-layers.svg b/pkgs/website/public/diagrams/three-layers.svg deleted file mode 100644 index 2fad6d6df..000000000 --- a/pkgs/website/public/diagrams/three-layers.svg +++ /dev/null @@ -1 +0,0 @@ -

Step definitions

Handler functions

Tasks

Results

DSL Layer
User Intent

SQL Core
Workflow Orchestration

Worker
Task Execution

 \ No newline at end of file diff --git a/pkgs/website/src/assets/analyze_website.mermaid b/pkgs/website/src/assets/analyze_website.mermaid deleted file mode 100644 index 03835dd70..000000000 --- a/pkgs/website/src/assets/analyze_website.mermaid +++ /dev/null @@ -1,5 +0,0 @@ -graph TD - A[website] --> B[summary] - A --> C[tags] - B --> D[saveToDb] - C --> D diff --git a/pkgs/website/src/assets/architecture-diagram.mermaid b/pkgs/website/src/assets/architecture-diagram.mermaid deleted file mode 100644 index c3e052cea..000000000 --- a/pkgs/website/src/assets/architecture-diagram.mermaid +++ /dev/null @@ -1,24 +0,0 @@ -graph TB - subgraph Postgres["Postgres Database"] - pgflow["pgflow Schema"] - pgmq["PGMQ Queues"] - definitions["Flow Definitions"] - runs["Flow Runs"] - end - - subgraph Workers["Edge Workers"] - worker1["Worker 1"] - worker2["Worker 2"] - end - - subgraph Clients["Client Applications"] - dsl["TypeScript DSL"] - client["Client SDK (Coming Soon)"] - end - - dsl -->|Define Flows| definitions - client -->|Start Flows| pgflow - pgflow -->|Enqueue Tasks| pgmq - pgmq -->|Poll Tasks| Workers - Workers -->|Complete Tasks| pgflow - pgflow -->|Update Run Status| runs diff --git a/pkgs/website/src/assets/architecture-diagrams/step-state-machine.mermaid b/pkgs/website/src/assets/architecture-diagrams/step-state-machine.mermaid deleted file mode 100644 index d865544c7..000000000 --- a/pkgs/website/src/assets/architecture-diagrams/step-state-machine.mermaid +++ /dev/null @@ -1,4 +0,0 @@ -stateDiagram-v2 - created --> started: dependencies met - started --> completed: success - started --> failed: error diff --git a/pkgs/website/src/assets/architecture-diagrams/task-execution.mermaid b/pkgs/website/src/assets/architecture-diagrams/task-execution.mermaid deleted file mode 100644 index 9d7db8795..000000000 --- a/pkgs/website/src/assets/architecture-diagrams/task-execution.mermaid +++ /dev/null @@ -1,22 +0,0 @@ -graph TD - subgraph "Step Lifecycle" - STEP["Step: created → started → completed"] - end - subgraph "Task Execution (inside Postgres)" - QUEUE["pgmq Queue"] - CORE["SQL Core"] - TASKS["step_tasks table"] - end - subgraph "Worker Process" - WORKER["Edge Worker"] - end - - STEP -->|creates| TASKS - STEP -->|enqueues| QUEUE - - WORKER -->|1. read_with_poll| QUEUE - WORKER -->|2. start_tasks| CORE - WORKER -->|3. complete_task/fail_task| CORE - - CORE -->|updates| TASKS - CORE -->|advances| STEP \ No newline at end of file diff --git a/pkgs/website/src/assets/architecture-diagrams/three-layers.mermaid b/pkgs/website/src/assets/architecture-diagrams/three-layers.mermaid deleted file mode 100644 index 0095239e2..000000000 --- a/pkgs/website/src/assets/architecture-diagrams/three-layers.mermaid +++ /dev/null @@ -1,17 +0,0 @@ -graph TB - DSL["DSL Layer
User Intent"] - CORE["SQL Core
Workflow Orchestration"] - WORKER["Worker
Task Execution"] - - DSL -->|"Step definitions"| CORE - DSL -.->|"Handler functions"| WORKER - CORE -->|"Tasks"| WORKER - WORKER -->|"Results"| CORE - - classDef dslStyle fill:#2d4a3e,stroke:#4a7c5d,stroke-width:3px,color:#e0e0e0,font-size:16px - classDef coreStyle fill:#3d3a52,stroke:#5d5a7c,stroke-width:3px,color:#e0e0e0,font-size:16px - classDef workerStyle fill:#4a3c2d,stroke:#7c6a4a,stroke-width:3px,color:#e0e0e0,font-size:16px - - class DSL dslStyle - class CORE coreStyle - class WORKER workerStyle diff --git a/pkgs/website/src/assets/hero_content_pipeline.mermaid b/pkgs/website/src/assets/hero_content_pipeline.mermaid deleted file mode 100644 index cc6924e8a..000000000 --- a/pkgs/website/src/assets/hero_content_pipeline.mermaid +++ /dev/null @@ -1,7 +0,0 @@ -graph TD - fetchArticle[fetchArticle] --> summarize[summarize] - fetchArticle --> extractKeywords[extractKeywords] - fetchArticle --> generateImage[generateImage] - summarize --> publish[publish] - extractKeywords --> publish - generateImage --> publish diff --git a/pkgs/website/src/content/docs/concepts/data-model.mdx b/pkgs/website/src/content/docs/concepts/data-model.mdx index 733a38a74..a1c46e6ab 100644 --- a/pkgs/website/src/content/docs/concepts/data-model.mdx +++ b/pkgs/website/src/content/docs/concepts/data-model.mdx @@ -88,9 +88,58 @@ Created and modified during execution. Each run creates new records tracking pro The relationships between these tables form the complete pgflow data model: - - Schema ERD Diagram - +```d2 +...@../../../assets/pgflow-theme.d2 + +direction: right + +# Legend +legend: "" { + style.stroke-width: 0 + style.fill: transparent + + def: Static Definition + def.class: info + run: Runtime State + run.class: success +} + +# Flow tables +flow: Flows { + fl: flows + fl.class: info + st: steps + st.class: info + dp: deps + dp.class: info + + rn: runs + rn.class: success + ss: step_states + ss.class: success + tk: step_tasks + tk.class: success + + fl -> st: has + fl -> rn: instantiates + st -> dp: depends + st -> ss: state + rn -> ss: contains + ss -> tk: spawns +} + +# Worker tables +worker: Workers { + wf: worker_functions + wf.class: info + wk: workers + wk.class: success + + wf -> wk: spawns via cron +} + +flow.tk -> worker.wk: executed by +``` ## How the Categories Work Together diff --git a/pkgs/website/src/content/docs/concepts/three-layer-architecture.mdx b/pkgs/website/src/content/docs/concepts/three-layer-architecture.mdx index 60a55f9b7..997cc62a2 100644 --- a/pkgs/website/src/content/docs/concepts/three-layer-architecture.mdx +++ b/pkgs/website/src/content/docs/concepts/three-layer-architecture.mdx @@ -11,7 +11,27 @@ pgflow separates concerns across three distinct layers. Each layer operates at a ## Overview -Three layers of pgflow architecture +```d2 +...@../../../assets/pgflow-theme.d2 + +direction: down + +dsl: "DSL Layer\nUser Intent" +dsl.class: success + +core: "SQL Core\nWorkflow Orchestration" +core.class: info + +worker: "Worker\nTask Execution" +worker.class: warning + +dsl -> core: Step definitions +dsl -> worker: Handler functions { + style.stroke-dash: 3 +} +core -> worker: Tasks +worker -> core: Results +``` The diagram shows two key distinctions: diff --git a/pkgs/website/src/content/docs/reference/queue-worker/how-it-works.mdx b/pkgs/website/src/content/docs/reference/queue-worker/how-it-works.mdx index a9ad59139..764d772ee 100644 --- a/pkgs/website/src/content/docs/reference/queue-worker/how-it-works.mdx +++ b/pkgs/website/src/content/docs/reference/queue-worker/how-it-works.mdx @@ -10,7 +10,34 @@ import { Aside } from "@astrojs/starlight/components"; **Edge Worker** is a task queue worker that gets messages from a queue and calls user functions with their payload. -[![](https://mermaid.ink/img/pako:eNplkcFugzAMhl8lyrl9AQ47VLBxqdSqlZAGHEziASokyEkmTaXvvoR0o1VziGL_n_9Y9pULLZEnvFItwdSxc1op5o9xTUxU_OQmaMAgy2SL7N0pYXutTMUjGU5WlItYaLog1VFAJSv14paCXdweyw8f-2MZLnZ06LBelXxXRk_DztAM-Gp9KA-kpRP-W7bdvs3Ga4aNaAy0OC_WdzD4B4IQVsLMvvkIZMUiA4mu_8ZHYjW5MxNp4dUnKC9zUHJA-h9R_VQTG-sQyDYINlTs-IaPSCP00q_gGvCK2w5HP53EPyXQJczp5jlwVp9-lOCJJYcbTtq13V_gJgkW0x78lEeefMFgfHYC9an1GqPsraZ9XPiy99svlAqmtA?type=png)](https://mermaid.live/edit#pako:eNplkcFugzAMhl8lyrl9AQ47VLBxqdSqlZAGHEziASokyEkmTaXvvoR0o1VziGL_n_9Y9pULLZEnvFItwdSxc1op5o9xTUxU_OQmaMAgy2SL7N0pYXutTMUjGU5WlItYaLog1VFAJSv14paCXdweyw8f-2MZLnZ06LBelXxXRk_DztAM-Gp9KA-kpRP-W7bdvs3Ga4aNaAy0OC_WdzD4B4IQVsLMvvkIZMUiA4mu_8ZHYjW5MxNp4dUnKC9zUHJA-h9R_VQTG-sQyDYINlTs-IaPSCP00q_gGvCK2w5HP53EPyXQJczp5jlwVp9-lOCJJYcbTtq13V_gJgkW0x78lEeefMFgfHYC9an1GqPsraZ9XPiy99svlAqmtA) +```d2 +...@../../../../assets/pgflow-theme.d2 + +direction: down + +edge: Supabase Edge Functions { + ew: Edge Worker + ew.class: info +} + +db: Database { + pgmq: PGMQ Queue + pgmq.class: success + hb: Workers Table + hb.class: success +} + +prod: Your Application +prod.class: warning +handler: Task Handler +handler.class: warning + +prod -> db.pgmq: sends message +db.pgmq -> edge.ew: reads messages +edge.ew -> db.pgmq: archives messages +edge.ew -> handler: processes messages +edge.ew -> db.hb: sends heartbeats +``` ### Features diff --git a/pkgs/website/src/content/docs/tutorials/ai-web-scraper/backend.mdx b/pkgs/website/src/content/docs/tutorials/ai-web-scraper/backend.mdx index a8eef0c68..f763df442 100644 --- a/pkgs/website/src/content/docs/tutorials/ai-web-scraper/backend.mdx +++ b/pkgs/website/src/content/docs/tutorials/ai-web-scraper/backend.mdx @@ -283,7 +283,21 @@ export const AnalyzeWebsite = new Flow({ slug: "analyzeWebsite", maxAttem Flow structure: -AI Web Scraper Workflow +```d2 +...@../../../../assets/pgflow-theme.d2 + +direction: down + +website: "website" +summary: "summary" +tags: "tags" +saveToDb: "saveToDb" + +website -> summary +website -> tags +summary -> saveToDb +tags -> saveToDb +``` - `website` → fetches the URL (root step) - `summary` & `tags` → run in parallel (both need website content) diff --git a/pkgs/website/src/content/docs/tutorials/ai-web-scraper/index.mdx b/pkgs/website/src/content/docs/tutorials/ai-web-scraper/index.mdx index beaf59460..edec69b43 100644 --- a/pkgs/website/src/content/docs/tutorials/ai-web-scraper/index.mdx +++ b/pkgs/website/src/content/docs/tutorials/ai-web-scraper/index.mdx @@ -44,7 +44,21 @@ You'll create a practical AI web scraper workflow that: 5. Auto-retries when things go wrong (because APIs sometimes fail) -AI Web Scraper Workflow +```d2 +...@../../../../assets/pgflow-theme.d2 + +direction: down + +website: "website" +summary: "summary" +tags: "tags" +saveToDb: "saveToDb" + +website -> summary +website -> tags +summary -> saveToDb +tags -> saveToDb +``` ## Project Structure