From c0d172897e59554492882e9ba68db8109a08a890 Mon Sep 17 00:00:00 2001 From: Mike Rud Andersen Date: Fri, 10 Apr 2026 14:34:03 +0200 Subject: [PATCH 1/6] Fixes to make a newly started development env. work --- .gitignore | 1 + README.md | 15 ++++++++++----- flake.lock | 6 +++--- flake.nix | 16 ++++++++-------- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 10e0dd4..8e9ea76 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ /scratch.sql /api-key.txt /test-assets/ +/.idea todo.md diff --git a/README.md b/README.md index 0d09c5c..2eb30bc 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,16 @@ -# -run like this +# Initial setup ```bash nix develop -pg_setup -pg_ctl -D /PATH/TO/REPO/hostmap/.dev_postgres//data -l logfile start -DATABASE_URL="postgres://:postgres@localhost:5432/hostmap-dev" cargo run -- test-assets/minimalTargetList.json +pg_initial_setup +cargo sqlx migrate run +cargo run -- test-assets/minimalTargetList.json +``` + +# Development +```bash +nix develop +pg_start ``` ``` diff --git a/flake.lock b/flake.lock index 7436957..e747480 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "crane": { "locked": { - "lastModified": 1758758545, - "narHash": "sha256-NU5WaEdfwF6i8faJ2Yh+jcK9vVFrofLcwlD/mP65JrI=", + "lastModified": 1775790182, + "narHash": "sha256-pG2RWVQY0Pe+rmmXJx+Jpyi+JcgjWzS18m7fcD1B64Q=", "owner": "ipetkov", "repo": "crane", - "rev": "95d528a5f54eaba0d12102249ce42f4d01f4e364", + "rev": "534982f1c41834b101e381b07b1121a4f065a374", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 34a8657..99a3ad3 100644 --- a/flake.nix +++ b/flake.nix @@ -6,7 +6,6 @@ crane = { url = "github:ipetkov/crane"; - inputs.nixpkgs.follows = "nixpkgs"; }; pre-commit-hooks = { @@ -143,12 +142,16 @@ export PGHOST=localhost export PGUSER=$USER export PGPASSWORD=postgres - #export PGDATABASE=hostmap-dev - export PGDATABASE=hostmap_restore + export PGDATABASE=hostmap-dev export DATABASE_URL=postgres://$PGUSER:$PGPASSWORD@$PGHOST:$PGPORT/$PGDATABASE - alias pg_start="pg_ctl -D $PGDATA -l $PG/postgres.log start" - alias pg_stop="pg_ctl -D $PGDATA stop" + pg_start() { + pg_isready || pg_ctl -D $PGDATA -l $PG/postgres.log start + } + + pg_stop() { + pg_isready && pg_ctl -D $PGDATA stop + } pg_initial_setup() { pg_stop; @@ -156,9 +159,6 @@ initdb -D $PGDATA && echo "unix_socket_directories = '$PGDATA'" >> $PGDATA/postgresql.conf && pg_start && createdb } - - # start the server if it is not running - pg_ctl -D .dev_postgres/data/ status &> /dev/null && echo "Server already running" || pg_ctl -D $PGDATA -l $PG/postgres.log start ''; }; }; From d052476ab0e0b805de1e177dff1fccc1dab4f9a7 Mon Sep 17 00:00:00 2001 From: Mike Rud Andersen Date: Mon, 13 Apr 2026 07:19:32 +0200 Subject: [PATCH 2/6] direnv and gitignore --- .envrc | 11 +++++++++++ .gitignore | 16 +++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 .envrc diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3e596d5 --- /dev/null +++ b/.envrc @@ -0,0 +1,11 @@ +if nix flake show &> /dev/null; then + echo "-----------------------------------------" + echo "Launching DBC Deployment DevShell (flake)" + echo "-----------------------------------------" + use flake . +else + echo "---------------------------------------------" + echo "Launching DBC Deployment DevShell (nix-shell)" + echo "---------------------------------------------" + use nix +fi diff --git a/.gitignore b/.gitignore index 8e9ea76..d970db3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,24 @@ /target /pg /.env -/postgres.log /.pre-commit-config.yaml /result -/.dev_postgres /setup.sh + +# Dev dir's +/postgres.log +/.dev_postgres + +# ide's +/.idea /.vscode + +# Temps /scratch.sql /api-key.txt /test-assets/ -/.idea todo.md + +# direnv +.direnv/* + From a5a88ad5f56e203927239091126b18844d468986 Mon Sep 17 00:00:00 2001 From: Mike Rud Andersen Date: Thu, 16 Apr 2026 14:51:59 +0200 Subject: [PATCH 3/6] Added options to run the server --- README.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2eb30bc..14bb0b2 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,27 @@ cargo run -- test-assets/minimalTargetList.json nix develop pg_start ``` +**The following environment variables a set from flake.nix**\ + +PG=$PWD/.dev_postgres\ +PGDATA=$PG/data\ +PGPORT=5432\ +PGHOST=localhost\ +PGUSER=$USER\ +PGPASSWORD=postgres\ +PGDATABASE=hostmap-dev\ +DATABASE_URL=postgres://$PGUSER:$PGPASSWORD@$PGHOST:$PGPORT/$PGDATABASE + +**run server:**\ + +For the server to run in development we need a git repository and an API key. The API key is obtained from the GitLab UI.\ +If the API key is saved in the file `./api-key.txt` it will be ignored by git. ``` -run server: -cargo run server --database-url 'postgres://heze:postgres@localhost:5432/hostmap-dev' +cargo run server --database-url $DATABASE_URL --repo-url https://gitlab.dbc.dk/me/my-deployment --grouping-key host_group_name --api-key-file ./api-key.txt --columns "loc" +``` run scraper: +``` cargo run scraper --host-group-file ./test-assets/minimalTargetList.json --scrape-interval 5 ``` From 337eeb074b9c28ce0406b3aa5af69f4053a2619a Mon Sep 17 00:00:00 2001 From: Mike Rud Andersen Date: Tue, 21 Apr 2026 07:43:43 +0200 Subject: [PATCH 4/6] Updated README. Slightly better log messages --- README.md | 8 +++++++- src/activation_logger/mod.rs | 2 +- src/server/mod.rs | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 14bb0b2..8f83d47 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,13 @@ If the API key is saved in the file `./api-key.txt` it will be ignored by git. cargo run server --database-url $DATABASE_URL --repo-url https://gitlab.dbc.dk/me/my-deployment --grouping-key host_group_name --api-key-file ./api-key.txt --columns "loc" ``` +run activation logger: +``` +./target/debug/hostmap activation-logger --url-path /hostmap/hostmap-activation-logs.csv --activation-log-file /var/log/hostmap-activation-logs/hostmap-activation-logs.csv --server-ip 0.0.0.0 --port 9488 +``` + run scraper: ``` -cargo run scraper --host-group-file ./test-assets/minimalTargetList.json --scrape-interval 5 +cargo run scraper --hosts-file ./test-assets/minimalTargetList.json --scrape-interval 5 --activation-logger-port 9488 --api-key-file ./api-key.txt ``` + diff --git a/src/activation_logger/mod.rs b/src/activation_logger/mod.rs index 06ce969..61d3fab 100644 --- a/src/activation_logger/mod.rs +++ b/src/activation_logger/mod.rs @@ -23,7 +23,7 @@ pub(crate) async fn run( }: ActivationLoggerArgs, ) { let bind_addr = format!("{}:{}", server_ip, port); - tracing::info!("Starting server at http://{}", &bind_addr); + tracing::info!("Starting activation-logger server at http://{}", &bind_addr); tracing::info!( "Serving csv log file, {:?}, at http://{}{}", &activation_log_file.clone(), diff --git a/src/server/mod.rs b/src/server/mod.rs index 8dffdc2..4c93d00 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -171,7 +171,7 @@ pub async fn run( ) }); - tracing::info!("Starting server at http://{}", &bind_addr); + tracing::info!("Starting hostmap overview server at http://{}", &bind_addr); axum::serve(listener, router.into_make_service()) .with_graceful_shutdown(async { From 6fd8284c98bb23cc3b19b3def62c0a1a73053db9 Mon Sep 17 00:00:00 2001 From: Mike Rud Andersen Date: Mon, 18 May 2026 22:08:32 +0200 Subject: [PATCH 5/6] Filter out dry-activate from overview --- README.md | 87 +++++++++++++++++++++++- src/server/repository/host_repository.rs | 1 + 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8f83d47..76af741 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,87 @@ -# Initial setup +# Project overview +HostMap is a set of tools used to track what is installed on machines managed by *Platform*.\ +It consists of: +1. One or more servers running a web-server and a database hosting the data and overview. +2. A wrapper around the `switch-to-configuration` script on each managed host.\ +The wrapper script is logging information about each activation -- Timestamp, username, the store path and action taken. +3. An activation-logger running on each managed host, which can serve the collected activations. +4. A scraper running on the HostMap servers will contact each Nix-host in the configuration and fetch the list and post it to the web server where it is stored in the database. +5. A `hostmap-update` script running on the build server that will save information about the building of hostname, git commit hashes, branch name, store path and a timestamp. +The files generated by `hostmap-update` is transferred to the hostmap web server + +## Host configuration example: +```json +[ + { + "hostname": "gitlab-t01", + "host_url": "gitlab-t01.pzz.dk", + "metadata": { + "env": "stg", + "host_group_name": "hosts-prod" + } + }, + { + "hostname": "gitlab-p01", + "host_url": "gitlab-p01.pzz.dk", + "metadata": { + "env": "stg", + "host_group_name": "hosts-prod" + } + }, + { + "hostname": "localhost", + "host_url": "localhost", + "metadata": { + "env": "stg", + "host_group_name": "hosts-prod" + } + } +] +``` + +## Scraped file Example: +```csv +2026-04-13 09:31:04+02:00;srhb;/nix/store/r1d177x9dan0jgj0280w75li5fqlpp8w-nixos-system-hosts-p03-25.11pre-git;boot +2026-04-21 12:07:42+02:00;mran;/nix/store/wc8sgkdad9wnb2q9bds5mlsfy7999990-nixos-system-hosts-p03-25.11pre-git;switch +2026-04-22 09:31:04+02:00;srhb;/nix/store/r1d177x9dan0jgj0280w75li5f999991-nixos-system-hosts-p03-25.11pre-git;dry-activate +``` + +## Nix git build example: +```json +[ + { + "hostname": "artifactory-p101", + "store_path": "/nix/store/ldk0002skbab10v3b8xy343b33c2vffr-nixos-system-wharfix-p02-25.05pre-git", + "commit_hash": "dda93d97b7856c4ff7d3d959d0790ca6ad80d2bc", + "branch": "hostmap-ci-take-1", + "created_at": "2025-10-14 13:53:19+02:00" + }, + { + "hostname": "artifactory-p201", + "store_path": "/nix/store/ldk0002skbab10v3b8xy343b33c2vffr-nixos-system-wharfix-p02-25.05pre-git", + "commit_hash": "dda93d97b7856c4ff7d3d959d0790ca6ad80d2bd", + "branch": "hostmap-ci-take-2", + "created_at": "2025-10-14 13:53:19+02:00" + }, + { + "hostname": "artifactory-p301", + "store_path": "/nix/store/ldk0002skbab10v3b8xy343b33c2vffr-nixos-system-wharfix-p02-25.05pre-git", + "commit_hash": "dda93d97b7856c4ff7d3d959d0790ca6ad80d2be", + "branch": "hostmap-ci-take-3", + "created_at": "2025-10-15 13:53:19+02:00" + }, + { + "hostname": "ca-platform-p01", + "store_path": "/nix/store/ldk0002skbab10v3b8xy343b33c2vffr-nixos-system-wharfix-p02-25.05pre-git", + "commit_hash": "dda93d97b7856c4ff7d3d959d0790ca6ad80d2bf", + "branch": "hostmap-ci-take-4", + "created_at": "2025-10-16 13:53:19+02:00" + } +] +``` + +# Development +## Initial setup ```bash nix develop @@ -7,7 +90,7 @@ cargo sqlx migrate run cargo run -- test-assets/minimalTargetList.json ``` -# Development +## Development start ```bash nix develop pg_start diff --git a/src/server/repository/host_repository.rs b/src/server/repository/host_repository.rs index c6458bb..1df03c6 100644 --- a/src/server/repository/host_repository.rs +++ b/src/server/repository/host_repository.rs @@ -80,6 +80,7 @@ SELECT DISTINCT ON (ac.hostname) ac.activation_id, ac.activated_at, ac.username, ac.store_path, ac.activation_type, ac.hostname FROM activation ac + where ac.activation_type != 'dry-activate' ORDER BY ac.hostname, ac.activated_at DESC ) SELECT DISTINCT ON(l.hostname) l.activation_id, l.activated_at, l.username, From c467c3a1b5930644043c07ca5d36770a3f582857 Mon Sep 17 00:00:00 2001 From: Mike Rud Andersen Date: Thu, 21 May 2026 11:43:43 +0200 Subject: [PATCH 6/6] Removed references to DBC --- .envrc | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.envrc b/.envrc index 3e596d5..e746cc9 100644 --- a/.envrc +++ b/.envrc @@ -1,11 +1,11 @@ if nix flake show &> /dev/null; then echo "-----------------------------------------" - echo "Launching DBC Deployment DevShell (flake)" + echo "Launching DevShell (flake)" echo "-----------------------------------------" use flake . else echo "---------------------------------------------" - echo "Launching DBC Deployment DevShell (nix-shell)" + echo "Launching DevShell (nix-shell)" echo "---------------------------------------------" use nix fi diff --git a/README.md b/README.md index 76af741..0595c82 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ For the server to run in development we need a git repository and an API key. Th If the API key is saved in the file `./api-key.txt` it will be ignored by git. ``` -cargo run server --database-url $DATABASE_URL --repo-url https://gitlab.dbc.dk/me/my-deployment --grouping-key host_group_name --api-key-file ./api-key.txt --columns "loc" +cargo run server --database-url $DATABASE_URL --repo-url https://somegit.myServer.dk/me/my-deployment --grouping-key host_group_name --api-key-file ./api-key.txt --columns "loc" ``` run activation logger: