Exclude dry activate

.envrc

@@ -0,0 +1,11 @@
+if nix flake show &> /dev/null; then
+  echo "-----------------------------------------"
+  echo "Launching DevShell (flake)"
+  echo "-----------------------------------------"
+  use flake .
+else
+  echo "---------------------------------------------"
+  echo "Launching DevShell (nix-shell)"
+  echo "---------------------------------------------"
+  use nix
+fi

.gitignore

@@ -1,13 +1,24 @@
 /target
 /pg
 /.env
-/postgres.log
 /.pre-commit-config.yaml
 /result
-/.dev_postgres
 /setup.sh
+
+# Dev dir's
+/postgres.log
+/.dev_postgres
+
+# ide's
+/.idea
 /.vscode
+
+# Temps
 /scratch.sql
 /api-key.txt
 /test-assets/
 todo.md
+
+# direnv
+.direnv/*
+

README.md

@@ -1,17 +1,127 @@
-#
-run like this
+# Project overview
+HostMap is a set of tools used to track what is installed on machines managed by *Platform*.\
+It consists of:
+1. One or more servers running a web-server and a database hosting the data and overview.
+2. A wrapper around the `switch-to-configuration` script on each managed host.\
+The wrapper script is logging information about each activation -- Timestamp, username, the store path and action taken.
+3. An activation-logger running on each managed host, which can serve the collected activations.
+4. A scraper running on the HostMap servers will contact each Nix-host in the configuration and fetch the list and post it to the web server where it is stored in the database.
+5. A `hostmap-update` script running on the build server that will save information about the building of hostname, git commit hashes, branch name, store path and a timestamp.
+The files generated by `hostmap-update` is transferred to the hostmap web server 
 
+## Host configuration example:
+```json
+[
+  {
+    "hostname": "gitlab-t01",
+    "host_url": "gitlab-t01.pzz.dk",
+    "metadata": {
+      "env": "stg",
+      "host_group_name": "hosts-prod"
+    }
+  },
+  {
+    "hostname": "gitlab-p01",
+    "host_url": "gitlab-p01.pzz.dk",
+    "metadata": {
+      "env": "stg",
+      "host_group_name": "hosts-prod"
+    }
+  },
+  {
+    "hostname": "localhost",
+    "host_url": "localhost",
+    "metadata": {
+      "env": "stg",
+      "host_group_name": "hosts-prod"
+    }
+  }
+]
+```
+
+## Scraped file Example:
+```csv
+2026-04-13 09:31:04+02:00;srhb;/nix/store/r1d177x9dan0jgj0280w75li5fqlpp8w-nixos-system-hosts-p03-25.11pre-git;boot
+2026-04-21 12:07:42+02:00;mran;/nix/store/wc8sgkdad9wnb2q9bds5mlsfy7999990-nixos-system-hosts-p03-25.11pre-git;switch
+2026-04-22 09:31:04+02:00;srhb;/nix/store/r1d177x9dan0jgj0280w75li5f999991-nixos-system-hosts-p03-25.11pre-git;dry-activate
+```
+
+## Nix git build example:
+```json
+[
+  {
+    "hostname": "artifactory-p101",
+    "store_path": "/nix/store/ldk0002skbab10v3b8xy343b33c2vffr-nixos-system-wharfix-p02-25.05pre-git",
+    "commit_hash": "dda93d97b7856c4ff7d3d959d0790ca6ad80d2bc",
+    "branch": "hostmap-ci-take-1",
+    "created_at": "2025-10-14 13:53:19+02:00"
+  },
+  {
+    "hostname": "artifactory-p201",
+    "store_path": "/nix/store/ldk0002skbab10v3b8xy343b33c2vffr-nixos-system-wharfix-p02-25.05pre-git",
+    "commit_hash": "dda93d97b7856c4ff7d3d959d0790ca6ad80d2bd",
+    "branch": "hostmap-ci-take-2",
+    "created_at": "2025-10-14 13:53:19+02:00"
+  },
+  {
+    "hostname": "artifactory-p301",
+    "store_path": "/nix/store/ldk0002skbab10v3b8xy343b33c2vffr-nixos-system-wharfix-p02-25.05pre-git",
+    "commit_hash": "dda93d97b7856c4ff7d3d959d0790ca6ad80d2be",
+    "branch": "hostmap-ci-take-3",
+    "created_at": "2025-10-15 13:53:19+02:00"
+  },
+  {
+    "hostname": "ca-platform-p01",
+    "store_path": "/nix/store/ldk0002skbab10v3b8xy343b33c2vffr-nixos-system-wharfix-p02-25.05pre-git",
+    "commit_hash": "dda93d97b7856c4ff7d3d959d0790ca6ad80d2bf",
+    "branch": "hostmap-ci-take-4",
+    "created_at": "2025-10-16 13:53:19+02:00"
+  }
+]
+```
+
+# Development
+## Initial setup
+
+```bash
+nix develop
+pg_initial_setup
+cargo sqlx migrate run
+cargo run -- test-assets/minimalTargetList.json
+```
+
+## Development start
 ```bash
 nix develop
-pg_setup
-pg_ctl -D /PATH/TO/REPO/hostmap/.dev_postgres//data -l logfile start
-DATABASE_URL="postgres://<USERNAME>:postgres@localhost:5432/hostmap-dev" cargo run -- test-assets/minimalTargetList.json
+pg_start
+```
+**The following environment variables a set from flake.nix**\
+
+PG=$PWD/.dev_postgres\
+PGDATA=$PG/data\
+PGPORT=5432\
+PGHOST=localhost\
+PGUSER=$USER\
+PGPASSWORD=postgres\
+PGDATABASE=hostmap-dev\
+DATABASE_URL=postgres://$PGUSER:$PGPASSWORD@$PGHOST:$PGPORT/$PGDATABASE
+
+**run server:**\
+
+For the server to run in development we need a git repository and an API key. The API key is obtained from the GitLab UI.\
+If the API key is saved in the file `./api-key.txt` it will be ignored by git.
+
+```
+cargo run server --database-url $DATABASE_URL --repo-url https://somegit.myServer.dk/me/my-deployment --grouping-key host_group_name --api-key-file ./api-key.txt --columns "loc" 
 ```
 
+run activation logger:
+```
+./target/debug/hostmap activation-logger --url-path /hostmap/hostmap-activation-logs.csv --activation-log-file /var/log/hostmap-activation-logs/hostmap-activation-logs.csv --server-ip 0.0.0.0 --port 9488
 ```
-run server:
-cargo run server --database-url 'postgres://heze:postgres@localhost:5432/hostmap-dev'
 
 run scraper:
-cargo run scraper --host-group-file ./test-assets/minimalTargetList.json --scrape-interval 5
 ```
+cargo run scraper --hosts-file ./test-assets/minimalTargetList.json --scrape-interval 5 --activation-logger-port 9488 --api-key-file ./api-key.txt
+```
+

flake.nix

@@ -6,7 +6,6 @@
 
     crane = {
       url = "github:ipetkov/crane";
-      inputs.nixpkgs.follows = "nixpkgs";
     };
 
     pre-commit-hooks = {
@@ -143,22 +142,23 @@
               	export PGHOST=localhost
               	export PGUSER=$USER
               	export PGPASSWORD=postgres
-              	#export PGDATABASE=hostmap-dev
-              	export PGDATABASE=hostmap_restore
+              	export PGDATABASE=hostmap-dev
               	export DATABASE_URL=postgres://$PGUSER:$PGPASSWORD@$PGHOST:$PGPORT/$PGDATABASE
 
-              	alias pg_start="pg_ctl -D $PGDATA -l $PG/postgres.log start"
-              	alias pg_stop="pg_ctl -D $PGDATA stop"
+              	pg_start() {
+              	    pg_isready || pg_ctl -D $PGDATA -l $PG/postgres.log start
+                }
+
+              	pg_stop() {
+              	    pg_isready && pg_ctl -D $PGDATA stop
+              	}
 
               	pg_initial_setup() {
               		pg_stop;
               		rm -rf $PG;
               		initdb -D $PGDATA &&
               		echo "unix_socket_directories = '$PGDATA'" >> $PGDATA/postgresql.conf && pg_start && createdb
               	}
-
-                      # start the server if it is not running
-              	pg_ctl -D .dev_postgres/data/ status &> /dev/null && echo "Server already running" || pg_ctl -D $PGDATA -l $PG/postgres.log start
             '';
           };
       };

src/activation_logger/mod.rs

@@ -23,7 +23,7 @@ pub(crate) async fn run(
     }: ActivationLoggerArgs,
 ) {
     let bind_addr = format!("{}:{}", server_ip, port);
-    tracing::info!("Starting server at http://{}", &bind_addr);
+    tracing::info!("Starting activation-logger server at http://{}", &bind_addr);
     tracing::info!(
         "Serving csv log file, {:?}, at http://{}{}",
         &activation_log_file.clone(),

src/server/mod.rs

@@ -171,7 +171,7 @@ pub async fn run(
             )
         });
 
-    tracing::info!("Starting server at http://{}", &bind_addr);
+    tracing::info!("Starting hostmap overview server at http://{}", &bind_addr);
 
     axum::serve(listener, router.into_make_service())
         .with_graceful_shutdown(async {

src/server/repository/host_repository.rs

@@ -80,6 +80,7 @@ SELECT
 DISTINCT ON (ac.hostname)
 ac.activation_id, ac.activated_at, ac.username, ac.store_path, ac.activation_type, ac.hostname
 FROM activation ac
+     where ac.activation_type != 'dry-activate'
      ORDER BY ac.hostname, ac.activated_at DESC
 )
 SELECT DISTINCT ON(l.hostname) l.activation_id, l.activated_at, l.username,