diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..cf350f1 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# PostgreSQL init script must use LF as line endings +# Run `git ls-files --eol` to view applied attributes +configuration/os2iot-postgresql/initdb/*.sh text eol=lf +configuration/postgres/initdb/*.sh text eol=lf diff --git a/.gitignore b/.gitignore index ac48180..0d086e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .idea/ +.env +ca.srl ca.crt ca.key server.crt server.key -server.csr \ No newline at end of file +server.csr diff --git a/Development.md b/Development.md new file mode 100644 index 0000000..f2c4a55 --- /dev/null +++ b/Development.md @@ -0,0 +1,92 @@ +# Development + +> [!WARNING] +> This document is still work in progress. + +Start backend and frontend in development mode (cf. [`docker-compose.development.yml`](docker-compose.development.yml)): + +``` shell +task dev:start +task setup:chirpstack +``` + +Nginx now exposes the frontend on and the backend (API) on , e.g. +. + +> [!NOTE] +> You may have to wait quite a while until everything is up and running. And you may have to reload in your browser a +> couple of times … + +Check the status development services: + +``` shell +task dev:status +curl http://0.0.0.0:8888 +curl http://0.0.0.0:8888/api/v1/docs +``` + +## Frontend + +> [!WARNING] +> Incomplete section ahead! + +Built with Angular. + +## Backend + +> [!WARNING] +> Incomplete section ahead! + +Built with [Nest (NestJS)](https://docs.nestjs.com/). + +``` shell +task dev:log SERVICE=backend +``` + +### Add new entity + +1. Create a new class in `src/entities`, e.g. + + ``` node + // src/entities/contact-person.entity.ts + import { DbBaseEntity } from "./base.entity"; + + @Entity("contact_person") + export class ContactPerson extends DbBaseEntity { + // … + } + ``` + +2. Add the entity to `TypeOrmModule.forFeature` in `src/modules/shared.module.ts`: + + ``` node + // src/modules/shared.module.ts + + @Module({ + imports: [ + TypeOrmModule.forFeature([ + // … + ContactPerson, + ]), + // … + ``` + +3. Generate a migration: + + ``` shell + docker compose --project-name os2iot-docker exec os2iot-backend npm run generate-migration src/migration/name-of-migration + ``` + +### Add entity property (in API) + +``` shell + +``` + +--- + +Talk to the database: + +``` shell +docker compose --project-name os2iot-docker exec os2iot-postgresql psql os2iot os2iot +``` diff --git a/README.md b/README.md index 65553ec..723508d 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,26 @@ task open - Email: `global-admin@os2iot.dk` - Password: `hunter2` +#### ChirpStack Integration + +> [!NOTE] +> The backend must be able to talk to ChirpStack to create new applications etc. + +Run + +```bash +task setup:chirpstack +``` + +to generate and set an API key in `.env`, or do it manually: + +1. Open ChirpStack UI: `task open:chirpstack` +2. Login with `admin` / `admin` +3. Go to **API Keys** → Create a new API key (`/#/api-keys/create`) +4. Enter key name and press Submit. +4. Add to `.env` file: `CHIRPSTACK_API_KEY=your-key-here` +5. Restart the backend: `docker compose up --detach os2iot-backend` + **Available tasks:** ```bash @@ -71,23 +91,19 @@ task clean # Remove containers, volumes, and images task open # Open frontend in browser ``` -### ChirpStack Integration (Optional) +## Backend API -If you're using LoRaWAN features, you need to configure the ChirpStack API key: +Run -```bash -# Run the setup task - it will guide you through the process -task setup:chirpstack +``` shell +task backend:api-key:create ``` -Or manually: -1. Open ChirpStack UI: `task open:chirpstack` -2. Login with `admin` / `admin` -3. Go to **API Keys** → Create a new API key -4. Add to `.env` file: `CHIRPSTACK_API_KEY=your-key-here` -5. Restart backend: `docker compose up -d os2iot-backend` +to generate a backend API key. Use to fetch data: -Without this configuration, you'll see `InvalidToken` errors in the backend logs - these can be ignored if you're not using LoRaWAN features. +``` shell +curl --header 'X-API-KEY: …' "http://$(docker compose port nginx 80)/api/v1/application" +``` ## Configuration @@ -98,6 +114,10 @@ Edit the files in the configuration folder to adjust settings for each requireme - Postgres from the official image. - Chirpstack using their Docker Compose +## Development + +See [Development](./Development.md) for some details on how to start the containers in development mode. + ## Troubleshooting FAQ ### Docker File Sharing issues @@ -116,23 +136,6 @@ Docker doesn't have access to mount the volumes. Solution: On Windows: Go to Docker Desktop (tray icon) -> Settings -> Resources -> File Sharing -> Add the directory which is the parent directory of "OS2IoT-docker" or a parent of that. -> Apply & Restart -### error: database "os2iot-e2e" does not exist - -``` -[ExceptionHandler] Unable -to connect to the database. Retrying (1)... -error: database "os2iot-e2e" does not exist - at Parser.parseErrorMessage -``` - -Cause: -Database has not been setup correctly on local machine. - -Solution: -docker compose down --volumes -dos2unix configuration/os2iot-postgresql/initdb/* # Run from git bash on Windows -docker compose up - ### error: Error: connect ETIMEDOUT xxx.xxx.xxx.xxx:xxxx at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1141:16) Cause: diff --git a/Taskfile.yml b/Taskfile.yml index 8cdd0e6..4f1a81e 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -17,7 +17,6 @@ tasks: desc: Run all setup steps (clone repos, fix line endings, generate certs) cmds: - task: setup:clone-repos - - task: setup:fix-line-endings - task: setup:certs - task: setup:check @@ -40,20 +39,6 @@ tasks: echo "OS2IoT-frontend already exists" fi - setup:fix-line-endings: - desc: Fix line endings on database init scripts (for Windows/cross-platform) - silent: true - cmds: - - | - if command -v dos2unix &> /dev/null; then - dos2unix configuration/os2iot-postgresql/initdb/*.sh 2>/dev/null || true - dos2unix configuration/postgres/initdb/*.sh 2>/dev/null || true - dos2unix configuration/postgresql/initdb/*.sh 2>/dev/null || true - echo "Line endings fixed" - else - echo "dos2unix not installed - skipping (install with: apt install dos2unix)" - fi - setup:certs: desc: Generate MQTT broker certificates (CA and server) silent: true @@ -111,7 +96,7 @@ tasks: fi TOKEN=$(curl -s -X POST "http://localhost:${NGINX_PORT}/api/v1/auth/login" \ -H "Content-Type: application/json" \ - -d '{"username":"global-admin@os2iot.dk","password":"hunter2"}' | jq -r '.accessToken') + -d '{"username":"global-admin@os2iot.dk","password":"hunter2"}' | docker run -i --rm ghcr.io/jqlang/jq -r '.accessToken') if [ "$TOKEN" = "null" ] || [ -z "$TOKEN" ]; then echo "✗ Failed to login - ensure services are running" exit 1 @@ -131,28 +116,7 @@ tasks: silent: true cmds: - | - CHIRPSTACK_PORT=$(docker compose port chirpstack 8080 2>/dev/null | cut -d: -f2) - if [ -z "$CHIRPSTACK_PORT" ]; then - echo "✗ ChirpStack not running (run: docker compose up --detach)" - exit 1 - fi - echo "=== ChirpStack API Key Setup ===" - echo "" - echo "1. Opening ChirpStack UI at http://localhost:${CHIRPSTACK_PORT}" - echo "2. Login with: admin / admin" - echo "3. Go to: API Keys (in the left menu)" - echo "4. Click 'Add API key'" - echo "5. Give it a name and click 'Submit'" - echo "6. Copy the generated token" - echo "" - # Try to open in browser - if command -v xdg-open &> /dev/null; then - xdg-open "http://localhost:${CHIRPSTACK_PORT}" 2>/dev/null & - elif command -v open &> /dev/null; then - open "http://localhost:${CHIRPSTACK_PORT}" 2>/dev/null & - fi - echo "Paste your API key here (or press Ctrl+C to cancel):" - read -r API_KEY + API_KEY=$(docker compose exec chirpstack chirpstack --config /etc/chirpstack/ create-api-key --name test-development | grep 'token: ' | sed 's/^[^:]*: *//') if [ -n "$API_KEY" ]; then if [ -f ".env" ]; then if grep -q "^CHIRPSTACK_API_KEY=" .env; then @@ -214,3 +178,83 @@ tasks: else echo "Open in browser: $URL" fi + + backend:api-key:create: + desc: Create backend API key + silent: true + cmds: + - | + NGINX_PORT=$(docker compose port nginx 80 | cut -d: -f2) + if [ -z "$NGINX_PORT" ]; then + echo "✗ Services not running (run: docker compose up --detach)" + exit 1 + fi + TOKEN=$(curl -s -X POST "http://localhost:${NGINX_PORT}/api/v1/auth/login" \ + -H "Content-Type: application/json" \ + -d '{"username":"global-admin@os2iot.dk","password":"hunter2"}' | docker run -i --rm ghcr.io/jqlang/jq -r '.accessToken') + if [ "$TOKEN" = "null" ] || [ -z "$TOKEN" ]; then + echo "✗ Failed to login - ensure services are running" + exit 1 + fi + + # Permissions + # + # id | name + # ----+--------------------------------------------------- + # 2 | Default Organization - Læserettigheder + # 3 | Default Organization - Applikationsadministrator + # 4 | Default Organization - Organisationsadministrator + + RESULT=$(curl -s -X POST "http://localhost:${NGINX_PORT}/api/v1/api-key" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"name":"test-development", "expiresOn": "2100-01-01T00:00:00Z", "permissionIds": [ 2, 3, 4 ]}') + if echo "$RESULT" | grep -q '"key"'; then + API_KEY=$(echo "$RESULT" | docker run -i --rm ghcr.io/jqlang/jq --raw-output '.key') + echo "✓ API key:" + echo + echo "$API_KEY" + else + echo "Error creating API key: $RESULT" + fi + + dev:start: + desc: Start containers in development mode + cmds: + - task: dev:compose + vars: + TASK_ARGS: up --build --detach --wait + + dev:stop: + desc: Stop containers in development mode + cmds: + - task: dev:compose + vars: + TASK_ARGS: stop + + dev:status: + desc: Show status of development containers + cmds: + # We use `printf` to escape having Task processing the placeholders. + - docker compose --file docker-compose.development.yml ps os2iot-frontend os2iot-backend --format '{{printf "{{.Image}} {{.State}}\t{{.Status}}"}}' + + dev:log: + desc: "Follow development container log; examples: `task {{.TASK}} SERVICE=frontend` or `task {{.TASK}} --interactive`" + # https://taskfile.dev/blog/if-and-variable-prompt#prompt-for-required-variables + requires: + vars: + - name: SERVICE + enum: [frontend, backend] + cmds: + - task: dev:compose + vars: + TASK_ARGS: logs --follow os2iot-{{.SERVICE}} + + dev:compose: + desc: Start containers in development mode + cmds: + - docker compose {{range .COMPOSE_FILES}} --file {{.}}{{end}} {{.TASK_ARGS}} {{.CLI_ARGS}} + vars: + COMPOSE_FILES: + - docker-compose.yml + - docker-compose.development.yml diff --git a/configuration/os2iot-postgresql/initdb/001-init-os2iot.sh b/configuration/os2iot-postgresql/initdb/001-init-os2iot.sh old mode 100644 new mode 100755 diff --git a/configuration/os2iot-postgresql/initdb/002-init-os2iot-e2e.sh b/configuration/os2iot-postgresql/initdb/002-init-os2iot-e2e.sh old mode 100644 new mode 100755 diff --git a/configuration/postgres/initdb/001-init-chirpstack.sh b/configuration/postgres/initdb/001-init-chirpstack.sh old mode 100644 new mode 100755 diff --git a/configuration/postgres/initdb/002-chirpstack_extensions.sh b/configuration/postgres/initdb/002-chirpstack_extensions.sh old mode 100644 new mode 100755 diff --git a/docker-compose.development.yml b/docker-compose.development.yml new file mode 100644 index 0000000..26e463a --- /dev/null +++ b/docker-compose.development.yml @@ -0,0 +1,19 @@ +services: + os2iot-frontend: + # https://mherman.org/blog/dockerizing-an-angular-app/ + build: + dockerfile: "Dockerfile" + volumes: + - '../OS2IoT-frontend:/app' + - '/app/node_modules' + + os2iot-backend: + build: + target: dev + volumes: + - '../OS2IoT-backend:/app' + - '/app/node_modules' + + nginx: + ports: + - "8888:80" diff --git a/docker-compose.yml b/docker-compose.yml index 64a42f5..85eeb99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,7 +99,7 @@ services: BASE_URL: /api/v1/ TABLE_PAGE_SIZE: 25 healthcheck: - test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/127.0.0.1/8081'"] + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081 || exit 1"] interval: 10s timeout: 5s retries: 5 @@ -155,13 +155,12 @@ services: MQTT_SUPER_USER_PASSWORD: ${MQTT_SUPER_USER_PASSWORD:-super_user_password} CHIRPSTACK_API_KEY: ${CHIRPSTACK_API_KEY:-} # Create in ChirpStack UI: http://localhost:8080 -> API Keys volumes: - - ./configuration/mosquitto-broker-os2iot/ca.crt:/tmp/os2iot/backend/dist/resources/ca.crt - - ./configuration/mosquitto-broker-os2iot/ca.key:/tmp/os2iot/backend/dist/resources/ca.key - - ./configuration/mosquitto-broker-os2iot/ca.crt:/tmp/os2iot/backend/resources/ca.crt - - ./configuration/mosquitto-broker-os2iot/ca.key:/tmp/os2iot/backend/resources/ca.key + - ./configuration/mosquitto-broker-os2iot/ca.crt:/app/dist/resources/ca.crt + - ./configuration/mosquitto-broker-os2iot/ca.key:/app/dist/resources/ca.key + - ./configuration/mosquitto-broker-os2iot/ca.crt:/app/resources/ca.crt + - ./configuration/mosquitto-broker-os2iot/ca.key:/app/resources/ca.key os2iot-postgresql: - restart: always image: postgis/postgis # We need postgis to store geodata ports: - "5433"