diff --git a/README.md b/README.md index e14c0ae..0cced6e 100644 --- a/README.md +++ b/README.md @@ -109,3 +109,9 @@ COMPOSE_PROJECT_NAME=quantumleap COMPOSE_DOMAIN=quantumleap.srvitkiotlab.itkdev.dk COMPOSE_FILES=docker-compose.yml,docker-compose.prod.yml ``` + +## Test data + +``` shell +task test-data:load +``` diff --git a/Taskfile.yml b/Taskfile.yml index 1086be1..62aa621 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -11,6 +11,11 @@ dotenv: vars: DOCKER_COMPOSE: '{{.TASK_DOCKER_COMPOSE | default "docker compose"}}' +includes: + test-data: + taskfile: ./test-data/Taskfile.yml + dir: ./test-data + tasks: grafana:reset-database: desc: Reset Grafana database @@ -47,3 +52,7 @@ tasks: compose: cmds: - '{{.DOCKER_COMPOSE}} {{.TASK_ARGS}} {{.CLI_ARGS}}' + + default: + silent: true + cmd: task --list diff --git a/test-data/Taskfile.yml b/test-data/Taskfile.yml new file mode 100644 index 0000000..06bc58b --- /dev/null +++ b/test-data/Taskfile.yml @@ -0,0 +1,37 @@ +# yaml-language-server: $schema=https://taskfile.dev/schema.json + +version: '3' + +tasks: + load: + desc: Load test data + prompt: Really load test data (and reset the current test data)? + cmds: + - task: :compose + vars: + TASK_ARGS: rm --force --stop test-data-timescale + - task: :compose + vars: + TASK_ARGS: up test-data-timescale --detach --wait + - task: info + + info: + desc: Show test data info + cmds: + - task: :compose + vars: + TASK_ARGS: exec test-data-timescale psql quantumleap quantumleap --command '\dt' + - task: :compose + vars: + TASK_ARGS: exec test-data-timescale psql quantumleap quantumleap --command 'SELECT COUNT(*), MIN(time_index), MAX(time_index) FROM "etrefrigerator-sensor"' + + generate-data:etrefrigerator-sensor: + desc: Generate etrefrigerator-sensor.csv + cmds: + - task: python + vars: + TASK_ARGS: generate-etrefrigerator-sensor.py + + python: + internal: true + cmd: docker run --rm --volume $PWD:/app --workdir /app python:3 python {{.TASK_ARGS}} diff --git a/test-data/docker-compose.yml b/test-data/docker-compose.yml new file mode 100644 index 0000000..5c9095e --- /dev/null +++ b/test-data/docker-compose.yml @@ -0,0 +1,16 @@ +services: + test-data-timescale: + image: timescale/timescaledb-ha:${TIMESCALE_VERSION:-pg17.9-ts2.25.2-oss} + volumes: + # https://www.w3tutorials.net/blog/how-to-create-user-database-in-script-for-docker-postgres/ + - ./test-data/initdb.d/:/docker-entrypoint-initdb.d/ + - ./test-data/:/test-data/ + environment: + - POSTGRES_PASSWORD=* + networks: + - quantumleap + healthcheck: + test: ["CMD-SHELL", "pg_isready -U quantumleap"] + interval: 10s + timeout: 5s + retries: 5 diff --git a/test-data/generate-etrefrigerator-sensor.py b/test-data/generate-etrefrigerator-sensor.py new file mode 100644 index 0000000..e360b82 --- /dev/null +++ b/test-data/generate-etrefrigerator-sensor.py @@ -0,0 +1,53 @@ +import csv +import datetime as dt +import random +import sys +import uuid + +random.seed(19750523) + +number_of_rows=100_000_000 +number_of_rows=1_000_000 + +filename='initdb.d/etrefrigerator-sensor.csv' + +entity_type = 'refrigerator-sensor' +entity_id = 'refrigerator-sensor:5e318760-Milesight' +fiware_servicepath = '' +instanceid = f'urn:ngsi-ld:{uuid.UUID(int=random.getrandbits(128))}' +appliance = 'Fryser' +department = 'Department' +floor = 'floor' +name = 'name' +room = 'room' + +writer = csv.writer(sys.stdout) +time = dt.datetime(2025, 1, 1).astimezone() + +for i in range(number_of_rows): + time_index = 0 + __original_ngsi_entity__ = None # '{}' + battery = random.randint(0, 100) + humidity = round(random.uniform(0.00, 100.00), 2) + temperature = round(random.uniform(-10, 20), 2) + + writer.writerow([ + entity_id, + entity_type, + time.isoformat(), + fiware_servicepath, + __original_ngsi_entity__, + instanceid, + appliance, + battery, + department, + floor, + humidity, + name, + room, + temperature, + ]) + + time += dt.timedelta(minutes=10) + +# print(f'{number_of_rows} rows written to file {filename}.') diff --git a/test-data/initdb.d/.gitignore b/test-data/initdb.d/.gitignore new file mode 100644 index 0000000..afed073 --- /dev/null +++ b/test-data/initdb.d/.gitignore @@ -0,0 +1 @@ +*.csv diff --git a/test-data/initdb.d/01-init.sql b/test-data/initdb.d/01-init.sql new file mode 100644 index 0000000..43357ab --- /dev/null +++ b/test-data/initdb.d/01-init.sql @@ -0,0 +1,10 @@ +-- # @todo Rewrite to use environment variables (cf. https://stackoverflow.com/a/70976611) +CREATE ROLE quantumleap LOGIN PASSWORD '*'; +GRANT pg_execute_server_program TO quantumleap; + +CREATE DATABASE quantumleap OWNER quantumleap ENCODING 'UTF8'; + +\connect quantumleap + +CREATE EXTENSION IF NOT EXISTS postgis CASCADE; +CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; diff --git a/test-data/initdb.d/02-load-data.sql b/test-data/initdb.d/02-load-data.sql new file mode 100644 index 0000000..806311a --- /dev/null +++ b/test-data/initdb.d/02-load-data.sql @@ -0,0 +1,46 @@ +-- pg_dump --schema-only --schema="public" --table=etrefrigerator-sensor quantumleap + +\connect quantumleap + +-- +-- Name: etrefrigerator-sensor; Type: TABLE; Schema: public; Owner: quantumleap +-- + +CREATE TABLE public."etrefrigerator-sensor" ( + entity_id text, + entity_type text, + time_index timestamp with time zone NOT NULL, + fiware_servicepath text, + __original_ngsi_entity__ jsonb, + instanceid text, + appliance text, + battery bigint, + department text, + floor text, + humidity double precision, + name text, + room text, + temperature double precision +); + + +ALTER TABLE public."etrefrigerator-sensor" OWNER TO quantumleap; + +-- +-- Name: etrefrigerator-sensor_time_index_idx; Type: INDEX; Schema: public; Owner: quantumleap +-- + +CREATE INDEX "etrefrigerator-sensor_time_index_idx" ON public."etrefrigerator-sensor" USING btree (time_index DESC); + + +-- +-- Name: ix_etrefrigerator-sensor_eid_and_tx; Type: INDEX; Schema: public; Owner: quantumleap +-- + +CREATE INDEX "ix_etrefrigerator-sensor_eid_and_tx" ON public."etrefrigerator-sensor" USING btree (entity_id, time_index DESC); + + +-- docker run --rm --volume $PWD:/app --workdir /app python:3 python generate-etrefrigerator-sensor.py + +COPY public."etrefrigerator-sensor" (entity_id, entity_type, time_index, fiware_servicepath, __original_ngsi_entity__, instanceid, appliance, battery, department, floor, humidity, name, room, temperature) +FROM PROGRAM 'python3 /test-data/generate-etrefrigerator-sensor.py' CSV;