Rodeo bufr library

datastore/datastore/Dockerfile

@@ -19,9 +19,9 @@ RUN apt-get update \
     && apt-get install -y --no-install-recommends \
     protobuf-compiler libprotobuf-dev \
     # install protoc-gen-go
-    && go install google.golang.org/protobuf/cmd/protoc-gen-go@latest \
+    && go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.9 \
     # install protoc-gen-go-grpc
-    && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest \
+    && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 \
     # install grpc healtcheck service \
     && arch=$(echo "$(uname)-$(dpkg --print-architecture)" | tr '[:upper:]' '[:lower:]') \
     && wget -q --output-document "/bin/grpc_health_probe" "https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${VERSION_GRPC_HEALTH_PROBE}/grpc_health_probe-${arch}" \

docker-compose.yml

@@ -108,6 +108,9 @@ services:
     environment:
       - DSHOST=${DSHOST:-store}
       - DSPORT=${DSPORT:-50050}
+      - DYNAMICTIME=$DYNAMICTIME
+      - LOTIME=$LOTIME
+      - HITIME=$HITIME
       - MQTT_HOST=${MQTT_HOST:-mqtt}
       - MQTT_USERNAME=${MQTT_USERNAME}
       - MQTT_PASSWORD=${MQTT_PASSWORD}

ingest/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim-bookworm
+FROM python:3.11-slim-bookworm
 
 SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"]
 
@@ -7,7 +7,7 @@ ENV DOCKER_PATH="/app"
 # hadolint ignore=DL3008
 RUN apt-get update \
     && apt-get -y upgrade \
-    && apt-get install -y --no-install-recommends git libeccodes-data rapidjson-dev pybind11-dev make g++ libudunits2-0\
+    && apt-get install -y --no-install-recommends git libeccodes-data libudunits2-0\
     # Cleanup
     && rm -rf /usr/tmp  \
     && apt-get autoremove -y \
@@ -17,15 +17,9 @@ COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto"
 COPY "./api" "${DOCKER_PATH}/api/"
 COPY "requirements.txt" "${DOCKER_PATH}/api/"
 COPY "./src/" "${DOCKER_PATH}/src/"
-COPY "./pyproject.toml" "${DOCKER_PATH}/pyproject.toml"
-COPY "./setup.py" "${DOCKER_PATH}/setup.py"
 
-RUN pip install --no-cache-dir --upgrade -r "${DOCKER_PATH}/api/requirements.txt" \
-    && pip install --no-cache-dir --upgrade pybind11~="2.11.1"
+RUN pip install --no-cache-dir --upgrade -r "${DOCKER_PATH}/api/requirements.txt"
 
-# hadolint ignore=DL3013
-WORKDIR "${DOCKER_PATH}"/src/ingest/bufr/
-RUN make
 WORKDIR /
 
 # Compiling the protobuf file
@@ -40,7 +34,6 @@ RUN python "api/generate_standard_name.py"
 
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --upgrade pip \
-    && pip install --no-cache-dir --upgrade "${DOCKER_PATH}/" \
     && mkdir -p /tmp/metrics
 
 ENV PROMETHEUS_MULTIPROC_DIR=/tmp/metrics

ingest/README.md

@@ -6,6 +6,9 @@
 |----------------------------|----------------------------|-----------------------------------------------------------------------------|
 | `DSHOST`                   | `store`                    | Host address for the data store. Defaults to `store` if not set.            |
 | `DSPORT`                   | `50050`                    | Port for the data store connection. Defaults to `50050` if not set.         |
+| `DYNAMICTIME`              | `true`                     | Whether the valid time range is _dynamic_ or _static_ (defined below). |
+| `LOTIME`                   | `86400`                    | The _earliest_ valid time as seconds to be either [1] subtracted from the current time (if the valid time range is _dynamic_) or [2] added to UNIX epoch (1970-01-01T00:00:00Z) (if the valid time range is _static_). In the case of a _static_ valid time range, the `LOTIME` can optionally be specified as an ISO-8601 datetime of the exact form `2023-10-10T00:00:00Z`. |
+| `HITIME`                   | `-600`                     | Same as `LOTIME`, but for the _latest_ valid time. Note a default leeway of 10 minutes into the future to reduce risk of missing visual observations. |
 | `MQTT_HOST`                |                            | Host address for the MQTT broker.                                           |
 | `MQTT_USERNAME`            |                            | Username for authentication with the MQTT broker.                           |
 | `MQTT_PASSWORD`            |                            | Password for authentication with the MQTT broker.                           |

ingest/api/messages.py

@@ -7,7 +7,7 @@
 from api.model import JsonMessageSchema
 
 
-from ingest.bufr.create_mqtt_message_from_bufr import (
+from bufr_tools.create_mqtt_message_from_bufr import (
     build_all_json_payloads_from_bufr,
 )
 

ingest/requirements-dev.txt

@@ -8,34 +8,32 @@ annotated-types==0.7.0
     # via
     #   -r requirements.txt
     #   pydantic
-anyio==4.4.0
+anyio==4.11.0
     # via
     #   -r requirements.txt
     #   httpx
     #   starlette
-attrs==24.2.0
+attrs==25.4.0
     # via
     #   -r requirements.txt
     #   jsonschema
     #   referencing
-certifi==2024.8.30
+certifi==2025.10.5
     # via
     #   -r requirements.txt
     #   httpcore
     #   httpx
     #   requests
-charset-normalizer==3.3.2
+charset-normalizer==3.4.4
     # via
     #   -r requirements.txt
     #   requests
-click==8.1.7
+click==8.3.0
     # via
     #   -r requirements.txt
     #   uvicorn
-coverage[toml]==7.6.3
+coverage[toml]==7.11.0
     # via pytest-cov
-exceptiongroup==1.2.2
-    # via -r requirements.txt
 fastapi==0.108.0
     # via -r requirements.txt
 grpcio==1.62.1
@@ -46,86 +44,94 @@ grpcio-tools==1.62.1
     # via -r requirements.txt
 gunicorn==22.0.0
     # via -r requirements.txt
-h11==0.14.0
+h11==0.16.0
     # via
     #   -r requirements.txt
     #   httpcore
     #   uvicorn
-httpcore==1.0.6
+httpcore==1.0.9
     # via httpx
 httpx==0.27.2
     # via -r requirements-dev.in
-idna==3.10
+idna==3.11
     # via
     #   -r requirements.txt
     #   anyio
     #   httpx
     #   requests
-iniconfig==2.0.0
+iniconfig==2.3.0
     # via pytest
 isodate==0.6.1
     # via -r requirements.txt
 jsonschema==4.19.2
     # via -r requirements.txt
-jsonschema-specifications==2023.12.1
+jsonschema-specifications==2025.9.1
     # via
     #   -r requirements.txt
     #   jsonschema
 numpy==1.25.2
     # via -r requirements.txt
-packaging==24.1
+packaging==25.0
     # via
     #   -r requirements.txt
     #   gunicorn
     #   pytest
 paho-mqtt==2.1.0
     # via -r requirements.txt
-pluggy==1.5.0
+pluggy==1.6.0
     # via pytest
-prometheus-client==0.20.0
+prometheus-client==0.23.1
     # via
     #   -r requirements.txt
     #   prometheus-fastapi-instrumentator
-prometheus-fastapi-instrumentator==7.0.0
+prometheus-fastapi-instrumentator==7.1.0
     # via -r requirements.txt
-protobuf==4.25.4
+protobuf==4.25.8
     # via
     #   -r requirements.txt
     #   grpcio-tools
-pydantic==2.9.1
+pybind11==2.11.2
+    # via
+    #   -r requirements.txt
+    #   rodeo-bufr-tools
+pydantic==2.12.3
     # via
     #   -r requirements.txt
     #   fastapi
-pydantic-core==2.23.3
+pydantic-core==2.41.4
     # via
     #   -r requirements.txt
     #   pydantic
-pytest==8.3.3
+pygments==2.19.2
+    # via pytest
+pytest==8.4.2
     # via
     #   -r requirements-dev.in
     #   pytest-cov
     #   pytest-timeout
 pytest-cov==5.0.0
     # via -r requirements-dev.in
-pytest-timeout==2.3.1
-    # via -r requirements-dev.in
-python-dateutil==2.8.2
+pytest-timeout==2.4.0
     # via -r requirements-dev.in
-python-multipart==0.0.9
+python-dateutil==2.9.0.post0
+    # via -r requirements.txt
+python-multipart==0.0.20
     # via -r requirements.txt
-referencing==0.35.1
+referencing==0.37.0
     # via
     #   -r requirements.txt
     #   jsonschema
     #   jsonschema-specifications
 requests==2.31.0
     # via -r requirements.txt
-rpds-py==0.20.0
+rodeo-bufr-tools==0.3.2
+    # via -r requirements.txt
+rpds-py==0.28.0
     # via
     #   -r requirements.txt
     #   jsonschema
     #   referencing
-six==1.16.0
+six==1.17.0
     # via
     #   -r requirements.txt
     #   isodate
@@ -140,13 +146,20 @@ starlette==0.32.0.post1
     #   -r requirements.txt
     #   fastapi
     #   prometheus-fastapi-instrumentator
-typing-extensions==4.12.2
+typing-extensions==4.15.0
     # via
     #   -r requirements.txt
+    #   anyio
     #   fastapi
     #   pydantic
     #   pydantic-core
-urllib3==2.2.3
+    #   referencing
+    #   typing-inspection
+typing-inspection==0.4.2
+    # via
+    #   -r requirements.txt
+    #   pydantic
+urllib3==2.5.0
     # via
     #   -r requirements.txt
     #   requests

ingest/requirements.in

@@ -16,3 +16,5 @@ python-multipart~=0.0.9
 requests == 2.31.0
 isodate~=0.6.1
 prometheus-fastapi-instrumentator
+rodeo-bufr-tools ~= 0.3.3
+python-dateutil