diff --git a/README.md b/README.md index e7d0d35..ec1f64a 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ for use with [Docker](https://www.docker.com/). Check out our [onebusaway-deployment](https://github.com/oneBusAway/onebusaway-deployment) repository, which features OpenTofu (Terraform) IaC configuration for deploying OneBusAway to AWS, Azure, Google Cloud Platform, Render, Kubernetes, and other platforms. +### Deploying Docker Images + +The 'simplest' way to deploy to services compatible with Docker image deployment is by creating immutable Docker images with your static data bundle pre-generated inside of the image. [See deployment-examples/README.md for more information](deployment-examples/README.md). + ### Deploy to Render [Render](https://www.render.com) is an easy-to-use Platform-as-a-Service (PaaS) provider. You can host OneBusAway on Render by either manually configuring it or by clicking the button below. diff --git a/deployment-examples/README.md b/deployment-examples/README.md new file mode 100644 index 0000000..1616579 --- /dev/null +++ b/deployment-examples/README.md @@ -0,0 +1,25 @@ +# Building and Deploying Immutable Docker Images + +The simplest way to deploy a OneBusAway server to a compatible cloud provider (e.g. Render, Heroku) is by creating an immutable Docker image prebuilt with your transit agency's static GTFS data feed. + +With a little hosting provider-specific automation, you should be able to automatically deploy the latest version of your image from the container registry to which you upload the image. + +The Open Transit Software Foundation uses this approach with [OBACloud](https://onebusawaycloud.com), our OneBusAway as a Service offering, to build and deploy new containers in an easy, predictable, and horizontally scalable manner. + +## Instructions + +*All of the files referenced below can be found in the [immutable](./immutable) directory.* + +### Setup and Image Building + +1. Create a private repository on GitHub that will be the source for your images. +1. Copy `Dockerfile.mbta`, `docker-compose.yaml`, `bin`, and `config` into your repository. +1. Run `mkdir -p .github/workflows` in your repository and then copy `docker.yaml` to `.github/workflows/docker.yaml` +1. Test your new Docker image by running it with `docker compose up oba_app` +1. Assuming it builds successfully, access it at http://localhost:8080 and validate it using the `bin/validate.sh` script at the root of this repo. +1. Once you have successfully validated your image, commit your changes to GitHub and create a new Release of your repository. +1. The creation of the Release will kick off a new Action to build your Docker images. It will take 10-15 minutes to create the new images. Once it finishes, you'll be able to find them in your organization's Packages page. + +### Update the Image + +To update your image—for instance to force an update to the static GTFS feed, simply increment the `"REVISION"` value. You can automate this using `sed` or manually change the value. Then, create a new Release via the GitHub UI, command line, or API. diff --git a/deployment-examples/immutable/Dockerfile.mbta b/deployment-examples/immutable/Dockerfile.mbta new file mode 100644 index 0000000..6fd3825 --- /dev/null +++ b/deployment-examples/immutable/Dockerfile.mbta @@ -0,0 +1,27 @@ +# Inherit from the base onebusaway image +FROM opentransitsoftwarefoundation/onebusaway-api-webapp:2.6.0-latest + +COPY config/tdf-log4j2.xml /usr/local/tomcat/webapps/onebusaway-transit-data-federation-webapp/WEB-INF/classes/log4j2.xml + +# Set environment variables permanently +ENV JDBC_DRIVER="org.postgresql.Driver" +ENV TZ="America/New_York" +ENV GTFS_URL="https://cdn.mbta.com/MBTA_GTFS.zip" +ENV ALERTS_URL="https://cdn.mbta.com/realtime/Alerts.pb" +ENV TRIP_UPDATES_URL="https://cdn.mbta.com/realtime/TripUpdates.pb" +ENV VEHICLE_POSITIONS_URL="https://cdn.mbta.com/realtime/VehiclePositions.pb" +ENV REFRESH_INTERVAL="30" +ENV AGENCY_ID_LIST='["1","3"]' +ENV REVISION="1" + +# Set these at runtime: +# ENV JDBC_URL="" +# ENV JDBC_USER="" +# ENV JDBC_PASSWORD="" + +# Create a directory for the custom data bundle +RUN mkdir -p /bundle +RUN /oba/build_bundle.sh + +# Unset GTFS_URL so that build_bundle.sh doesn't run after build time +ENV GTFS_URL="" diff --git a/deployment-examples/immutable/bin/detect-changed-agencies.sh b/deployment-examples/immutable/bin/detect-changed-agencies.sh new file mode 100755 index 0000000..7975948 --- /dev/null +++ b/deployment-examples/immutable/bin/detect-changed-agencies.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# detect-changed-agencies.sh + +# Usage information +function usage { + echo "Usage: $0 [--current-tag TAG] [--previous-tag TAG] [--json]" >&2 + echo " --current-tag TAG : Specify the current release tag (default: latest tag)" >&2 + echo " --previous-tag TAG : Specify the previous release tag (default: auto-detect)" >&2 + echo " --json : Output results as JSON array" >&2 + exit 1 +} + +# Parse command line arguments +JSON_OUTPUT=false +CURRENT_TAG="" +PREVIOUS_TAG="" + +while [[ "$#" -gt 0 ]]; do + case $1 in + --current-tag) CURRENT_TAG="$2"; shift ;; + --previous-tag) PREVIOUS_TAG="$2"; shift ;; + --json) JSON_OUTPUT=true ;; + -h|--help) usage ;; + *) echo "Unknown parameter: $1" >&2; usage ;; + esac + shift +done + +# If no current tag specified, use the latest tag +if [ -z "$CURRENT_TAG" ]; then + CURRENT_TAG=$(git tag --sort=-creatordate | head -n 1) + if [ -z "$CURRENT_TAG" ]; then + echo "No tags found in repository. Please specify --current-tag." >&2 + exit 1 + fi + echo "Using latest tag: $CURRENT_TAG" >&2 +fi + +# If no previous tag specified, find the previous one +if [ -z "$PREVIOUS_TAG" ]; then + PREVIOUS_TAG=$(git tag --sort=-creatordate | grep -v "$CURRENT_TAG" | head -n 1) + if [ -z "$PREVIOUS_TAG" ]; then + echo "No previous tag found. Please specify --previous-tag." >&2 + exit 1 + fi + echo "Using previous tag: $PREVIOUS_TAG" >&2 +fi + +# Get changed Dockerfiles between the tags +CHANGED_FILES=$(git diff --name-only "$PREVIOUS_TAG" "$CURRENT_TAG" | grep "^Dockerfile\." || true) + +# Extract agency names +AGENCIES=() +while read -r file; do + if [[ "$file" == Dockerfile.* ]]; then + AGENCY=$(echo "$file" | sed 's/Dockerfile\.//') + AGENCIES+=("$AGENCY") + fi +done <<< "$CHANGED_FILES" + +# Output results +if [ "${#AGENCIES[@]}" -eq 0 ]; then + echo "No Dockerfile changes detected between $PREVIOUS_TAG and $CURRENT_TAG" >&2 + if [ "$JSON_OUTPUT" = true ]; then + echo "[]" + fi + exit 0 +fi + +if [ "$JSON_OUTPUT" = true ]; then + # Output as JSON array (requires jq) - on a single line + printf '%s\n' "${AGENCIES[@]}" | sort | uniq | jq -R . | jq -s -c . +else + # Output as simple list + echo "Changed agencies between $PREVIOUS_TAG and $CURRENT_TAG:" >&2 + printf '%s\n' "${AGENCIES[@]}" | sort | uniq +fi diff --git a/deployment-examples/immutable/bin/detect-dockerfile-changes.sh b/deployment-examples/immutable/bin/detect-dockerfile-changes.sh new file mode 100755 index 0000000..fb334f4 --- /dev/null +++ b/deployment-examples/immutable/bin/detect-dockerfile-changes.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# detect-dockerfile-changes.sh + +# Get the previous and current tags (passed as arguments) +PREVIOUS_TAG=$1 +CURRENT_TAG=$2 + +if [ -z "$PREVIOUS_TAG" ] || [ -z "$CURRENT_TAG" ]; then + echo "Usage: $0 " + exit 1 +fi + +# Check if both tags exist +if ! git rev-parse --verify "$PREVIOUS_TAG" > /dev/null 2>&1; then + echo "Tag $PREVIOUS_TAG does not exist" + exit 1 +fi + +if ! git rev-parse --verify "$CURRENT_TAG" > /dev/null 2>&1; then + echo "Tag $CURRENT_TAG does not exist" + exit 1 +fi + +# Get changed Dockerfiles between the two tags +CHANGED_FILES=$(git diff --name-only "$PREVIOUS_TAG" "$CURRENT_TAG" | grep "^Dockerfile\.") + +# Output the list of files +if [ -z "$CHANGED_FILES" ]; then + echo "No Dockerfile changes detected" + exit 0 +fi + +echo "$CHANGED_FILES" \ No newline at end of file diff --git a/deployment-examples/immutable/bin/extract-agencies.sh b/deployment-examples/immutable/bin/extract-agencies.sh new file mode 100755 index 0000000..e6d7315 --- /dev/null +++ b/deployment-examples/immutable/bin/extract-agencies.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# extract-agencies.sh + +# Read the list of changed Dockerfiles (either from stdin or a file) +if [ -n "$1" ]; then + CHANGED_FILES=$(cat "$1") +else + CHANGED_FILES=$(cat) +fi + +# Extract agency names from Dockerfile changes +AGENCIES=() +while read -r file; do + if [[ "$file" == Dockerfile.* ]]; then + AGENCY=$(echo "$file" | sed 's/Dockerfile\.//') + AGENCIES+=("$AGENCY") + fi +done <<< "$CHANGED_FILES" + +# Output as a simple list by default +if [ "${#AGENCIES[@]}" -eq 0 ]; then + echo "No agencies extracted" + exit 0 +fi + +# Format output +if [ "$2" = "--json" ]; then + # Output as JSON array (requires jq) + printf '%s\n' "${AGENCIES[@]}" | jq -R . | jq -s . +else + # Output as simple list + printf '%s\n' "${AGENCIES[@]}" | sort | uniq +fi \ No newline at end of file diff --git a/deployment-examples/immutable/bin/get-previous-tag.sh b/deployment-examples/immutable/bin/get-previous-tag.sh new file mode 100755 index 0000000..4d3371e --- /dev/null +++ b/deployment-examples/immutable/bin/get-previous-tag.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# get-previous-tag.sh + +# Get the current tag (you would pass this as an argument when testing) +CURRENT_TAG=$1 + +if [ -z "$CURRENT_TAG" ]; then + echo "Usage: $0 " + exit 1 +fi + +# Get the previous tag (excluding the current one) +PREVIOUS_TAG=$(git tag --sort=-creatordate | grep -v "$CURRENT_TAG" | head -n 1) + +if [ -z "$PREVIOUS_TAG" ]; then + echo "No previous tag found" + exit 1 +fi + +echo "$PREVIOUS_TAG" \ No newline at end of file diff --git a/deployment-examples/immutable/bin/validate.sh b/deployment-examples/immutable/bin/validate.sh new file mode 100755 index 0000000..4eb8c07 --- /dev/null +++ b/deployment-examples/immutable/bin/validate.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env ruby + +require 'yaml' +require 'open-uri' +require 'json' +require 'minitest/autorun' + +API_KEY="org.onebusaway.iphone" +BASE_URL="http://localhost:8080/api/where" + +SERVICES = { + dash: { + agency_id: '1', + route_id: '1_35', + stop_id: '1_512', + lat: 38.83334, + lon: -77.09186, + route_min_count: 10 + }, + raba: { + agency_id: '25', + route_id: '25_24', + stop_id: '25_2000', + lat: 40.3061885, + lon: -122.121985, + search_radius: 20_000, + route_min_count: 10 + }, + sdmts: { + agency_id: 'MTS', + route_id: 'MTS_992', + stop_id: 'MTS_60499', + lat: 32.899853, + lon: -116.731188, + route_min_count: 100, + search_radius: 20_000, + agencies_count: 4 + }, + sta: { + agency_id: 'STA', + route_id: 'STA_63', + stop_id: 'STA_CONC', + lat: 47.622782, + lon: -117.390875, + route_min_count: 50 + }, + tampa: { + agency_id: '1', + route_id: '1_10', + stop_id: '1_4340', + lat: 27.950712, + lon: -82.397875, + route_min_count: 30, + agencies_count: 2 + }, + unitrans: { + agency_id: 'unitrans', + route_id: 'unitrans_O', + stop_id: 'unitrans_22102', + lat: 38.555308, + lon: -121.73599, + route_min_count: 20 + } +} + +def call_api(endpoint, params = {}) + url = "#{BASE_URL}/#{endpoint}?key=#{API_KEY}" + url += "&#{URI.encode_www_form(params)}" unless params.empty? + response = URI.open(url).read + JSON.parse(response) +end + +def service_name + compose = YAML.load_file(File.join(__dir__, '..', 'docker-compose.yaml')) + compose.dig('services', 'oba_app', 'build', 'dockerfile').split('.').last.to_sym +end + +def load_data + service = SERVICES[service_name] + raise "Service not found: #{service_name}" unless service + service +end + +puts "Running API tests for service: #{service_name}" + +class ApiTests < Minitest::Test + def setup + @service = load_data + end + + def test_current_time + response = call_api("current-time.json") + assert_equal(200, response["code"]) + assert_match(/\d+/, response["currentTime"].to_s) + end + + def test_agencies_with_coverage + response = call_api("agencies-with-coverage.json") + agencies = response.dig('data', 'list') + assert_equal(@service[:agencies_count] || 1, agencies.length) + agency = agencies.filter {|a| a['agencyId'] == @service[:agency_id] }.first + assert_in_delta(@service[:lat], agency['lat']) + assert_in_delta(@service[:lon], agency['lon']) + end + + def test_routes_for_agency + response = call_api("routes-for-agency/#{@service[:agency_id]}.json") + routes = response.dig('data', 'list') + assert_operator(@service[:route_min_count], :<, routes.length) + route = routes.first + assert_equal(@service[:agency_id], route['agencyId']) + refute_empty(route['nullSafeShortName']) + end + + def test_stops_for_route + response = call_api("stops-for-route/#{@service[:route_id]}.json") + entry = response.dig('data', 'entry') + assert_equal(["polylines", "routeId", "stopGroupings", "stopIds"], entry.keys) + assert_equal(@service[:route_id], entry['routeId']) + end + + def test_stop + response = call_api("stop/#{@service[:stop_id]}.json") + data = response.dig('data', 'entry') + assert_equal(@service[:stop_id], data['id']) + end + + def test_stops_for_location + params = {lat: @service[:lat], lon: @service[:lon]} + if @service[:search_radius] + params[:radius] = @service[:search_radius] + end + response = call_api("stops-for-location.json", params) + assert_operator(0, :<, response.dig('data', 'list').count) + end + + def test_arrivals_and_departures_for_stop + response = call_api("arrivals-and-departures-for-stop/#{@service[:stop_id]}.json") + data = response.dig('data', 'entry', 'arrivalsAndDepartures') + + departures = data.collect {|d| d['predictedDepartureTime'] }.compact.select {|d| d > 0 } + assert_operator(0, :<, departures.count, 'has real time data') + end +end diff --git a/deployment-examples/immutable/config/tdf-log4j2.xml b/deployment-examples/immutable/config/tdf-log4j2.xml new file mode 100644 index 0000000..730c848 --- /dev/null +++ b/deployment-examples/immutable/config/tdf-log4j2.xml @@ -0,0 +1,60 @@ + + + + + + + + + + + + %d{ISO8601} %-5p [%F:%L] : %m%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/deployment-examples/immutable/docker-compose.yaml b/deployment-examples/immutable/docker-compose.yaml new file mode 100644 index 0000000..1745bde --- /dev/null +++ b/deployment-examples/immutable/docker-compose.yaml @@ -0,0 +1,32 @@ +services: + oba_database_pg: + image: postgres:16 + container_name: oba_database_pg + environment: + POSTGRES_USER: oba_user + POSTGRES_PASSWORD: oba_password + POSTGRES_DB: oba_database + ports: + - "5432:5432" + volumes: + - type: volume + source: pg-data + target: /var/lib/postgresql/data + restart: always + + oba_app: + depends_on: + - oba_database_pg + build: + dockerfile: Dockerfile.mbta + environment: + - JDBC_URL=jdbc:postgresql://oba_database_pg:5432/oba_database + - JDBC_DRIVER=org.postgresql.Driver + - JDBC_USER=oba_user + - JDBC_PASSWORD=oba_password + + ports: + - "8080:8080" + +volumes: + pg-data: diff --git a/deployment-examples/immutable/docker.yaml b/deployment-examples/immutable/docker.yaml new file mode 100644 index 0000000..e78da47 --- /dev/null +++ b/deployment-examples/immutable/docker.yaml @@ -0,0 +1,80 @@ +name: Docker +on: + release: + types: [published] + +jobs: + detect-changes: + runs-on: ubuntu-latest + outputs: + changed_agencies: ${{ steps.detect-changes.outputs.changed_agencies }} + has_changes: ${{ steps.detect-changes.outputs.has_changes }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect changed agencies + id: detect-changes + run: | + # Make the script executable + chmod +x ./bin/detect-changed-agencies.sh + + # Get changed agencies as JSON + CHANGED_AGENCIES=$(./bin/detect-changed-agencies.sh --current-tag ${{ github.ref_name }} --json) + echo "Detected changes: $CHANGED_AGENCIES" + + # Set the matrix output + echo "changed_agencies=$CHANGED_AGENCIES" >> $GITHUB_OUTPUT + + # Check if we have any changes (empty array is "[]") + if [ "$CHANGED_AGENCIES" != "[]" ]; then + echo "has_changes=true" >> $GITHUB_OUTPUT + else + echo "has_changes=false" >> $GITHUB_OUTPUT + fi + + buildx: + needs: detect-changes + if: ${{ needs.detect-changes.outputs.has_changes == 'true' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + agency: ${{ fromJson(needs.detect-changes.outputs.changed_agencies) }} + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.agency }} + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/amd64,linux/arm64 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + - name: Inspect builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push images + uses: docker/build-push-action@v5 + with: + platforms: linux/amd64,linux/arm64 + file: Dockerfile.${{ matrix.agency }} + push: true + tags: | + ghcr.io/${{ github.repository }}-${{ matrix.agency }}:latest + ghcr.io/${{ github.repository }}-${{ matrix.agency }}:${{ github.ref_name }}