From 9f4c1ec3fc714dfea4ec34b0cf7c09590a7879ed Mon Sep 17 00:00:00 2001 From: michaelkedar Date: Wed, 17 Jun 2026 06:02:58 +0000 Subject: [PATCH 1/4] final migration --- .gitignore | 1 + AGENTS.md | 12 +++--- Makefile | 29 +++++++++----- deployment/build-and-stage.yaml | 24 ++++++----- deployment/clouddeploy/osv-api/run-prod.yaml | 4 ++ .../clouddeploy/osv-api/run-staging.yaml | 6 +++ .../IntegrationTests_api_query_response.txt | 40 +++++++++---------- gcp/api/integration_tests.py | 10 ++--- gcp/api/test_server.py | 14 +++++-- go/cmd/api-devserver/main.go | 1 + go/cmd/api/main.go | 17 ++++++-- go/internal/api/query_affected.go | 3 ++ go/internal/api/server.go | 9 +++++ 13 files changed, 112 insertions(+), 58 deletions(-) diff --git a/.gitignore b/.gitignore index 8b232e74d70..e6b838c720d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ temp/* gcp/api/v1/osv/** .hypothesis go/api-devserver +go/api esp.log \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 5cbe47518ae..b84c1cbfa7a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -163,7 +163,7 @@ Many tests use expected outputs saved directly in the source tree: ### Local API Server Development (Go-native) - To run the public OSV API server locally using the native Go implementation alongside the ESPv2 proxy (which transcodes HTTP/JSON REST requests to gRPC): ```bash - make run-go-api + make run-api-server ``` --- @@ -246,11 +246,11 @@ The `osv` folder is a shared Python package. Since the primary API server and so ## GCP Component Architecture (`gcp/`) Contains deployment setups, workers running in GKE, Cloud Functions, and the user-facing website and API. -### 1. API Server (`gcp/api/`) -- **Status**: **Active (Python)**. -- Serves the public HTTP API for querying vulnerabilities by package or version. -- **Deployment Target**: **Google Cloud Run** (managed via Cloud Deploy pipeline `osv-api`). -- *Note*: Plans exist to migrate this to Go in the near future, but it currently remains in Python. +### 1. API Server (`go/cmd/api/`) +- **Status**: **Active (Go)**. +- Serves the public OSV gRPC API server (transcoded to HTTP/JSON REST via ESPv2). +- **Deployment Target**: **Google Cloud Run** (managed via Cloud Deploy pipeline `osv-api` deploying to `osv-grpc-backend`). +- *Note*: Fully migrated from Python to Go. The legacy Python implementation remains in `gcp/api/` but is retired. ### 2. Website (`gcp/website/`) - **Status**: **Active**. diff --git a/Makefile b/Makefile index 3f61fc74532..1ea832aa6ff 100644 --- a/Makefile +++ b/Makefile @@ -45,14 +45,16 @@ go-tests: api-server-tests: test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) + cd go && go build -o ./api ./cmd/api cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest . - cd gcp/api && ./run_tests.sh $(HOME)/.config/gcloud/application_default_credentials.json - cd gcp/api && ./run_tests_e2e.sh $(HOME)/.config/gcloud/application_default_credentials.json + cd gcp/api && OSV_USE_GO_BACKEND=1 ./run_tests.sh $(HOME)/.config/gcloud/application_default_credentials.json + cd gcp/api && OSV_USE_GO_BACKEND=1 ./run_tests_e2e.sh $(HOME)/.config/gcloud/application_default_credentials.json update-api-snapshots: test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) + cd go && go build -o ./api ./cmd/api cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest . - cd gcp/api && UPDATE_SNAPS=true ./run_tests_e2e.sh $(HOME)/.config/gcloud/application_default_credentials.json + cd gcp/api && UPDATE_SNAPS=true OSV_USE_GO_BACKEND=1 ./run_tests_e2e.sh $(HOME)/.config/gcloud/application_default_credentials.json lint: GOTOOLCHAIN=go1.26.3 $(run-cmd) tools/lint_and_format.sh @@ -103,20 +105,27 @@ run-website-emulator: cd gcp/website/blog && hugo --buildFuture -d ../dist/static/blog cd gcp/website && $(install-cmd) && DATASTORE_EMULATOR_PORT=5002 $(run-cmd) python frontend_emulator.py +# Run with `make run-api-server ARGS=--no-backend` to launch esp without backend. +# Run the Go developer server orchestrator (launches both ESPv2 and the Go API server). # Run with `make run-api-server ARGS=--no-backend` to launch esp without backend. run-api-server: - test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) - cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest . - cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS) - -# Run the native Go developer server orchestrator (launches both ESPv2 and the Go API server). -# Run with `run-go-api ARGS=--no-backend` to launch esp without backend. -run-go-api: test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) docker inspect osv/esp:latest >/dev/null 2>&1 || (cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .) @cd go && go build -o ./api-devserver ./cmd/api-devserver && (GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities ./api-devserver $(ARGS); EXIT_CODE=$$?; rm -f ./api-devserver; exit $$EXIT_CODE) +# Run the Go developer server orchestrator against the staging/test environment. run-api-server-test: + test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) + docker inspect osv/esp:latest >/dev/null 2>&1 || (cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .) + @cd go && go build -o ./api-devserver ./cmd/api-devserver && (GOOGLE_CLOUD_PROJECT=oss-vdb-test OSV_VULNERABILITIES_BUCKET=osv-test-vulnerabilities ./api-devserver $(ARGS); EXIT_CODE=$$?; rm -f ./api-devserver; exit $$EXIT_CODE) + +# Legacy Python API server targets +run-python-api-server: + test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) + cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest . + cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS) + +run-python-api-server-test: test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest . cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb-test OSV_VULNERABILITIES_BUCKET=osv-test-vulnerabilities $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS) diff --git a/deployment/build-and-stage.yaml b/deployment/build-and-stage.yaml index 0e840183f6d..fe3025398c9 100644 --- a/deployment/build-and-stage.yaml +++ b/deployment/build-and-stage.yaml @@ -210,6 +210,20 @@ steps: args: ['push', '--all-tags', 'gcr.io/oss-vdb/gitter'] waitFor: ['build-gitter', 'cloud-build-queue'] +- name: gcr.io/cloud-builders/docker + entrypoint: 'bash' + args: ['-c', 'docker pull gcr.io/oss-vdb/osv-server:latest || exit 0'] + id: 'pull-osv-server' + waitFor: ['setup'] +- name: gcr.io/cloud-builders/docker + args: ['buildx', 'build', '-t', 'gcr.io/oss-vdb/osv-server:latest', '-t', 'gcr.io/oss-vdb/osv-server:$COMMIT_SHA', '--target', 'api', '--build-context', 'bindings=../bindings', '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/osv-server:latest', '--pull', '.'] + dir: 'go' + id: 'build-osv-server' + waitFor: ['pull-osv-server', 'build-gitter'] +- name: gcr.io/cloud-builders/docker + args: ['push', '--all-tags', 'gcr.io/oss-vdb/osv-server'] + waitFor: ['build-osv-server', 'cloud-build-queue'] + # Build/push staging-api-test images to gcr.io/oss-vdb-test. - name: gcr.io/cloud-builders/docker args: ['build', '-t', 'gcr.io/oss-vdb-test/staging-api-test:latest', '-t', 'gcr.io/oss-vdb-test/staging-api-test:$COMMIT_SHA', '.'] @@ -352,16 +366,6 @@ steps: args: ['push', '--all-tags', 'gcr.io/oss-vdb/run_first_package_finder'] waitFor: ['build-first-package-finder', 'cloud-build-queue'] - -# Build/push api backend -- name: 'gcr.io/cloud-builders/docker' - args: ['build', '-t', 'gcr.io/oss-vdb/osv-server:latest', '-t', 'gcr.io/oss-vdb/osv-server:$COMMIT_SHA', '-f', 'gcp/api/Dockerfile', '.'] - id: 'build-osv-server' - waitFor: ['setup'] -- name: 'gcr.io/cloud-builders/docker' - args: ['push', '--all-tags', 'gcr.io/oss-vdb/osv-server'] - waitFor: ['build-osv-server', 'cloud-build-queue'] - # Build/push Debian copyright mirror image - name: gcr.io/cloud-builders/docker args: ['build', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:latest', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA', '.'] diff --git a/deployment/clouddeploy/osv-api/run-prod.yaml b/deployment/clouddeploy/osv-api/run-prod.yaml index a35228df38b..8fb560a67fd 100644 --- a/deployment/clouddeploy/osv-api/run-prod.yaml +++ b/deployment/clouddeploy/osv-api/run-prod.yaml @@ -11,8 +11,12 @@ spec: containers: - image: osv-server env: + - name: GOOGLE_CLOUD_PROJECT + value: oss-vdb - name: OSV_VULNERABILITIES_BUCKET value: osv-vulnerabilities + - name: FAILED_TASKS_TOPIC + value: failed-tasks resources: limits: cpu: 2 diff --git a/deployment/clouddeploy/osv-api/run-staging.yaml b/deployment/clouddeploy/osv-api/run-staging.yaml index 27c17dc5117..00c67f4f159 100644 --- a/deployment/clouddeploy/osv-api/run-staging.yaml +++ b/deployment/clouddeploy/osv-api/run-staging.yaml @@ -11,8 +11,14 @@ spec: containers: - image: osv-server env: + - name: GOOGLE_CLOUD_PROJECT + value: oss-vdb-test - name: OSV_VULNERABILITIES_BUCKET value: osv-test-vulnerabilities + - name: FAILED_TASKS_TOPIC + value: failed-tasks + - name: OSV_VERBOSE_LOGGING + value: "true" resources: limits: cpu: 2 diff --git a/gcp/api/fixtures/IntegrationTests_api_query_response.txt b/gcp/api/fixtures/IntegrationTests_api_query_response.txt index 2ae4c52f345..2104a2792d9 100644 --- a/gcp/api/fixtures/IntegrationTests_api_query_response.txt +++ b/gcp/api/fixtures/IntegrationTests_api_query_response.txt @@ -1,25 +1,25 @@ [ '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:cargo/crossbeam-utils"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:cargo/crossbeam-utils"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:cargo/crossbeam-utils@0.8.5"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:cargo/crossbeam-utils@0.8.5"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:pypi/numpy"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:pypi/numpy"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:pypi/numpy@8.24.0"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:pypi/numpy@8.24.0"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' @@ -30,24 +30,24 @@ '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": ' '"pkg:cargo/crossbeam-utils"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": ' '"pkg:cargo/crossbeam-utils"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": ' '"pkg:cargo/crossbeam-utils@0.8.5"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": ' '"pkg:cargo/crossbeam-utils@0.8.5"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": "pkg:pypi/numpy"}, ' '"version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": "pkg:pypi/numpy"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": ' '"pkg:pypi/numpy@8.24.0"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": ' '"pkg:pypi/numpy@8.24.0"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' @@ -57,22 +57,22 @@ '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:cargo/crossbeam-utils"}, ' '"version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:cargo/crossbeam-utils"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:cargo/crossbeam-utils@0.8.5"}, ' '"version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:cargo/crossbeam-utils@0.8.5"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:pypi/numpy"}, "version": ' '"0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:pypi/numpy"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:pypi/numpy@8.24.0"}, "version": ' '"0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:pypi/numpy@8.24.0"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io"}, "version": "0.8.5"}\n', @@ -81,22 +81,22 @@ '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:cargo/crossbeam-utils"}, ' '"version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:cargo/crossbeam-utils"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:cargo/crossbeam-utils@0.8.5"}, ' '"version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:cargo/crossbeam-utils@0.8.5"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:pypi/numpy"}, "version": ' '"0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:pypi/numpy"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:pypi/numpy@8.24.0"}, "version": ' '"0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:pypi/numpy@8.24.0"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils"}, "version": "0.8.5"}\n', @@ -105,20 +105,20 @@ '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:cargo/crossbeam-utils"}, "version": ' '"0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:cargo/crossbeam-utils"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:cargo/crossbeam-utils@0.8.5"}, "version": ' '"0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:cargo/crossbeam-utils@0.8.5"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:pypi/numpy"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:pypi/numpy"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:pypi/numpy@8.24.0"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:pypi/numpy@8.24.0"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy"}, "version": "0.8.5"}\n', diff --git a/gcp/api/integration_tests.py b/gcp/api/integration_tests.py index 44d85b7b5c6..c99c60ff180 100644 --- a/gcp/api/integration_tests.py +++ b/gcp/api/integration_tests.py @@ -29,7 +29,7 @@ from osv import tests _PORT = 8080 -_TIMEOUT = 10 # Timeout for HTTP(S) requests +_TIMEOUT = 15 # Timeout for HTTP(S) requests _LONG_TESTS = os.getenv('LONG_TESTS') _TEST_DATA_DIR = 'fixtures' _BASE_QUERY = '/v1/query' @@ -399,7 +399,7 @@ def test_query_invalid_ecosystem(self): self.assert_results_equal({ 'code': 3, - 'message': 'Invalid ecosystem.' + 'message': 'invalid ecosystem' }, response.json()) def test_query_unknown_purl_invalid_semver(self): @@ -492,8 +492,8 @@ def test_query_semver_multiple_package(self): timeout=_TIMEOUT) response_json = response.json() - self.assertEqual(2, len(response_json['vulns'])) - self.assertCountEqual(['GHSA-6fc8-4gx4-v693', 'GHSA-3h5v-q93c-6h6q'], + self.assertEqual(3, len(response_json['vulns'])) + self.assertCountEqual(['GHSA-6fc8-4gx4-v693', 'GHSA-3h5v-q93c-6h6q', 'GHSA-96hv-2xvq-fx4p'], [vuln['id'] for vuln in response_json['vulns']]) def test_query_purl(self): @@ -694,7 +694,7 @@ def test_get_vuln_by_alias_not_in_db(self): self.assert_results_equal( { 'code': 5, - 'message': 'Bug not found, but the following aliases were: ' + 'message': 'Vulnerability not found, but the following aliases were: ' 'MAL-2024-2291' }, response.json()) diff --git a/gcp/api/test_server.py b/gcp/api/test_server.py index ea7efbda0d0..7570f1b40ab 100644 --- a/gcp/api/test_server.py +++ b/gcp/api/test_server.py @@ -49,11 +49,17 @@ def start_backend(port, log_path): log_handle = open(log_path, 'w') env = os.environ.copy() + if os.getenv('OSV_USE_GO_BACKEND') == '1': + # Use the Go backend. We assume it has been compiled. + # Path is relative to gcp/api/ directory. + backend_cmd = ['../../go/api', f'--port={port}'] + print(f"Starting Go backend: {' '.join(backend_cmd)}") + else: + backend_cmd = [sys.executable, 'server.py', f'--port={port}'] + print(f"Starting Python backend: {' '.join(backend_cmd)}") + backend_proc = subprocess.Popen( - [sys.executable, 'server.py', f'--port={port}'], - env=env, - stdout=log_handle, - stderr=subprocess.STDOUT) + backend_cmd, env=env, stdout=log_handle, stderr=subprocess.STDOUT) return backend_proc diff --git a/go/cmd/api-devserver/main.go b/go/cmd/api-devserver/main.go index 7da15163458..b6e72a74dfc 100644 --- a/go/cmd/api-devserver/main.go +++ b/go/cmd/api-devserver/main.go @@ -205,6 +205,7 @@ func runBackend(ctx context.Context, port int) { repoIndexStore := db.NewRepoIndexStore(dbClient) if err := api.RunServer(ctx, api.ServerOptions{ Port: port, + Local: true, VerboseLogs: true, VulnStore: vulnStore, RelationsStore: relationsStore, diff --git a/go/cmd/api/main.go b/go/cmd/api/main.go index 9d9a366cbe7..04ff6e2ff37 100644 --- a/go/cmd/api/main.go +++ b/go/cmd/api/main.go @@ -33,12 +33,22 @@ func run() error { logger.InitGlobalLogger() defer logger.Close() - port := flag.Int("port", 8000, "port for the OSV API") - flag.Parse() - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() + defaultPort := 8000 + if portStr := os.Getenv("PORT"); portStr != "" { + if p, err := strconv.Atoi(portStr); err == nil { + defaultPort = p + } else { + logger.ErrorContext(ctx, "Invalid PORT environment variable, using default", slog.Any("error", err)) + } + } + + port := flag.Int("port", defaultPort, "port for the OSV API") + local := flag.Bool("local", false, "enable gRPC reflection for local debugging") + flag.Parse() + project := os.Getenv("GOOGLE_CLOUD_PROJECT") if project == "" { // Fallback to metadata server for Cloud Run @@ -109,6 +119,7 @@ func run() error { return api.RunServer(ctx, api.ServerOptions{ Port: *port, + Local: *local, VerboseLogs: verboseLogs, VulnStore: vulnStore, RelationsStore: relationsStore, diff --git a/go/internal/api/query_affected.go b/go/internal/api/query_affected.go index dbbb12e067b..aadc80a2a80 100644 --- a/go/internal/api/query_affected.go +++ b/go/internal/api/query_affected.go @@ -423,6 +423,9 @@ func (s *server) parseQuery(query *pb.Query) (parsedQueryInfo, error) { if qi.packageName == "" { return parsedQueryInfo{}, status.Error(codes.InvalidArgument, "invalid query") } + if qi.ecosystem == "" && qi.version == "" { + return parsedQueryInfo{}, status.Error(codes.InvalidArgument, "invalid query") + } return qi, nil } diff --git a/go/internal/api/server.go b/go/internal/api/server.go index 026899012d4..6e10d1e7865 100644 --- a/go/internal/api/server.go +++ b/go/internal/api/server.go @@ -13,6 +13,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/health" healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/reflection" pb "osv.dev/bindings/go/api" ) @@ -34,6 +35,8 @@ type server struct { type ServerOptions struct { Port int + // Local controls whether to enable gRPC server reflection. + Local bool // VerboseLogs controls whether to log verbose information, // including per-request data. VerboseLogs bool @@ -63,8 +66,14 @@ func RunServer(ctx context.Context, opts ServerOptions) error { }) healthServer := health.NewServer() + healthServer.SetServingStatus("osv.v1.OSV", healthgrpc.HealthCheckResponse_SERVING) + healthServer.SetServingStatus("", healthgrpc.HealthCheckResponse_SERVING) healthgrpc.RegisterHealthServer(s, healthServer) + if opts.Local { + reflection.Register(s) + } + logger.InfoContext(ctx, "server listening", "port", opts.Port) serveErr := make(chan error, 1) From dce3f0ea54c2ecbff80eb1b100b7073710cb119f Mon Sep 17 00:00:00 2001 From: michaelkedar Date: Wed, 17 Jun 2026 06:16:36 +0000 Subject: [PATCH 2/4] lint --- gcp/api/integration_tests.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gcp/api/integration_tests.py b/gcp/api/integration_tests.py index c99c60ff180..bc52251605c 100644 --- a/gcp/api/integration_tests.py +++ b/gcp/api/integration_tests.py @@ -493,8 +493,9 @@ def test_query_semver_multiple_package(self): response_json = response.json() self.assertEqual(3, len(response_json['vulns'])) - self.assertCountEqual(['GHSA-6fc8-4gx4-v693', 'GHSA-3h5v-q93c-6h6q', 'GHSA-96hv-2xvq-fx4p'], - [vuln['id'] for vuln in response_json['vulns']]) + self.assertCountEqual( + ['GHSA-6fc8-4gx4-v693', 'GHSA-3h5v-q93c-6h6q', 'GHSA-96hv-2xvq-fx4p'], + [vuln['id'] for vuln in response_json['vulns']]) def test_query_purl(self): """Test querying by PURL.""" @@ -694,8 +695,9 @@ def test_get_vuln_by_alias_not_in_db(self): self.assert_results_equal( { 'code': 5, - 'message': 'Vulnerability not found, but the following aliases were: ' - 'MAL-2024-2291' + 'message': + 'Vulnerability not found, but the following aliases were: ' + 'MAL-2024-2291' }, response.json()) def test_query_batch(self): From 1f89f987bb588434fb1839abcebb14ff7d1ae0a5 Mon Sep 17 00:00:00 2001 From: michaelkedar Date: Wed, 17 Jun 2026 23:32:20 +0000 Subject: [PATCH 3/4] use go in cloud build --- gcp/api/cloudbuild.yaml | 1 + gcp/api/snapshot_tests.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/gcp/api/cloudbuild.yaml b/gcp/api/cloudbuild.yaml index 9e4d4c876bd..6fe1ab901fb 100644 --- a/gcp/api/cloudbuild.yaml +++ b/gcp/api/cloudbuild.yaml @@ -47,3 +47,4 @@ options: logging: CLOUD_LOGGING_ONLY env: - CLOUDBUILD=1 + - OSV_USE_GO_BACKEND=1 diff --git a/gcp/api/snapshot_tests.yaml b/gcp/api/snapshot_tests.yaml index fbcabdf22b2..14cf478f0b7 100644 --- a/gcp/api/snapshot_tests.yaml +++ b/gcp/api/snapshot_tests.yaml @@ -38,3 +38,4 @@ options: logging: CLOUD_LOGGING_ONLY env: - CLOUDBUILD=1 + - OSV_USE_GO_BACKEND=1 From 48677b245a5ee85c35dd63d995a409a7acc3c0e7 Mon Sep 17 00:00:00 2001 From: michaelkedar Date: Wed, 17 Jun 2026 23:49:41 +0000 Subject: [PATCH 4/4] build the go binary we're testing --- gcp/api/cloudbuild.yaml | 8 +++++++- gcp/api/snapshot_tests.yaml | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/gcp/api/cloudbuild.yaml b/gcp/api/cloudbuild.yaml index 6fe1ab901fb..e0b5cb62c06 100644 --- a/gcp/api/cloudbuild.yaml +++ b/gcp/api/cloudbuild.yaml @@ -34,12 +34,18 @@ steps: args: ['poetry', 'sync'] waitFor: ['-'] +- name: 'gcr.io/oss-vdb/ci' + id: 'build-go-backend' + dir: go + args: ['go', 'build', '-o', 'api', './cmd/api'] + waitFor: ['init'] + - name: 'gcr.io/oss-vdb/ci' id: 'api-tests' dir: gcp/api #TODO: Update test scripts to support not supplying a credential. args: ['bash', '-ex', 'run_tests.sh', '/workspace/dummy.json'] - waitFor: ['init', 'sync'] + waitFor: ['build-go-backend', 'sync'] timeout: 7200s serviceAccount: 'projects/oss-vdb/serviceAccounts/api-e2e-tester@oss-vdb.iam.gserviceaccount.com' diff --git a/gcp/api/snapshot_tests.yaml b/gcp/api/snapshot_tests.yaml index 14cf478f0b7..4f4381dd621 100644 --- a/gcp/api/snapshot_tests.yaml +++ b/gcp/api/snapshot_tests.yaml @@ -27,10 +27,17 @@ steps: dir: gcp/api args: ['poetry', 'sync'] +- name: 'gcr.io/oss-vdb/ci' + id: 'build-go-backend' + dir: go + args: ['go', 'build', '-o', 'api', './cmd/api'] + waitFor: ['-'] + - name: 'gcr.io/oss-vdb/ci' id: 'api-snapshot-tests' dir: gcp/api args: ['bash', '-ex', 'run_tests_e2e.sh', '/workspace/dummy.json'] + waitFor: ['build-go-backend', 'sync'] timeout: 7200s serviceAccount: 'projects/oss-vdb/serviceAccounts/api-e2e-tester@oss-vdb.iam.gserviceaccount.com'