feat: experiment setting up infra for private vuln feeds

deployment/terraform/environments/private-osv/main.tf

@@ -0,0 +1,42 @@
+module "osv_pipeline" {
+  source = "../../modules/osv_pipeline"
+
+  project_id                                 = "oss-vdb-test"
+  datastore_name                             = "datastore-private"
+  worker_service_account_id                  = "worker-private"
+  vulnerabilities_export_bucket              = "osv-test-vulnerabilities-private"
+  affected_commits_backups_bucket            = "osv-test-affected-commits-private"
+  pubsub_topic_name                          = "private-tasks"
+  pubsub_topic_failed_tasks_name             = "failed-private-tasks"
+  pubsub_subscription_default_work_pool_name = "private-default-pool"
+  pubsub_subscription_recovery_name          = "private-recovery"
+  cluster_name                               = "workers-private"
+  cluster_location                           = "us-central1-f"
+  cluster_master_cidr                        = "172.16.0.80/28"
+  gitter_disk_name                           = "gitter-disk-private"
+  gitter_disk_size_gb                        = 6144
+  importer_reconciler_git_cache_disk_name    = "importer-reconciler-git-cache-private"
+  importer_reconciler_git_cache_size_gb      = 200
+  subnet_name                                = "my-subnet-0-private"
+  subnet_cidr                                = "10.45.80.0/22"
+  router_name                                = "router-private"
+  nat_name                                   = "nat-config-private"
+}
+
+
+terraform {
+  backend "gcs" {
+    bucket = "oss-vdb-tf"
+    prefix = "private-osv"
+  }
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 7.35.0"
+    }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = "~> 7.35.0"
+    }
+  }
+}

deployment/terraform/modules/osv_pipeline/database.tf

@@ -0,0 +1,55 @@
+# Datastore
+resource "google_firestore_database" "datastore" {
+  project     = var.project_id
+  name        = var.datastore_name
+  location_id = "us-west2"
+  type        = "DATASTORE_MODE"
+}
+
+# GCP Bucket where protos and full JSON exports are stored
+resource "google_storage_bucket" "osv_vulnerabilities_export" {
+  project                     = var.project_id
+  name                        = var.vulnerabilities_export_bucket
+  location                    = "US"
+  uniform_bucket_level_access = true
+
+  versioning {
+    enabled = true
+  }
+
+  lifecycle_rule {
+    action {
+      type = "Delete"
+    }
+    condition {
+      num_newer_versions = 673
+      with_state         = "ARCHIVED"
+    }
+  }
+
+  lifecycle_rule {
+    action {
+      type = "Delete"
+    }
+    condition {
+      days_since_noncurrent_time = 7
+      with_state                 = "ANY"
+    }
+  }
+}
+
+# GCP bucket where affected commits are backed up.
+resource "google_storage_bucket" "affected_commits_backups_bucket" {
+  project                     = var.project_id
+  name                        = var.affected_commits_backups_bucket
+  location                    = "US"
+  uniform_bucket_level_access = true
+  lifecycle_rule {
+    action {
+      type = "Delete"
+    }
+    condition {
+      age = var.affected_commits_backups_bucket_retention_days
+    }
+  }
+}

deployment/terraform/modules/osv_pipeline/gke.tf

@@ -0,0 +1,215 @@
+# GKE "workers" cluster and node pools
+
+resource "google_container_cluster" "workers" {
+  project    = var.project_id
+  name       = var.cluster_name
+  location   = var.cluster_location
+  subnetwork = google_compute_subnetwork.my_subnet_0.self_link
+
+  private_cluster_config {
+    enable_private_endpoint = false
+    enable_private_nodes    = true
+    master_ipv4_cidr_block  = var.cluster_master_cidr
+  }
+
+  # We need to define this for private clusters, but all fields are optional.
+  ip_allocation_policy {}
+
+  addons_config {
+    gce_persistent_disk_csi_driver_config {
+      enabled = true
+    }
+    gcp_filestore_csi_driver_config {
+      enabled = true
+    }
+  }
+
+  # We can't create a cluster with no node pool defined, but we want to only use
+  # separately managed node pools. So we create the smallest possible default
+  # node pool and immediately delete it.
+  remove_default_node_pool = true
+  initial_node_count       = 1
+  lifecycle {
+    ignore_changes = [
+      # importing from oss-vdb has initial_node_count set to 0, which is actually not a valid configuration for creating a cluster.
+      # Updating this value in terraform forces a replacement, even though the default pool is destroyed. Ignore it to prevent disruption.
+      initial_node_count,
+    ]
+  }
+
+  monitoring_config {
+    managed_prometheus {
+      enabled = true
+    }
+  }
+}
+
+resource "google_container_node_pool" "default_pool" {
+  project  = var.project_id
+  name     = "default-pool"
+  cluster  = google_container_cluster.workers.name
+  location = google_container_cluster.workers.location
+
+  lifecycle {
+    # Terraform doesn't automatically know to recreate node pools when the cluster is recreated.
+    replace_triggered_by = [
+      google_container_cluster.workers.id,
+    ]
+  }
+
+  autoscaling {
+    min_node_count  = 1
+    max_node_count  = 1000
+    location_policy = "BALANCED"
+  }
+
+
+  node_config {
+    service_account = google_service_account.worker_sa.email
+    machine_type    = "n1-highmem-2"
+    disk_type       = "pd-ssd"
+    disk_size_gb    = 64
+    local_ssd_count = 1
+
+    oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"]
+
+  }
+}
+
+resource "google_container_node_pool" "highend" {
+  project  = var.project_id
+  name     = "highend"
+  cluster  = google_container_cluster.workers.name
+  location = google_container_cluster.workers.location
+  # For using the ephemeral storage local ssd config
+  provider = google-beta
+
+  lifecycle {
+    # Terraform doesn't automatically know to recreate node pools when the cluster is recreated.
+    replace_triggered_by = [
+      google_container_cluster.workers.id,
+    ]
+  }
+
+  autoscaling {
+    min_node_count  = 0
+    max_node_count  = 100
+    location_policy = "BALANCED"
+  }
+
+
+  node_config {
+    service_account = google_service_account.worker_sa.email
+    machine_type    = "n2-highmem-32"
+    disk_type       = "pd-ssd"
+    disk_size_gb    = 100
+    ephemeral_storage_config { // This is used for emptyDir storage in kubernetes
+      // Minimum is 4 ssds for n2-highmem-32, for 375GB * 4 = 1.5TB of storage
+      local_ssd_count = 4
+    }
+
+    oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"]
+
+    labels = {
+      workloadType = "highend"
+    }
+
+    taint {
+      effect = "NO_EXECUTE"
+      key    = "workloadType"
+      value  = "highend"
+    }
+
+  }
+}
+
+resource "google_container_node_pool" "importer_pool" {
+  project    = var.project_id
+  name       = "importer-pool"
+  cluster    = google_container_cluster.workers.name
+  location   = google_container_cluster.workers.location
+  node_count = 1
+
+  lifecycle {
+    # Terraform doesn't automatically know to recreate node pools when the cluster is recreated.
+    replace_triggered_by = [
+      google_container_cluster.workers.id,
+    ]
+  }
+
+  node_config {
+    service_account = google_service_account.worker_sa.email
+    machine_type    = "n2-highmem-4"
+    disk_type       = "pd-ssd"
+    disk_size_gb    = 64
+    local_ssd_count = 1
+
+    oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"]
+
+    labels = {
+      workloadType = "importer-pool"
+    }
+
+    taint {
+      effect = "NO_EXECUTE"
+      key    = "workloadType"
+      value  = "importer-pool"
+    }
+  }
+}
+
+# Will deal with this properly when we unify test and prod.
+# resource "google_container_node_pool" "worker_pool_temp" {
+#   count    = var.project_id == "oss-vdb-test" ? 1 : 0
+#   project  = var.project_id
+#   name     = "worker-pool-temp"
+#   cluster  = google_container_cluster.workers.name
+#   location = google_container_cluster.workers.location
+# 
+#   lifecycle {
+#     replace_triggered_by = [
+#       google_container_cluster.workers.id,
+#     ]
+#   }
+# 
+#   autoscaling {
+#     min_node_count  = 0
+#     max_node_count  = 250
+#     location_policy = "BALANCED"
+#   }
+# 
+#   node_config {
+#     service_account = google_service_account.worker_sa.email
+#     machine_type    = "n4-highcpu-2"
+#     disk_type       = "hyperdisk-balanced"
+# 
+#     oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"]
+# 
+#     labels = {
+#       workloadType = "worker-pool"
+#     }
+#     taint {
+#       effect = "NO_EXECUTE"
+#       key    = "workloadType"
+#       value  = "worker-pool"
+#     }
+#   }
+# }
+
+# 6TiB SSD disk used by the gitter caching service
+resource "google_compute_disk" "gitter_disk" {
+  project = var.project_id
+  name    = var.gitter_disk_name
+  type    = "pd-ssd"
+  zone    = google_container_cluster.workers.location
+  size    = var.gitter_disk_size_gb
+}
+
+# SSD for Importer Reconciler Git Cache
+resource "google_compute_disk" "importer_reconciler_git_cache" {
+  project = var.project_id
+  name    = var.importer_reconciler_git_cache_disk_name
+  type    = "pd-ssd"
+  zone    = google_container_cluster.workers.location
+  size    = var.importer_reconciler_git_cache_size_gb
+}