feat(zebra): implement data deletion workers

Author: loadez

github_hooks/db/migrate/20251118091829_add_expires_at_to_jobs.rb

@@ -0,0 +1,5 @@
+class AddExpiresAtToJobs < ActiveRecord::Migration[6.1]
+  def change
+    add_column :jobs, :expires_at, :datetime
+  end
+end

github_hooks/db/migrate/20251119103829_add_expires_created_index_at_jobs_table.rb

@@ -0,0 +1,11 @@
+class AddExpiresCreatedIndexAtJobsTable < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def change
+    add_index :jobs,
+              %i[expires_at created_at],
+              name: "index_jobs_on_expires_created_not_null",
+              algorithm: :concurrently,
+              where: "expires_at IS NOT NULL"
+  end
+end

github_hooks/db/migrate/20251119104017_add_organization_created_index_at_jobs_table.rb

@@ -0,0 +1,11 @@
+class AddOrganizationCreatedIndexAtJobsTable < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def change
+    add_index :jobs,
+              %i[organization_id created_at],
+              name: "index_jobs_on_organization_created_expires_is_null",
+              algorithm: :concurrently,
+              where: "expires_at IS NULL"
+  end
+end

zebra/Makefile

@@ -36,6 +36,8 @@ ZEBRA_CALLBACK_TOKEN_KEYS?="testing"
 CONTAINER_ENV_VARS= \
 	-e CI=$(CI) \
 	-e MIX_ENV=$(MIX_ENV) \
+	-e START_JOB_DELETION_POLICY_MARKER=$(START_JOB_DELETION_POLICY_MARKER) \
+	-e START_JOB_DELETION_POLICY_WORKER=$(START_JOB_DELETION_POLICY_WORKER) \
 	-e START_PUBLIC_JOB_API=$(START_PUBLIC_JOB_API) \
 	-e START_INTERNAL_JOB_API=$(START_INTERNAL_JOB_API) \
 	-e START_INTERNAL_TASK_API=$(START_INTERNAL_TASK_API) \

zebra/config/config.exs

@@ -24,6 +24,14 @@ config :zebra, Zebra.Workers.TaskFinisher, timeout: 10_000
 config :zebra, Zebra.Workers.Dispatcher, timeout: 1_000
 config :zebra, Zebra.Workers.Monitor, timeout: 60_000
 
+config :zebra, Zebra.Workers.JobDeletionPolicyWorker,
+  naptime: 1_000, # 1 second
+  longnaptime: 3_600_000, # 1 hour
+  limit: 100
+
+config :zebra, Zebra.Workers.JobDeletionPolicyMarker,
+  days: 14
+
 config :zebra, Zebra.Workers.Scheduler,
   cooldown_period: 1_000,
   batch_size: 3

zebra/junit.json

@@ -388,12 +388,14 @@ defmodule InternalApi.Artifacthub.ListItem do
 
   @type t :: %__MODULE__{
           name: String.t(),
-          is_directory: boolean
+          is_directory: boolean,
+          size: integer
         }
-  defstruct [:name, :is_directory]
+  defstruct [:name, :is_directory, :size]
 
   field(:name, 1, type: :string)
   field(:is_directory, 2, type: :bool)
+  field(:size, 3, type: :int64)
 end
 
 defmodule InternalApi.Artifacthub.Artifact do

zebra/junit.xml

@@ -508,12 +508,39 @@ defmodule InternalApi.RBAC.Permission do
   field(:scope, 4, type: InternalApi.RBAC.Scope, enum: true)
 end
 
+defmodule InternalApi.RBAC.ListSubjectsRequest do
+  @moduledoc false
+  use Protobuf, syntax: :proto3
+
+  @type t :: %__MODULE__{
+          org_id: String.t(),
+          subject_ids: [String.t()]
+        }
+  defstruct [:org_id, :subject_ids]
+
+  field(:org_id, 1, type: :string)
+  field(:subject_ids, 2, repeated: true, type: :string)
+end
+
+defmodule InternalApi.RBAC.ListSubjectsResponse do
+  @moduledoc false
+  use Protobuf, syntax: :proto3
+
+  @type t :: %__MODULE__{
+          subjects: [InternalApi.RBAC.Subject.t()]
+        }
+  defstruct [:subjects]
+
+  field(:subjects, 1, repeated: true, type: InternalApi.RBAC.Subject)
+end
+
 defmodule InternalApi.RBAC.SubjectType do
   @moduledoc false
   use Protobuf, enum: true, syntax: :proto3
 
   field(:USER, 0)
   field(:GROUP, 1)
+  field(:SERVICE_ACCOUNT, 2)
 end
 
 defmodule InternalApi.RBAC.Scope do
@@ -588,6 +615,8 @@ defmodule InternalApi.RBAC.RBAC.Service do
     InternalApi.RBAC.RefreshCollaboratorsRequest,
     InternalApi.RBAC.RefreshCollaboratorsResponse
   )
+
+  rpc(:ListSubjects, InternalApi.RBAC.ListSubjectsRequest, InternalApi.RBAC.ListSubjectsResponse)
 end
 
 defmodule InternalApi.RBAC.RBAC.Stub do