feat: oban scheduler for alerts, enable/disable, execution history

Makefile

@@ -343,27 +343,27 @@ deploy.prod.versioned:
 		--gcs-log-dir="gs://logflare-prod_cloudbuild-logs/logs"
 	gcloud builds submit . \
 		--config=./cloudbuild/prod/pre-deploy.yaml \
-		--substitutions=_IMAGE_TAG=$(VERSION),_NORMALIZED_IMAGE_TAG=$(NORMALIZED_VERSION),_CLUSTER=prod-b \
+		--substitutions=_IMAGE_TAG=$(VERSION),_NORMALIZED_IMAGE_TAG=$(NORMALIZED_VERSION),_CLUSTER=prod-b,_LOGFLARE_ALERTS_ENABLED=false \
 		--region=europe-west3 \
 		--gcs-log-dir="gs://logflare-prod_cloudbuild-logs/logs"
 	gcloud builds submit . \
 		--config=./cloudbuild/prod/pre-deploy.yaml \
-		--substitutions=_IMAGE_TAG=$(VERSION),_NORMALIZED_IMAGE_TAG=$(NORMALIZED_VERSION),_CLUSTER=prod-c \
+		--substitutions=_IMAGE_TAG=$(VERSION),_NORMALIZED_IMAGE_TAG=$(NORMALIZED_VERSION),_CLUSTER=prod-c,_LOGFLARE_ALERTS_ENABLED=false \
 		--region=europe-west3 \
 		--gcs-log-dir="gs://logflare-prod_cloudbuild-logs/logs"
 	gcloud builds submit . \
 		--config=./cloudbuild/prod/pre-deploy.yaml \
-		--substitutions=_IMAGE_TAG=$(VERSION),_NORMALIZED_IMAGE_TAG=$(NORMALIZED_VERSION),_CLUSTER=prod-d \
+		--substitutions=_IMAGE_TAG=$(VERSION),_NORMALIZED_IMAGE_TAG=$(NORMALIZED_VERSION),_CLUSTER=prod-d,_LOGFLARE_ALERTS_ENABLED=false \
 		--region=europe-west3 \
 		--gcs-log-dir="gs://logflare-prod_cloudbuild-logs/logs"
 	gcloud builds submit . \
 		--config=./cloudbuild/prod/pre-deploy.yaml \
-		--substitutions=_IMAGE_TAG=$(VERSION),_NORMALIZED_IMAGE_TAG=$(NORMALIZED_VERSION),_CLUSTER=prod-e \
+		--substitutions=_IMAGE_TAG=$(VERSION),_NORMALIZED_IMAGE_TAG=$(NORMALIZED_VERSION),_CLUSTER=prod-e,_LOGFLARE_ALERTS_ENABLED=false \
 		--region=europe-west3 \
 		--gcs-log-dir="gs://logflare-prod_cloudbuild-logs/logs"
 	gcloud builds submit . \
 		--config=./cloudbuild/prod/pre-deploy.yaml \
-		--substitutions=_IMAGE_TAG=$(VERSION),_NORMALIZED_IMAGE_TAG=$(NORMALIZED_VERSION),_CLUSTER=prod-f \
+		--substitutions=_IMAGE_TAG=$(VERSION),_NORMALIZED_IMAGE_TAG=$(NORMALIZED_VERSION),_CLUSTER=prod-f,_LOGFLARE_ALERTS_ENABLED=false \
 		--region=europe-west3 \
 		--gcs-log-dir="gs://logflare-prod_cloudbuild-logs/logs"
 

assets/js/app.js

@@ -126,5 +126,6 @@ window.addEventListener("phx:page-loading-stop", (_info) => {
   // enable all tooltips
   $(function () {
     $('[data-toggle="tooltip"]').tooltip({ delay: { show: 100, hide: 200 } });
+    $('[data-toggle="popover"]').popover();
   });
 });

cloudbuild/staging/deploy.yaml

@@ -22,7 +22,7 @@ steps:
       - --metadata=google-monitoring-enabled=true,google-logging-enabled=true
       - --container-privileged
       - --container-restart-policy=always
-      - --container-env=LOGFLARE_GRPC_PORT=50051,RELEASE_COOKIE=${_COOKIE},LOGFLARE_METADATA_CLUSTER=${_CLUSTER},LOGFLARE_ALERTS_MIN_CLUSTER_SIZE=3
+      - --container-env=LOGFLARE_GRPC_PORT=50051,RELEASE_COOKIE=${_COOKIE},LOGFLARE_METADATA_CLUSTER=${_CLUSTER}
       - --no-shielded-secure-boot
       - --shielded-vtpm
       - --shielded-integrity-monitoring

config/config.exs

@@ -19,7 +19,7 @@ config :logflare,
   encryption_key_fallback: hardcoded_encryption_key,
   encryption_key_default: hardcoded_encryption_key
 
-config :logflare, Logflare.Alerting, min_cluster_size: 1, enabled: true
+config :logflare, Logflare.Alerting, enabled: true
 
 config :logflare, Logflare.Google, dataset_id_append: "_default"
 
@@ -151,31 +151,9 @@ config :open_api_spex, :cache_adapter, OpenApiSpex.Plug.PersistentTermCache
 
 config :logflare, Logflare.Cluster.Utils, min_cluster_size: 1
 
-# global scheduler, only 1 per cluster
-config :logflare, Logflare.Alerting.AlertsScheduler,
-  init_task: {Logflare.Alerting, :init_alert_jobs, []}
-
-# global scheduler, only 1 per cluster
-config :logflare, Logflare.Scheduler,
-  include_task_supervisor: false,
-  task_supervisor_name: Logflare.Scheduler.TaskSupervisor,
-  jobs: [
-    # source_cleanup: [
-    #   run_strategy: {Quantum.RunStrategy.All, :cluster},
-    #   schedule: "*/30 * * * *",
-    #   task: {Logflare.Sources, :shutdown_idle_sources, []}
-    # ],
-    recent_events_touch: [
-      run_strategy: Quantum.RunStrategy.Local,
-      schedule: "*/15 * * * *",
-      task: {Logflare.Sources, :recent_events_touch, []}
-    ],
-    alerts_scheduler_sync: [
-      run_strategy: Quantum.RunStrategy.Local,
-      schedule: "0 * * * *",
-      task: {Logflare.Alerting, :sync_alert_jobs, []}
-    ]
-  ]
+config :logflare, Oban,
+  engine: Oban.Engines.Basic,
+  repo: Logflare.Repo
 
 config :opentelemetry,
   sdk_disabled: true,

config/runtime.exs

@@ -109,18 +109,6 @@ config :logflare,
        ]
        |> filter_nil_kv_pairs.()
 
-config :logflare,
-       Logflare.Alerting,
-       [
-         min_cluster_size:
-           if(System.get_env("LOGFLARE_ALERTS_MIN_CLUSTER_SIZE") != nil,
-             do: String.to_integer(System.get_env("LOGFLARE_ALERTS_MIN_CLUSTER_SIZE")),
-             else: nil
-           ),
-         enabled: System.get_env("LOGFLARE_ALERTS_ENABLED", "true") == "true"
-       ]
-       |> filter_nil_kv_pairs.()
-
 config :logflare,
        LogflareWeb.Endpoint,
        filter_nil_kv_pairs.(
@@ -465,5 +453,32 @@ syn_endpoints_partitions =
   for n <- 0..System.schedulers_online(), do: "endpoints_#{n}" |> String.to_atom()
 
 config :syn,
-  scopes: [:core, :ui, :alerting] ++ syn_endpoints_partitions,
+  scopes: [:core, :ui] ++ syn_endpoints_partitions,
   event_handler: Logflare.SynEventHandler
+
+if System.get_env("LOGFLARE_ALERTS_ENABLED", "true") == "true" do
+  config :logflare, Oban,
+    queues: [default: 10, alerts: 5],
+    plugins: [
+      {Oban.Plugins.Pruner, max_age: 86_400},
+      {Oban.Plugins.Cron,
+       crontab: [
+         {"* * * * *", Logflare.Alerting.AlertSchedulerWorker},
+         {"*/15 * * * *", Logflare.Sources.RecentEventsTouchWorker}
+       ]}
+    ]
+
+  config :logflare, Logflare.Alerting, enabled: true
+else
+  config :logflare, Oban,
+    queues: [default: 10],
+    plugins: [
+      {Oban.Plugins.Pruner, max_age: 86_400},
+      {Oban.Plugins.Cron,
+       crontab: [
+         {"*/15 * * * *", Logflare.Sources.RecentEventsTouchWorker}
+       ]}
+    ]
+
+  config :logflare, Logflare.Alerting, enabled: false
+end

config/test.exs

@@ -57,4 +57,6 @@ config :logger,
 
 config :tesla, Logflare.Backends.Adaptor.WebhookAdaptor.Client, adapter: Tesla.Mock
 
+config :logflare, Oban, testing: :manual
+
 config :phoenix_test, otp_app: :logflare, endpoint: LogflareWeb.Endpoint

docs/docs.logflare.com/docs/self-hosting/index.md

@@ -37,7 +37,6 @@ All browser authentication will be disabled when in single-tenant mode.
 | `LOGFLARE_METADATA_CLUSTER`                    | string, defaults to `nil`                                               | Sets global logging/tracing metadata for the cluster name and affects the release node name (e.g., `logflare-production@<host>`). Useful for filtering logs by cluster name and distinguishing nodes in multi-cluster setups. See the [metadata](#Metadata) section. |
 | `LOGFLARE_PUBSUB_POOL_SIZE`                    | Integer, defaults to `10`                                               | Sets the number of `Phoenix.PubSub.PG2` partitions to be created. Should be configured to the number of cores of your server for optimal multi-node performance.                                                                                                     |
 | `LOGFLARE_ALERTS_ENABLED`                      | Boolean, defaults to `true`                                             | Flag for enabling and disabling query alerts.                                                                                                                                                                                                                        |
-| `LOGFLARE_ALERTS_MIN_CLUSTER_SIZE`             | Integer, defaults to `1`                                                | Sets the required cluster size for Query Alerts to be run. If cluster size is below the provided value, query alerts will not run.                                                                                                                                   |
 | `LOGFLARE_MIN_CLUSTER_SIZE`                    | Integer, defaults to `1`                                                | Sets the target cluster size, and emits a warning log periodically if the cluster is below the set number of nodes..                                                                                                                                                 |
 | `LOGFLARE_OTEL_ENDPOINT`                       | String, defaults to `nil`                                               | Sets the OpenTelemetry Endpoint to send traces to via gRPC. Port number can be included, such as `https://logflare.app:443`                                                                                                                                          |
 | `LOGFLARE_OTEL_SOURCE_UUID`                    | String, defaults to `nil`, optionally required for OpenTelemetry.       | Sets the appropriate header for ingesting OpenTelemetry events into a Logflare source.                                                                                                                                                                               |