Harden telemetry delivery and refine recipe metadata

docs/Privacy.md

@@ -13,4 +13,4 @@ In addition, Olive may collect additional telemetry data such as:
 - Performance data
 - Exception information
 
-Collection of this additional telemetry can be disabled by adding the `--disable_telemetry` flag to any Olive CLI command, or by setting the `OLIVE_DISABLE_TELEMETRY` environment variable to `1` before running. Telemetry is also automatically disabled when a CI/CD environment is detected (e.g., GitHub Actions, Azure Pipelines, Jenkins). If telemetry is enabled, but cannot be sent to Microsoft, it will be stored locally and sent when a connection is available. You can override the default cache location by setting the `OLIVE_TELEMETRY_CACHE_DIR` environment variable to a valid directory path.
+Collection of this additional telemetry can be disabled by adding the `--disable_telemetry` flag to any Olive CLI command, or by setting the `OLIVE_DISABLE_TELEMETRY` environment variable to `1` before running. In CI/CD environments (e.g., GitHub Actions, Azure Pipelines, Jenkins), Olive suppresses the general heartbeat/action/error events and only emits the `OliveRecipe` event. The `OliveRecipe` event may include recipe metadata such as pass types, explicitly configured target settings, the host system type and any explicitly configured host accelerator settings, whether a custom package config was provided, a redacted snapshot of custom package-config overrides, and a redacted snapshot of explicitly supplied config overrides. Outside CI/CD environments, if telemetry is enabled but cannot be sent to Microsoft, it will be stored locally and sent when a connection is available. You can override the default cache location by setting the `OLIVE_TELEMETRY_CACHE_DIR` environment variable to a valid directory path.

olive/cli/base.py

@@ -42,13 +42,24 @@ def _run_workflow(self):
             if self.args.dry_run:
                 print("Dry run mode enabled. Configuration file is generated but no optimization is performed.")
                 return None
-            workflow_output = olive_run(run_config)
+            workflow_output = olive_run(run_config, recipe_telemetry_metadata=self._get_recipe_telemetry_metadata())
             if not workflow_output.has_output_model():
                 print("No output model produced. Please check the log for details.")
             else:
                 print(f"Model is saved at {self.args.output_path}")
             return workflow_output
 
+    def _get_recipe_telemetry_metadata(self) -> dict[str, str]:
+        recipe_name = self.__class__.__name__
+        if recipe_name.endswith("Command"):
+            recipe_name = recipe_name[: -len("Command")]
+        return {
+            "recipe_name": recipe_name,
+            "recipe_command": recipe_name,
+            "recipe_source": "generated_cli",
+            "recipe_format": "generated",
+        }
+
     @staticmethod
     def _parse_extra_options(kv_items):
         from onnxruntime_genai import __version__ as OrtGenaiVersion

olive/cli/run.py

@@ -49,11 +49,14 @@ def register_subcommand(parser: ArgumentParser):
 
     @action
     def run(self):
+        from pathlib import Path
+
         from olive.common.config_utils import load_config_file
         from olive.workflows import run as olive_run
 
         # allow the run_config to be a dict already (for api use)
-        run_config = self.args.run_config
+        run_config_input = self.args.run_config
+        run_config = run_config_input
         if not isinstance(run_config, dict):
             run_config = load_config_file(run_config)
         if input_model_config := get_input_model_config(self.args, required=False):
@@ -73,6 +76,15 @@ def run(self):
             list_required_packages=self.args.list_required_packages,
             tempdir=self.args.tempdir,
             package_config=self.args.package_config,
+            recipe_telemetry_metadata={
+                "recipe_command": "WorkflowRun",
+                "recipe_source": "config_dict" if isinstance(run_config_input, dict) else "config_file",
+                "recipe_format": "dict"
+                if isinstance(run_config_input, dict)
+                else Path(run_config_input).suffix.lstrip(".").lower() or "unknown",
+                "execution_mode": "list_required_packages" if self.args.list_required_packages else "run",
+                "package_config_provided": bool(self.args.package_config),
+            },
         )
 
         if self.args.list_required_packages is True:

olive/systems/docker/docker_system.py

@@ -232,6 +232,8 @@ def _prepare_run_params(self) -> dict:
 
     def _prepare_environment(self, base_env) -> dict:
         """Prepare environment variables for container."""
+        from olive.telemetry.telemetry import is_ci_environment
+
         # Convert list to dict if needed
         if isinstance(base_env, list):
             environment = {env.split("=")[0]: env.split("=")[1] for env in base_env}
@@ -241,6 +243,8 @@ def _prepare_environment(self, base_env) -> dict:
         # Add default environment variables
         environment.setdefault("PYTHONPYCACHEPREFIX", "/tmp")
         environment["OLIVE_LOG_LEVEL"] = logging.getLevelName(logger.getEffectiveLevel())
+        if is_ci_environment():
+            environment["CI"] = "1"
 
         # Add HuggingFace token if needed
         if self.hf_token:

olive/systems/docker/workflow_runner.py

@@ -20,7 +20,7 @@ def runner_entry(config):
         config = json.load(f)
 
     logger.info("Running workflow with config: %s", config)
-    olive_run(config)
+    olive_run(config, emit_recipe_telemetry=False)
 
 
 if __name__ == "__main__":

olive/telemetry/constants.py

@@ -3,6 +3,6 @@
 # Licensed under the MIT License.
 # --------------------------------------------------------------------------
 
-"""OneCollector connection string."""
+"""Telemetry constants."""
 
-CONNECTION_STRING = "SW5zdHJ1bWVudGF0aW9uS2V5PTlkNWRkYWVjNjFlMjQ1NjdiNzg4YTIwYWVhMzI0NjMxLTcyMzdkN2M2LWVlNjEtNGNmZC1iYjdiLTU5MDNhOTcyYzJlNC03MDQ3"
+CONNECTION_STRING = "SW5zdHJ1bWVudGF0aW9uS2V5PTYyMTUwOTExZGMwMDRmYzliYjY3YmE5NjA2NDI3ZTU2LWVjNjFmOWFmLTVkN2EtNGQxOS1hZjMxLWI5Y2Q2OWU5ODdmMS02OTE1"

olive/telemetry/library/options.py

@@ -62,6 +62,7 @@ class OneCollectorExporterOptions:
     """Configuration options for OneCollector exporter."""
 
     connection_string: Optional[str] = None
+    service_name: Optional[str] = None
     transport_options: OneCollectorTransportOptions = field(default_factory=OneCollectorTransportOptions)
 
     # Internal fields populated during validation

olive/telemetry/library/telemetry_logger.py

@@ -6,6 +6,7 @@
 """High-level telemetry logger facade for easy usage."""
 
 import logging
+import threading
 import uuid
 from typing import Any, Callable, Optional
 
@@ -28,6 +29,7 @@ class TelemetryLogger:
 
     _instance: Optional["TelemetryLogger"] = None
     _default_logger: Optional["TelemetryLogger"] = None
+    _singleton_lock = threading.RLock()
     _logger: Optional[logging.Logger] = None
     _logger_exporter: Optional[OneCollectorLogExporter] = None
     _logger_provider: Optional[LoggerProvider] = None
@@ -39,9 +41,10 @@ def __new__(cls, options: Optional[OneCollectorExporterOptions] = None):
             options: Exporter options (only used on first instantiation)
 
         """
-        if cls._instance is None:
-            cls._instance = super().__new__(cls)
-            cls._instance._initialize(options)
+        with cls._singleton_lock:
+            if cls._instance is None:
+                cls._instance = super().__new__(cls)
+                cls._instance._initialize(options)
 
         return cls._instance
 
@@ -57,10 +60,13 @@ def _initialize(self, options: Optional[OneCollectorExporterOptions]) -> None:
             self._logger_exporter = OneCollectorLogExporter(options=options)
 
             # Create logger provider
+            service_name = (
+                options.service_name if options and options.service_name else __name__.split(".", maxsplit=1)[0]
+            )
             self._logger_provider = LoggerProvider(
                 resource=Resource.create(
                     {
-                        "service.name": __name__.split(".", maxsplit=1)[0],
+                        "service.name": service_name,
                         "service.version": VERSION,
                         "service.instance.id": str(uuid.uuid4()),  # Unique instance ID; can double as session ID
                     }
@@ -141,21 +147,27 @@ def shutdown(self) -> None:
             self._logger_provider.shutdown()
 
     @classmethod
-    def get_default_logger(cls, connection_string: Optional[str] = None) -> "TelemetryLogger":
+    def get_default_logger(
+        cls, connection_string: Optional[str] = None, service_name: Optional[str] = None
+    ) -> "TelemetryLogger":
         """Get or create the default telemetry logger.
 
         Args:
             connection_string: OneCollector connection string (only used on first call)
+            service_name: Logical application/service name for emitted telemetry (only used on first call)
 
         Returns:
             TelemetryLogger instance
 
         """
-        if cls._default_logger is None:
-            options = None
-            if connection_string:
-                options = OneCollectorExporterOptions(connection_string=connection_string)
-            cls._default_logger = cls(options=options)
+        with cls._singleton_lock:
+            if cls._default_logger is None:
+                options = None
+                if connection_string:
+                    options = OneCollectorExporterOptions(
+                        connection_string=connection_string, service_name=service_name
+                    )
+                cls._default_logger = cls(options=options)
 
         return cls._default_logger
 
@@ -167,17 +179,20 @@ def shutdown_default_logger(cls) -> None:
             cls._default_logger = None
 
 
-def get_telemetry_logger(connection_string: Optional[str] = None) -> TelemetryLogger:
+def get_telemetry_logger(
+    connection_string: Optional[str] = None, service_name: Optional[str] = None
+) -> TelemetryLogger:
     """Get or create the default telemetry logger.
 
     Args:
         connection_string: OneCollector connection string (only used on first call)
+        service_name: Logical application/service name for emitted telemetry (only used on first call)
 
     Returns:
         TelemetryLogger instance
 
     """
-    return TelemetryLogger.get_default_logger(connection_string=connection_string)
+    return TelemetryLogger.get_default_logger(connection_string=connection_string, service_name=service_name)
 
 
 def log_event(event_name: str, attributes: Optional[dict[str, Any]] = None) -> None: