Harden telemetry delivery and refine recipe metadata

.azure_pipelines/olive-ci.yaml

@@ -30,6 +30,12 @@ pr:
     - scripts/**
     - skills/**
 
+parameters:
+- name: run_telemetry_regressions
+  displayName: Run full telemetry regression suite
+  type: boolean
+  default: false
+
 variables:
   runCodesignValidationInjection: false
   skipComponentGovernanceDetection: true
@@ -40,7 +46,10 @@ jobs:
   parameters:
     name: Linux_CPU_CI_Unit_Test
     pool: $(OLIVE_POOL_UBUNTU2004)
-    pytest_marker: 'not openvino and not amd'
+    ${{ if parameters.run_telemetry_regressions }}:
+      pytest_marker: 'not openvino and not amd'
+    ${{ else }}:
+      pytest_marker: 'not openvino and not amd and (not telemetry_regression or telemetry_smoke)'
 
 # Skip the GPU CI until the pool issue is resolved (not able to allocate GPU resources).
 # - template: job_templates/olive-test-linux-gpu-template.yaml
@@ -55,4 +64,7 @@ jobs:
     name: Windows_CPU_CI_Unit_Test
     pool: $(OLIVE_POOL_WIN2019)
     windows: True
-    pytest_marker: 'not openvino and not amd'
+    ${{ if parameters.run_telemetry_regressions }}:
+      pytest_marker: 'not openvino and not amd'
+    ${{ else }}:
+      pytest_marker: 'not openvino and not amd and (not telemetry_regression or telemetry_smoke)'

CONTRIBUTING.md

@@ -13,6 +13,10 @@ Code coverage from these tests should aim at maintaining over 80% coverage.
 
 All changes should be covered by new or existing unit tests.
 
+Regular CI includes four telemetry smoke tests. Run the full regressions with
+`python -m pytest test/test_telemetry.py test/test_telemetry_storage.py`, or enable
+`run_telemetry_regressions` when manually running Olive CI.
+
 #### Style
 
 Test the *behavior*, instead of the *implementation*. To make what a test is testing clear, the test methods should be named following the pattern `test_<method or function name>_<expected behavior>_[when_<condition>]`.

docs/Privacy.md

@@ -5,12 +5,15 @@ The software may collect information about you and your use of the software and
 
 ***
 
-## Technical Details
-Olive uses the [OpenTelemetry](https://opentelemetry.io/) API for its implementation. Telemetry is turned ON by default. Based on user consent, this data may be periodically sent to Microsoft servers following GDPR and privacy regulations for anonymity and data access controls. Application, device, and version information is collected automatically.
+## Telemetry
+Telemetry is turned ON by default. It may include workflow and build metadata, device and platform information, performance measurements, diagnostics, and a pseudonymous device identifier. Local paths and credentials are redacted.
 
-In addition, Olive may collect additional telemetry data such as:
-- Invoked commands
-- Performance data
-- Exception information
+## Disabling Telemetry
 
-Collection of this additional telemetry can be disabled by adding the `--disable_telemetry` flag to any Olive CLI command, or by setting the `OLIVE_DISABLE_TELEMETRY` environment variable to `1` before running. Telemetry is also automatically disabled when a CI/CD environment is detected (e.g., GitHub Actions, Azure Pipelines, Jenkins). If telemetry is enabled, but cannot be sent to Microsoft, it will be stored locally and sent when a connection is available. You can override the default cache location by setting the `OLIVE_TELEMETRY_CACHE_DIR` environment variable to a valid directory path.
+- Add `--disable_telemetry` to an Olive CLI command.
+- Set `OLIVE_DISABLE_TELEMETRY=1` or `ORT_DISABLE_TELEMETRY=1` before running.
+- Call `olive.telemetry.disable_telemetry()`.
+
+Opt-out applies for the rest of the process and to Olive workflow containers it starts. Runtime opt-out stops new events and uploads. An upload already admitted may finish, and retained queued events may be sent by a later telemetry-enabled process. Opt out before first use to prevent all uploads.
+
+In CI, Olive sends only recipe metadata unless telemetry is disabled.

olive/cli/api.py

@@ -19,6 +19,7 @@
 from olive.cli.run import WorkflowRunCommand
 from olive.cli.session_params_tuning import SessionParamsTuningCommand
 from olive.engine.output import WorkflowOutput
+from olive.telemetry import disable_telemetry
 
 # pylint: disable=W0212
 
@@ -127,6 +128,8 @@ def _run_unified_command(command_class, **kwargs) -> Any:
     known_kwargs = {k: v for k, v in kwargs.items() if k in args_schema}
 
     args = _create_unified_args(command_class, known_kwargs)
+    if getattr(args, "disable_telemetry", False):
+        disable_telemetry()
 
     # Check if command handles unknown_args (like FineTuneCommand)
     constructor_params = inspect.signature(command_class.__init__).parameters

olive/cli/base.py

@@ -345,7 +345,9 @@ def _run_workflow(self):
                     mark_test_output_path(self.args.output_path)
                 print("Dry run mode enabled. Configuration file is generated but no optimization is performed.")
                 return None
-            workflow_output = olive_run(run_config)
+            workflow_output = olive_run(
+                run_config, recipe_telemetry_metadata=self._get_recipe_telemetry_metadata(), emit_error_telemetry=False
+            )
             if is_test:
                 mark_test_output_path(self.args.output_path)
                 save_discrepancy_check_results(workflow_output, self.args.output_path)
@@ -356,6 +358,17 @@ def _run_workflow(self):
                 print(f"Model is saved at {self.args.output_path}")
             return workflow_output
 
+    def _get_recipe_telemetry_metadata(self) -> dict[str, str]:
+        recipe_name = self.__class__.__name__
+        if recipe_name.endswith("Command"):
+            recipe_name = recipe_name[: -len("Command")]
+        return {
+            "recipe_name": recipe_name,
+            "recipe_command": recipe_name,
+            "recipe_source": "generated_cli",
+            "recipe_format": "generated",
+        }
+
     @staticmethod
     def _parse_extra_options(kv_items):
         from onnxruntime_genai import __version__ as OrtGenaiVersion
@@ -1015,7 +1028,11 @@ def add_search_options(sub_parser: ArgumentParser):
 
 def add_telemetry_options(sub_parser: ArgumentParser):
     """Add telemetry options to the sub_parser."""
-    sub_parser.add_argument("--disable_telemetry", action="store_true", help="Disable telemetry for this command.")
+    sub_parser.add_argument(
+        "--disable_telemetry",
+        action="store_true",
+        help="Disable all Olive telemetry for this process.",
+    )
     return sub_parser
 
 

olive/cli/init/__init__.py

@@ -4,7 +4,8 @@
 # --------------------------------------------------------------------------
 from argparse import ArgumentParser
 
-from olive.cli.base import BaseOliveCLICommand
+from olive.cli.base import BaseOliveCLICommand, add_telemetry_options
+from olive.telemetry import action
 
 
 class InitCommand(BaseOliveCLICommand):
@@ -21,8 +22,10 @@ def register_subcommand(parser: ArgumentParser):
             default="./olive-output",
             help="Default output directory for the generated command. Default is ./olive-output.",
         )
+        add_telemetry_options(sub_parser)
         sub_parser.set_defaults(func=InitCommand)
 
+    @action
     def run(self):
         from olive.cli.init.wizard import InitWizard
 

olive/cli/launcher.py

@@ -24,7 +24,7 @@
 from olive.cli.run_pass import RunPassCommand
 from olive.cli.session_params_tuning import SessionParamsTuningCommand
 from olive.cli.shared_cache import SharedCacheCommand
-from olive.telemetry import Telemetry
+from olive.telemetry import Telemetry, disable_telemetry
 
 
 def get_cli_parser(called_as_console_script: bool = True) -> ArgumentParser:
@@ -66,18 +66,20 @@ def main(raw_args=None, called_as_console_script: bool = True):
 
     args, unknown_args = parser.parse_known_args(raw_args)
 
-    telemetry = Telemetry()
-    if args.disable_telemetry:
-        telemetry.disable_telemetry()
-
     if not hasattr(args, "func"):
         parser.print_help()
         sys.exit(1)
 
-    # Run the command
-    service = args.func(parser, args, unknown_args)
-    service.run()
-    telemetry.shutdown()
+    if getattr(args, "disable_telemetry", False):
+        disable_telemetry()
+    telemetry = None
+    try:
+        telemetry = Telemetry.get_or_create_if_enabled()
+        service = args.func(parser, args, unknown_args)
+        service.run()
+    finally:
+        if telemetry is not None:
+            telemetry.shutdown()
 
 
 def legacy_call(deprecated_module: str, command_name: str, *args):

olive/cli/run.py

@@ -17,6 +17,7 @@
     mark_test_output_path,
     save_discrepancy_check_results,
     validate_test_output_path,
+    warn_unused_test_metrics,
 )
 from olive.telemetry import action
 
@@ -56,29 +57,38 @@ def register_subcommand(parser: ArgumentParser):
 
     @action
     def run(self):
+        from copy import deepcopy
+
         from olive.common.config_utils import load_config_file
         from olive.workflows import run as olive_run
 
         # allow the run_config to be a dict already (for api use)
-        run_config = self.args.run_config
-        if not isinstance(run_config, dict):
-            run_config = load_config_file(run_config)
-        if "builds" in run_config and self.args.test not in (None, False):
+        run_config_input = self.args.run_config
+        run_config = (
+            deepcopy(run_config_input) if isinstance(run_config_input, dict) else load_config_file(run_config_input)
+        )
+        config_overrides = {}
+        test = getattr(self.args, "test", None)
+        test_metrics = _flatten_test_metrics(getattr(self.args, "test_metrics", None))
+        test_llama_path = getattr(self.args, "test_llama_path", None)
+        warn_unused_test_metrics(test, test_metrics, test_llama_path)
+
+        if "builds" in run_config and test not in (None, False):
             raise ValueError("--test is not supported with multi-build run configurations.")
         if input_model_config := get_input_model_config(self.args, required=False):
             print("Replacing input model config in run config")
             run_config["input_model"] = input_model_config
-        elif self.args.test not in (None, False):
+            config_overrides["input_model"] = input_model_config
+        elif test not in (None, False):
             input_model = run_config.get("input_model")
             if not isinstance(input_model, dict) or input_model.get("type", "").lower() != "hfmodel":
                 raise ValueError("--test for olive run requires a Hugging Face input_model in the run config.")
             output_path = (
                 self.args.output_path or run_config.get("output_dir") or run_config.get("engine", {}).get("output_dir")
             )
-            validate_test_output_path(output_path, self.args.test)
-            run_config["input_model"] = add_hf_test_model_config(input_model, self.args.test, output_path)
-            test_metrics = _flatten_test_metrics(getattr(self.args, "test_metrics", None))
-            run_config = add_discrepancy_check_pass(run_config, test_metrics)
+            validate_test_output_path(output_path, test)
+            run_config["input_model"] = add_hf_test_model_config(input_model, test, output_path)
+            run_config = add_discrepancy_check_pass(run_config, test_metrics, test_llama_path)
 
         for arg_key, rc_key in [("output_path", "output_dir"), ("log_level", "log_severity_level")]:
             if (arg_value := getattr(self.args, arg_key)) is not None:
@@ -87,15 +97,30 @@ def run(self):
                 run_config.get("engine", {}).pop(rc_key, None)
                 # add value to run config directly
                 run_config[rc_key] = arg_value
+                config_overrides[rc_key] = arg_value
+
+        recipe_telemetry_metadata = {
+            "recipe_command": "WorkflowRun",
+            "recipe_source": "config_dict" if isinstance(run_config_input, dict) else "config_file",
+            "recipe_format": "dict"
+            if isinstance(run_config_input, dict)
+            else Path(run_config_input).suffix.lstrip(".").lower() or "unknown",
+            "execution_mode": "list_required_packages" if self.args.list_required_packages else "run",
+            "package_config_provided": bool(self.args.package_config),
+        }
+        if config_overrides:
+            recipe_telemetry_metadata["config_overrides"] = config_overrides
 
         output_path = run_config.get("output_dir") or run_config.get("engine", {}).get("output_dir")
         workflow_output = olive_run(
             run_config,
             list_required_packages=self.args.list_required_packages,
             tempdir=self.args.tempdir,
             package_config=self.args.package_config,
+            recipe_telemetry_metadata=recipe_telemetry_metadata,
+            emit_error_telemetry=False,
         )
-        if self.args.test not in (None, False):
+        if test not in (None, False):
             mark_test_output_path(output_path)
             save_discrepancy_check_results(workflow_output, output_path)
 

olive/systems/docker/docker_system.py

@@ -232,6 +232,8 @@ def _prepare_run_params(self) -> dict:
 
     def _prepare_environment(self, base_env) -> dict:
         """Prepare environment variables for container."""
+        from olive.telemetry.telemetry import Telemetry, is_ci_environment
+
         # Convert list to dict if needed
         if isinstance(base_env, list):
             environment = {env.split("=")[0]: env.split("=")[1] for env in base_env}
@@ -241,6 +243,10 @@ def _prepare_environment(self, base_env) -> dict:
         # Add default environment variables
         environment.setdefault("PYTHONPYCACHEPREFIX", "/tmp")
         environment["OLIVE_LOG_LEVEL"] = logging.getLevelName(logger.getEffectiveLevel())
+        if is_ci_environment():
+            environment["CI"] = "1"
+        if Telemetry.is_process_telemetry_disabled():
+            environment["OLIVE_DISABLE_TELEMETRY"] = "1"
 
         # Add HuggingFace token if needed
         if self.hf_token:

olive/systems/docker/workflow_runner.py

@@ -5,6 +5,7 @@
 
 from olive.common.hf.login import huggingface_login
 from olive.logging import get_olive_logger, set_verbosity_from_env
+from olive.telemetry.telemetry import Telemetry
 from olive.workflows import run as olive_run
 
 logger = get_olive_logger()
@@ -20,7 +21,12 @@ def runner_entry(config):
         config = json.load(f)
 
     logger.info("Running workflow with config: %s", config)
-    olive_run(config)
+    try:
+        olive_run(config, emit_error_telemetry=False, emit_recipe_telemetry=False)
+    finally:
+        telemetry = Telemetry.get_existing_instance()
+        if telemetry is not None:
+            telemetry.shutdown(flush=True)
 
 
 if __name__ == "__main__":

olive/telemetry/__init__.py

@@ -2,7 +2,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 # --------------------------------------------------------------------------
-from olive.telemetry.telemetry import Telemetry
+from olive.telemetry.telemetry import Telemetry, disable_telemetry
 from olive.telemetry.telemetry_extensions import action
 
-__all__ = ["Telemetry", "action"]
+__all__ = ["Telemetry", "action", "disable_telemetry"]

olive/telemetry/deviceid/__init__.py

@@ -1,3 +1,3 @@
-from olive.telemetry.deviceid.deviceid import get_encrypted_device_id_and_status
+from olive.telemetry.deviceid.deviceid import get_hashed_device_id_and_status
 
-__all__ = ["get_encrypted_device_id_and_status"]
+__all__ = ["get_hashed_device_id_and_status"]