From e16c50bd06e5b574cbd118328fdedc707a6118da Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Fri, 15 May 2026 13:43:40 +0300 Subject: [PATCH] Fixes, unified token support Signed-off-by: Denys Fedoryshchenko --- Makefile | 4 +- QUICKSTART.md | 3 +- examples/aws/config-arm64.json | 240 +++++++++++++++++++ examples/aws/config.json | 4 +- src/kernel_ci_cloud_labs/pull_labs_poller.py | 19 +- 5 files changed, 262 insertions(+), 8 deletions(-) create mode 100644 examples/aws/config-arm64.json diff --git a/Makefile b/Makefile index d6dc2b0..a3d114f 100644 --- a/Makefile +++ b/Makefile @@ -19,10 +19,10 @@ help: @echo " poller-once - Run a single poll cycle and exit" poller: - python -m kernel_ci_cloud_labs.pull_labs_poller --config $(CONFIG) + PYTHONPATH=src$${PYTHONPATH:+:$$PYTHONPATH} python -m kernel_ci_cloud_labs.pull_labs_poller --config $(CONFIG) poller-once: - python -m kernel_ci_cloud_labs.pull_labs_poller --config $(CONFIG) --once + PYTHONPATH=src$${PYTHONPATH:+:$$PYTHONPATH} python -m kernel_ci_cloud_labs.pull_labs_poller --config $(CONFIG) --once install: build test pip install -e . diff --git a/QUICKSTART.md b/QUICKSTART.md index 0783109..65344de 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -75,7 +75,7 @@ added alongside `test_config`: "runtime_name": "pull-labs-aws-ec2", "poll_interval_sec": 30, "cursor_file": "/tmp/pullab_cloud_cursor.json", - "kcidb_submit_url":"https://kcidb-restd.example.org/submit", + "kcidb_submit_url":"https://db.kernelci.org/submit", "kcidb_origin": "pullab_cloud_aws", "kcidb_jwt": null } @@ -92,6 +92,7 @@ variables, not committed to the file: | `KCIDB_SUBMIT_URL` | `kernelci.kcidb_submit_url` | kcidb-restd-rs submit URL | | `KCIDB_JWT` | `kernelci.kcidb_jwt` | JWT bearer token | | `KCIDB_REST` | (alternative to the two above) | `https://@host[/path]` — kci-dev-compatible single URL carrying both endpoint and token | +| `UNIFIED_TOKEN` | (fallback for `KERNELCI_API_TOKEN` *and* `KCIDB_JWT`) | Single token used for both when the dedicated env vars aren't set. Lower priority than the specific vars, higher than config-file values. | | `KCIDB_ORIGIN` | `kernelci.kcidb_origin` | Origin string in submitted rows | | `PULLAB_CURSOR_FILE` | `kernelci.cursor_file` | Where to persist the poll cursor | | `PULLAB_POLL_INTERVAL_SEC` | `kernelci.poll_interval_sec` | Sleep between empty polls | diff --git a/examples/aws/config-arm64.json b/examples/aws/config-arm64.json new file mode 100644 index 0000000..bde386d --- /dev/null +++ b/examples/aws/config-arm64.json @@ -0,0 +1,240 @@ +{ + "provider": "aws", + "region": "eu-west-2", + "storage": { + "type": "s3", + "bucket": "kernel-ci-exampleuser-results", + "results_prefix": "results" + }, + "external_storage": { + "type": "s3", + "bucket": "kernel-ci-exampleuser-storage" + }, + "force_recreate_roles": true, + "auth_credentials": { + "auth_provider": "aws" + }, + "roles": { + "kernel-ci-exampleuser-ecs-role": { + "description": "ECS task execution role with EC2 and SSM access", + "trust_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": [ + "ecs-tasks.amazonaws.com", + "ec2.amazonaws.com" + ] + }, + "Action": "sts:AssumeRole" + } + ] + }, + "policies": [ + "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy", + "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore", + "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" + ], + "inline_policies": { + "AllowPassRole": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "iam:PassRole", + "Resource": "arn:aws:iam::*:role/kernel-ci-exampleuser-ecs-role", + "Condition": { + "StringEquals": { + "iam:PassedToService": "ec2.amazonaws.com" + } + } + } + ] + }, + "AllowEC2": { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "EC2RunInstances", + "Effect": "Allow", + "Action": [ + "ec2:RunInstances", + "ec2:CreateTags" + ], + "Resource": "*" + }, + { + "Sid": "EC2TerminateOwnInstances", + "Effect": "Allow", + "Action": "ec2:TerminateInstances", + "Resource": "arn:aws:ec2:*:*:instance/*", + "Condition": { + "StringLike": { + "aws:ResourceTag/run_prefix": "run_*" + } + } + }, + { + "Sid": "EC2Describe", + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeImages", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeVpcs" + ], + "Resource": "*" + } + ] + }, + "AllowS3Access": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::kernel-ci-exampleuser-results", + "arn:aws:s3:::kernel-ci-exampleuser-results/*", + "arn:aws:s3:::kernel-ci-exampleuser-results-*", + "arn:aws:s3:::kernel-ci-exampleuser-results-*/*", + "arn:aws:s3:::kernel-ci-exampleuser-storage", + "arn:aws:s3:::kernel-ci-exampleuser-storage/*" + ] + } + ] + }, + "AllowSSMCommands": { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "SSMSendCommandDocument", + "Effect": "Allow", + "Action": "ssm:SendCommand", + "Resource": "arn:aws:ssm:*::document/AWS-RunShellScript" + }, + { + "Sid": "SSMSendCommandInstances", + "Effect": "Allow", + "Action": "ssm:SendCommand", + "Resource": "arn:aws:ec2:*:*:instance/*", + "Condition": { + "StringLike": { + "ssm:resourceTag/run_prefix": "run_*" + } + } + }, + { + "Sid": "SSMManageCommands", + "Effect": "Allow", + "Action": [ + "ssm:GetCommandInvocation", + "ssm:CancelCommand" + ], + "Resource": "*" + }, + { + "Sid": "SSMDescribeAndResolve", + "Effect": "Allow", + "Action": [ + "ssm:DescribeInstanceInformation", + "ssm:GetParameter" + ], + "Resource": "*" + } + ] + }, + "AllowIAMInstanceProfile": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:CreateInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:GetInstanceProfile" + ], + "Resource": "arn:aws:iam::*:instance-profile/kernel-ci-exampleuser-ecs-role" + } + ] + } + } + } + }, + "ecr": { + "repository": "kernel-ci-exampleuser-ecr", + "scan_on_push": false + }, + "docker": { + "dockerfile": "dockerfiles/aws/test.dockerfile", + "build_context": ".", + "force_rebuild": true + }, + "ecs": { + "cluster": "kernel-ci-exampleuser-cluster", + "task_definition": { + "family": "kernel-ci-exampleuser-task", + "cpu": "1024", + "memory": "2048", + "container_name": "kernel-ci-exampleuser-app", + "region": "eu-west-2" + } + }, + "cloudwatch": { + "log_groups": { + "/ecs/kernel-ci-exampleuser-task": { + "retention_days": 7 + }, + "/ec2/kernel-ci-exampleuser-vms": { + "retention_days": 3 + } + }, + "metrics_namespace": "kernel-ci-exampleuser-metrics" + }, + "kernelci": { + "api_base_uri": "https://staging.kernelci.org:9000/latest", + "api_token": null, + "runtime_name": "pull-labs-aws-ec2-arm64", + "poll_interval_sec": 30, + "cursor_file": "/tmp/pullab_cloud_cursor_arm64.json", + "kcidb_submit_url": "https://db.kernelci.org/submit", + "kcidb_origin": "pullab_cloud_aws_arm64", + "kcidb_jwt": null, + "comment": "arm64 (Graviton) variant. kcidb_jwt and api_token are normally provided via the KCIDB_JWT / KERNELCI_API_TOKEN env vars (or UNIFIED_TOKEN as a shared fallback for both); leave null in the file and inject at runtime." + }, + "test_config": { + "test_id": "test-all-tests", + "role_name": "kernel-ci-exampleuser-ecs-role", + "vms": [ + { + "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64", + "instance_type": "c7g.4xlarge", + "root_volume_size": 40, + "max_runtime": 3600, + "test": [ + "simple-unixbench" + ], + "min_count": 1 + }, + { + "ami_id": "resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64", + "instance_type": "c7g.4xlarge", + "root_volume_size": 40, + "max_runtime": 3600, + "test": [ + "unixbench-kernel-regression" + ], + "min_count": 1 + } + ] + } +} diff --git a/examples/aws/config.json b/examples/aws/config.json index a5304ef..e0ea898 100644 --- a/examples/aws/config.json +++ b/examples/aws/config.json @@ -206,10 +206,10 @@ "runtime_name": "pull-labs-aws-ec2", "poll_interval_sec": 30, "cursor_file": "/tmp/pullab_cloud_cursor.json", - "kcidb_submit_url": "https://kcidb-restd.example.org/submit", + "kcidb_submit_url": "https://db.kernelci.org/submit", "kcidb_origin": "pullab_cloud_aws", "kcidb_jwt": null, - "comment": "kcidb_jwt and api_token are normally provided via the KCIDB_JWT / KERNELCI_API_TOKEN env vars; leave null in the file and inject at runtime." + "comment": "kcidb_jwt and api_token are normally provided via the KCIDB_JWT / KERNELCI_API_TOKEN env vars (or UNIFIED_TOKEN as a shared fallback for both); leave null in the file and inject at runtime." }, "test_config": { "test_id": "test-all-tests", diff --git a/src/kernel_ci_cloud_labs/pull_labs_poller.py b/src/kernel_ci_cloud_labs/pull_labs_poller.py index a265649..cd9934a 100644 --- a/src/kernel_ci_cloud_labs/pull_labs_poller.py +++ b/src/kernel_ci_cloud_labs/pull_labs_poller.py @@ -62,6 +62,10 @@ # https://@[:][/path][/submit]. Used when KCIDB_SUBMIT_URL / # KCIDB_JWT are not both set. ENV_KCIDB_REST = "KCIDB_REST" +# Shared fallback when the KernelCI API token and the KCIDB JWT are the same +# value (common in single-credential deployments). Lower priority than the +# dedicated env vars but higher than config-file values. +ENV_UNIFIED_TOKEN = "UNIFIED_TOKEN" ENV_CURSOR_FILE = "PULLAB_CURSOR_FILE" ENV_POLL_INTERVAL = "PULLAB_POLL_INTERVAL_SEC" ENV_BASE_CONFIG = "PULLAB_BASE_CONFIG" @@ -226,7 +230,11 @@ def __init__( os.getenv(ENV_API_BASE_URI) or kc.get("api_base_uri"), "kernelci.api_base_uri", ) - self.api_token: Optional[str] = os.getenv(ENV_API_TOKEN) or kc.get("api_token") + self.api_token: Optional[str] = ( + os.getenv(ENV_API_TOKEN) + or os.getenv(ENV_UNIFIED_TOKEN) + or kc.get("api_token") + ) self.runtime_name: str = _required( os.getenv(ENV_RUNTIME_NAME) or kc.get("runtime_name"), "kernelci.runtime_name", @@ -237,7 +245,7 @@ def __init__( self.kcidb_submit_url: str = _required(kcidb_url, "kernelci.kcidb_submit_url") self.kcidb_jwt: str = _required( kcidb_jwt, - "kernelci.kcidb_jwt (env KCIDB_JWT or KCIDB_REST=https://@host/submit)", + "kernelci.kcidb_jwt (env KCIDB_JWT, KCIDB_REST=https://@host/submit, or UNIFIED_TOKEN)", ) self.kcidb_origin: str = _required( os.getenv(ENV_KCIDB_ORIGIN) or kc.get("kcidb_origin"), @@ -268,7 +276,9 @@ def _resolve_kcidb_endpoint( 1. KCIDB_SUBMIT_URL + KCIDB_JWT env vars (both set). 2. KCIDB_REST env var (kci-dev compatibility, format https://@host/submit). - 3. config.json: kernelci.kcidb_submit_url + kernelci.kcidb_jwt. + 3. UNIFIED_TOKEN env var as the JWT, paired with KCIDB_SUBMIT_URL + if set otherwise the config submit URL. + 4. config.json: kernelci.kcidb_submit_url + kernelci.kcidb_jwt. """ env_url = os.getenv(ENV_KCIDB_URL) env_jwt = os.getenv(ENV_KCIDB_JWT) @@ -283,6 +293,9 @@ def _resolve_kcidb_endpoint( "KCIDB_REST is set but could not be parsed — " "expected https://@host/submit" ) + unified = os.getenv(ENV_UNIFIED_TOKEN) + if unified: + return env_url or kc.get("kcidb_submit_url"), unified return kc.get("kcidb_submit_url"), kc.get("kcidb_jwt") # -- Polling --------------------------------------------------------