Skip to content

Commit e474102

Browse files
l46kokcopybara-github
l46kok
authored and
copybara-github
committed
Add support for running policy error cases in conformance tests
PiperOrigin-RevId: 911070494
1 parent 5fc1766 commit e474102

14 files changed

Lines changed: 232 additions & 122 deletions

File tree

‎conformance/src/test/java/dev/cel/conformance/policy/BUILD.bazel‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,17 @@ java_library(
1111
srcs = glob(["*.java"]),
1212
deps = [
1313
"//:auto_value",
14+
"//:java_truth",
1415
"//bundle:cel",
16+
"//bundle:environment",
17+
"//bundle:environment_yaml_parser",
18+
"//common:options",
19+
"//policy",
20+
"//policy:compiler_factory",
21+
"//policy:parser",
22+
"//policy:parser_factory",
23+
"//policy:validation_exception",
24+
"//policy/testing:k8s_test_tag_handler",
1525
"//runtime:function_binding",
1626
"//testing/testrunner:cel_expression_source",
1727
"//testing/testrunner:cel_test_context",

‎conformance/src/test/java/dev/cel/conformance/policy/PolicyConformanceTest.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,15 @@
1414

1515
package dev.cel.conformance.policy;
1616

17+
import static com.google.common.truth.Truth.assertThat;
18+
1719
import com.google.protobuf.Struct;
1820
import dev.cel.bundle.Cel;
1921
import dev.cel.bundle.CelFactory;
2022
import dev.cel.expr.conformance.proto3.TestAllTypes;
23+
import dev.cel.policy.CelPolicyParserFactory;
24+
import dev.cel.policy.CelPolicyValidationException;
25+
import dev.cel.policy.testing.K8sTagHandler;
2126
import dev.cel.runtime.CelFunctionBinding;
2227
import dev.cel.testing.testrunner.CelExpressionSource;
2328
import dev.cel.testing.testrunner.CelTestContext;
@@ -26,6 +31,7 @@
2631
import java.nio.file.Files;
2732
import java.nio.file.Path;
2833
import java.nio.file.Paths;
34+
import java.util.Locale;
2935
import org.junit.runners.model.Statement;
3036

3137
/** Statement representing a single CEL policy conformance test case. */
@@ -77,6 +83,13 @@ public void evaluate() throws Throwable {
7783
TestAllTypes.getDescriptor().getFile(),
7884
Struct.getDescriptor().getFile());
7985

86+
// Scopes the custom Kubernetes tag visitor exclusively to k8s tests to prevent non-standard
87+
// grammar leakage.
88+
if (name.startsWith("k8s/")) {
89+
contextBuilder.setCelPolicyParser(
90+
CelPolicyParserFactory.newYamlParserBuilder().addTagVisitor(new K8sTagHandler()).build());
91+
}
92+
8093
Path yamlConfigPath = Paths.get(dirPath, "config.yaml");
8194
Path textprotoConfigPath = Paths.get(dirPath, "config.textproto");
8295

@@ -86,6 +99,16 @@ public void evaluate() throws Throwable {
8699
contextBuilder.setConfigFile(textprotoConfigPath.toString());
87100
}
88101

89-
TestRunnerLibrary.runTest(testCase, contextBuilder.build());
102+
try {
103+
TestRunnerLibrary.runTest(testCase, contextBuilder.build());
104+
} catch (CelPolicyValidationException e) {
105+
if (testCase.output().kind() == CelTestCase.Output.Kind.EVAL_ERROR) {
106+
String expectedError = testCase.output().evalError().get(0).toString();
107+
assertThat(e.getMessage().toLowerCase(Locale.US))
108+
.contains(expectedError.toLowerCase(Locale.US));
109+
} else {
110+
throw e;
111+
}
112+
}
90113
}
91114
}

‎conformance/src/test/java/dev/cel/conformance/policy/PolicyConformanceTestRunner.java‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ public final class PolicyConformanceTestRunner extends ParentRunner<PolicyConfor
4444
private static final Splitter SPLITTER = Splitter.on(",").omitEmptyStrings();
4545
private static final String TESTS_YAML_FILE_NAME = "tests.yaml";
4646
private static final String TESTS_TEXTPROTO_FILE_NAME = "tests.textproto";
47+
private static final String POLICY_YAML_FILE_NAME = "policy.yaml";
4748
private static final TypeRegistry TYPE_REGISTRY =
4849
TypeRegistry.newBuilder()
4950
.add(Struct.getDescriptor())
@@ -73,12 +74,40 @@ private static ImmutableList<String> discoverTestDirs(String testdataDir) {
7374
if (!dir.exists() || !dir.isDirectory()) {
7475
return ImmutableList.of();
7576
}
76-
String[] directories = dir.list((current, name) -> new File(current, name).isDirectory());
77-
if (directories == null) {
77+
File[] topLevelDirs = dir.listFiles(File::isDirectory);
78+
if (topLevelDirs == null) {
7879
return ImmutableList.of();
7980
}
80-
Arrays.sort(directories);
81-
return ImmutableList.copyOf(directories);
81+
82+
ImmutableList.Builder<String> testDirsBuilder = ImmutableList.builder();
83+
Arrays.sort(topLevelDirs);
84+
for (File topLevelDir : topLevelDirs) {
85+
if (hasTestSuite(topLevelDir)) {
86+
testDirsBuilder.add(topLevelDir.getName());
87+
continue;
88+
}
89+
90+
// Check one level deeper to support nested tests like compile_errors/unreachable
91+
File[] subDirs = topLevelDir.listFiles(File::isDirectory);
92+
if (subDirs == null) {
93+
continue;
94+
}
95+
96+
Arrays.sort(subDirs);
97+
for (File subDir : subDirs) {
98+
if (hasTestSuite(subDir)) {
99+
testDirsBuilder.add(topLevelDir.getName() + "/" + subDir.getName());
100+
}
101+
}
102+
}
103+
104+
return testDirsBuilder.build();
105+
}
106+
107+
private static boolean hasTestSuite(File dir) {
108+
return (new File(dir, TESTS_YAML_FILE_NAME).exists()
109+
|| new File(dir, TESTS_TEXTPROTO_FILE_NAME).exists())
110+
&& new File(dir, POLICY_YAML_FILE_NAME).exists();
82111
}
83112

84113
private final ImmutableList<PolicyConformanceTest> tests;

‎policy/src/main/java/dev/cel/policy/CelPolicy.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ public abstract static class Builder implements RequiredFieldsChecker {
252252

253253
abstract Optional<Long> id();
254254

255-
abstract Optional<Result> result();
255+
public abstract Optional<Result> result();
256256

257257
abstract Optional<ValueString> explanation();
258258

‎policy/src/main/java/dev/cel/policy/RuleComposer.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ private Step optimizeRule(Cel cel, CelCompiledRule compiledRule) {
9393
assertComposedAstIsValid(
9494
cel,
9595
output.expr,
96-
"conflicting output types found.",
96+
"incompatible output types found.",
9797
matchOutput.sourceId(),
9898
lastOutputId);
9999
lastOutputId = matchOutput.sourceId();
@@ -115,7 +115,7 @@ private Step optimizeRule(Cel cel, CelCompiledRule compiledRule) {
115115
cel,
116116
output.expr,
117117
String.format(
118-
"failed composing the subrule '%s' due to conflicting output types.",
118+
"failed composing the subrule '%s' due to incompatible output types.",
119119
matchNestedRule.ruleId().map(ValueString::value).orElse("")),
120120
lastOutputId);
121121
break;

‎policy/src/main/java/dev/cel/policy/testing/BUILD.bazel‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
load("@rules_java//java:defs.bzl", "java_library")
2+
3+
package(
4+
default_applicable_licenses = [
5+
"//:license",
6+
],
7+
default_testonly = True,
8+
default_visibility = [
9+
"//policy/testing:__pkg__",
10+
],
11+
)
12+
13+
java_library(
14+
name = "k8s_tag_handler",
15+
srcs = ["K8sTagHandler.java"],
16+
tags = [
17+
],
18+
deps = [
19+
"//common/formats:value_string",
20+
"//common/formats:yaml_helper",
21+
"//policy",
22+
"//policy:parser",
23+
"//policy:policy_parser_context",
24+
"@maven//:com_google_guava_guava",
25+
"@maven//:org_yaml_snakeyaml",
26+
],
27+
)

‎policy/src/main/java/dev/cel/policy/testing/K8sTagHandler.java‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package dev.cel.policy.testing;
16+
17+
import com.google.common.annotations.VisibleForTesting;
18+
import dev.cel.common.formats.ValueString;
19+
import dev.cel.common.formats.YamlHelper;
20+
import dev.cel.common.formats.YamlHelper.YamlNodeType;
21+
import dev.cel.policy.CelPolicy;
22+
import dev.cel.policy.CelPolicy.Match;
23+
import dev.cel.policy.CelPolicyParser.TagVisitor;
24+
import dev.cel.policy.PolicyParserContext;
25+
import org.yaml.snakeyaml.nodes.Node;
26+
import org.yaml.snakeyaml.nodes.SequenceNode;
27+
28+
/**
29+
* K8sTagHandler is a {@link TagVisitor} implementation to support parsing Kubernetes
30+
* ValidatingAdmissionPolicy structures in testing and conformance environments.
31+
*/
32+
@VisibleForTesting
33+
public final class K8sTagHandler implements TagVisitor<Node> {
34+
35+
@Override
36+
public void visitPolicyTag(
37+
PolicyParserContext<Node> ctx,
38+
long id,
39+
String tagName,
40+
Node node,
41+
CelPolicy.Builder policyBuilder) {
42+
switch (tagName) {
43+
case "kind":
44+
policyBuilder.putMetadata("kind", ctx.newYamlString(node).value());
45+
break;
46+
case "metadata":
47+
YamlHelper.assertYamlType(ctx, id, node, YamlNodeType.MAP);
48+
break;
49+
case "spec":
50+
CelPolicy.Rule spec = ctx.parseRule(ctx, policyBuilder, node);
51+
policyBuilder.setRule(spec);
52+
break;
53+
default:
54+
TagVisitor.super.visitPolicyTag(ctx, id, tagName, node, policyBuilder);
55+
break;
56+
}
57+
}
58+
59+
@Override
60+
public void visitRuleTag(
61+
PolicyParserContext<Node> ctx,
62+
long id,
63+
String tagName,
64+
Node node,
65+
CelPolicy.Builder policyBuilder,
66+
CelPolicy.Rule.Builder ruleBuilder) {
67+
switch (tagName) {
68+
case "failurePolicy":
69+
policyBuilder.putMetadata(tagName, ctx.newYamlString(node).value());
70+
break;
71+
case "matchConstraints":
72+
YamlHelper.assertYamlType(ctx, id, node, YamlNodeType.MAP);
73+
break;
74+
case "validations":
75+
if (!YamlHelper.assertYamlType(ctx, id, node, YamlNodeType.LIST)) {
76+
return;
77+
}
78+
SequenceNode seqNode = (SequenceNode) node;
79+
for (Node valNode : seqNode.getValue()) {
80+
ruleBuilder.addMatches(ctx.parseMatch(ctx, policyBuilder, valNode));
81+
}
82+
break;
83+
default:
84+
TagVisitor.super.visitRuleTag(ctx, id, tagName, node, policyBuilder, ruleBuilder);
85+
break;
86+
}
87+
}
88+
89+
@Override
90+
public void visitMatchTag(
91+
PolicyParserContext<Node> ctx,
92+
long id,
93+
String tagName,
94+
Node node,
95+
CelPolicy.Builder policyBuilder,
96+
CelPolicy.Match.Builder matchBuilder) {
97+
if (!matchBuilder.result().isPresent()) {
98+
matchBuilder.setResult(
99+
Match.Result.ofOutput(ValueString.of(ctx.nextId(), "'invalid admission request'")));
100+
}
101+
switch (tagName) {
102+
case "expression":
103+
// The K8s expression to validate must return false in order to generate a violation
104+
// message.
105+
ValueString condition = ctx.newSourceString(node);
106+
String invertedCondition = "!(" + condition.value() + ")";
107+
matchBuilder.setCondition(ValueString.of(condition.id(), invertedCondition));
108+
break;
109+
case "messageExpression":
110+
matchBuilder.setResult(Match.Result.ofOutput(ctx.newSourceString(node)));
111+
break;
112+
default:
113+
TagVisitor.super.visitMatchTag(ctx, id, tagName, node, policyBuilder, matchBuilder);
114+
break;
115+
}
116+
}
117+
}

‎policy/src/test/java/dev/cel/policy/BUILD.bazel‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ java_library(
3030
"//policy:compiler_factory",
3131
"//policy:parser",
3232
"//policy:parser_factory",
33-
"//policy:policy_parser_context",
3433
"//policy:source",
3534
"//policy:validation_exception",
35+
"//policy/testing:k8s_test_tag_handler",
3636
"//runtime",
3737
"//runtime:function_binding",
3838
"//testing:cel_runtime_flavor",

‎policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,12 @@
3737
import dev.cel.extensions.CelOptionalLibrary;
3838
import dev.cel.parser.CelStandardMacro;
3939
import dev.cel.parser.CelUnparserFactory;
40-
import dev.cel.policy.PolicyTestHelper.K8sTagHandler;
4140
import dev.cel.policy.PolicyTestHelper.PolicyTestSuite;
4241
import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection;
4342
import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection.PolicyTestCase;
4443
import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection.PolicyTestCase.PolicyTestInput;
4544
import dev.cel.policy.PolicyTestHelper.TestYamlPolicy;
45+
import dev.cel.policy.testing.K8sTagHandler;
4646
import dev.cel.runtime.CelFunctionBinding;
4747
import dev.cel.runtime.CelLateFunctionBindings;
4848
import dev.cel.testing.CelRuntimeFlavor;

‎policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@
2222
import com.google.testing.junit.testparameterinjector.TestParameterInjector;
2323
import dev.cel.common.formats.ValueString;
2424
import dev.cel.policy.CelPolicy.Import;
25-
import dev.cel.policy.PolicyTestHelper.K8sTagHandler;
2625
import dev.cel.policy.PolicyTestHelper.TestYamlPolicy;
26+
import dev.cel.policy.testing.K8sTagHandler;
2727
import org.junit.Test;
2828
import org.junit.runner.RunWith;
2929

0 commit comments

Comments
 (0)