diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..3dbfbb4 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +omit = tests/* \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 91702c5..f8481a3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,8 +3,8 @@ name: CI on: push: branches: - - main - - release-* + - main + - release-* pull_request: {} workflow_dispatch: inputs: @@ -53,22 +53,38 @@ jobs: # - name: Lint # run: hatch run lint:check - unit-test: + test: runs-on: ubuntu-24.04 steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Setup Hatch - run: pipx install hatch==1.14.1 - - - name: Run Unit Tests - run: hatch run test:unit + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Setup Hatch + run: pipx install hatch==1.14.1 + + - name: Run Unit Tests + run: hatch run test:ci + + - name: Pytest coverage comment + uses: MishaKav/pytest-coverage-comment@v1.1.54 + with: + badge-title: Coverage + title: Coverage Report + pytest-xml-coverage-path: reports/pytest-coverage.xml + junitxml-title: Unit Tests + junitxml-path: reports/pytest-junit.xml + #hide-badge: false + #hide-report: false + #create-new-comment: false + #hide-comment: false + #report-only-changed-files: false + #remove-link-from-badge: false + #unique-id-for-comment: python3.8 # We want to build most packages for the amd64 and arm64 architectures. To # speed this up we build single-platform packages in parallel. We then upload diff --git a/.gitignore b/.gitignore index eb97897..c11df0c 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ coverage.xml .hypothesis/ .pytest_cache/ cover/ +reports/ # Translations *.mo diff --git a/README.md b/README.md index cae126b..06b895e 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ region = request.observed.composite.resource.spec.region region = request['observed']['composite']['resource']['spec']['region'] ``` Getting values from free form map and list values will not throw -errors for keys that do not exist, but will return an empty placeholder +errors for keys that do not exist, but will return an unknown placeholder which evaluates as False. For example, the following will evaluate as False with a just created RunFunctionResponse message: ```python @@ -97,7 +97,7 @@ Note that maps or lists that do exist but do not have any members will evaluate as True, contrary to Python dicts and lists. Use the `len` function to test if the map or list exists and has members. -When setting fields, all empty intermediary placeholders will automatically +When setting fields, all intermediary unknown placeholders will automatically be created. For example, this will create all items needed to set the region on the desired resource: ```python @@ -114,6 +114,7 @@ The following functions are provided to create Protobuf structures: | ----- | ----------- | | Map | Create a new Protobuf map | | List | Create a new Protobuf list | +| Unknown | Create a new Protobuf unknown placeholder | | Yaml | Create a new Protobuf structure from a yaml string | | Json | Create a new Protobuf structure from a json string | | Base64Encode | Encode a string into base 64 | diff --git a/examples/function-go-templating/conditions/README.md b/examples/function-go-templating/conditions/README.md deleted file mode 100644 index ce1b09d..0000000 --- a/examples/function-go-templating/conditions/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Writing to the Composite or Claim Status - -function-pythonic can write to the Composite or Claim Status. See [Communication Between Composition Functions and the Claim](https://github.com/crossplane/crossplane/blob/main/design/one-pager-fn-claim-conditions.md) for more information. - -## Testing This Function Locally - -You can run your function locally and test it using [`crossplane render`](https://docs.crossplane.io/latest/cli/command-reference/#render) -with these example manifests. - -```shell -crossplane render \ - xr.yaml composition.yaml functions.yaml -``` - -## Debugging This Function - -First we need to run the command in debug mode. In a terminal Window Run: - -```shell -# Run the function locally -$ go run . --insecure --debug -``` - -Next, set the python function `render.crossplane.io/runtime: Development` annotation so that -`crossplane render` communicates with the local process instead of downloading an image: - -```yaml -apiVersion: pkg.crossplane.io/v1beta1 -kind: Function -metadata: - name: function-pythonic - annotations: - render.crossplane.io/runtime: Development -spec: - package: xpkg.upbound.io/crossplane-contrib/function-pythonic:v0.1.0 -``` - -While the function is running in one terminal, open another terminal window and run `crossplane render`. -The function should output debug-level logs in the terminal. diff --git a/examples/function-go-templating/context/README.md b/examples/function-go-templating/context/README.md deleted file mode 100644 index 21587cd..0000000 --- a/examples/function-go-templating/context/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Writing to the Function Context - -function-go-templating can write to the Function Context - -## Testing This Function Locally - -You can run your function locally and test it using [`crossplane render`](https://docs.crossplane.io/latest/cli/command-reference/#render) -with these example manifests. - -```shell -crossplane render \ - --extra-resources environmentConfigs.yaml \ - --include-context \ - xr.yaml composition.yaml functions.yaml -``` - -Will produce an output like: - -```shell ---- -apiVersion: example.crossplane.io/v1 -kind: XR -metadata: - name: example-xr -status: - conditions: - - lastTransitionTime: "2024-01-01T00:00:00Z" - reason: Available - status: "True" - type: Ready - fromEnv: e ---- -apiVersion: render.crossplane.io/v1beta1 -fields: - apiextensions.crossplane.io/environment: - apiVersion: internal.crossplane.io/v1alpha1 - array: - - "1" - - "2" - complex: - a: b - c: - d: e - f: "1" - kind: Environment - nestedEnvUpdate: - hello: world - update: environment - newkey: - hello: world - other-context-key: - complex: - a: b - c: - d: e - f: "1" -kind: Context -``` - -## Debugging This Function - -First we need to run the command in debug mode. In a terminal Window Run: - -```shell -# Run the function locally -$ go run . --insecure --debug -``` - -Next, set the go-templating function `render.crossplane.io/runtime: Development` annotation so that -`crossplane render` communicates with the local process instead of downloading an image: - -```yaml -apiVersion: pkg.crossplane.io/v1beta1 -kind: Function -metadata: - name: crossplane-contrib-function-go-templating - annotations: - render.crossplane.io/runtime: Development -spec: - package: xpkg.upbound.io/crossplane-contrib/function-go-templating:v0.6.0 -``` - -While the function is running in one terminal, open another terminal window and run `crossplane render`. -The function should output debug-level logs in the terminal. diff --git a/examples/function-go-templating/extra-resources/composition.yaml b/examples/function-go-templating/extra-resources/composition.yaml index 3af7b2f..2c99563 100644 --- a/examples/function-go-templating/extra-resources/composition.yaml +++ b/examples/function-go-templating/extra-resources/composition.yaml @@ -8,7 +8,6 @@ spec: kind: XR mode: Pipeline pipeline: - - step: render-templates functionRef: name: function-pythonic @@ -18,7 +17,7 @@ spec: composite: | class Composite(BaseComposite): def compose(self): - buckets = self.requireds.bucket('s3.aws.upbound.io/v1beta1', 'Bucket', f"my-awesome-{self.spec.environment}-bucket") + buckets = self.requireds.bucket('s3.aws.upbound.io/v1beta1', 'Bucket', name=f"my-awesome-{self.spec.environment}-bucket") for ix, bucket in enumerate(buckets): r = self.resources[f"bucket-configmap-{ix}"]('kubernetes.crossplane.io/v1alpha1', 'Object') r.spec.providerConfigRef.name = 'kubernetes' diff --git a/examples/function-go-templating/extra-resources/extraResources.yaml b/examples/function-go-templating/extra-resources/extraResources.yaml index 8f3f48b..9b62ec7 100644 --- a/examples/function-go-templating/extra-resources/extraResources.yaml +++ b/examples/function-go-templating/extra-resources/extraResources.yaml @@ -11,3 +11,4 @@ spec: status: atProvider: id: random-bucket-id + diff --git a/examples/function-go-templating/functions/getComposedResource/README.md b/examples/function-go-templating/functions/getComposedResource/README.md deleted file mode 100644 index 4f2301d..0000000 --- a/examples/function-go-templating/functions/getComposedResource/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# getComposedResource -The getComposedResource function is a utility function used to facilitate the retrieval of composed resources within templated configurations, specifically targeting observed resources. By accepting a function request map and a resource name, it navigates the complex structure of a request to fetch the specified composed resource, making it easier and more user-friendly to access nested data. If the resource is found, it returns a map containing the resource's manifest; if not, it returns nil, indicating the resource does not exist or is inaccessible through the given path. -## Usage - - -Examples: - -```golang -// Retrieve the observed resource named "flexServer" from the function request -{{ $flexServer := getComposedResource . "flexServer" }} - -// Extract values from the observed resource -{{ $flexServerID := get $flexServer.status.atProvider "id" }} - - -``` diff --git a/examples/function-go-templating/functions/getComposedResource/composition.yaml b/examples/function-go-templating/functions/getComposedResource/composition.yaml index 7c0829c..13006a4 100644 --- a/examples/function-go-templating/functions/getComposedResource/composition.yaml +++ b/examples/function-go-templating/functions/getComposedResource/composition.yaml @@ -23,7 +23,6 @@ spec: s = self.resources.flexServer(apiVersion, 'FlexibleServer') s.spec.providerConfigRef.name = 'my-provider-cfg' s.spec.forProvider.storageMb = 32768 - if s.status.atProvider.id: - c = self.resources.flexServerConfig(apiVersion, 'FlexibleServerConfiguration') - c.spec.providerConfigRef.name = 'my-provider-cfg' - c.spec.forProvider.serverId = s.status.atProvider.id + c = self.resources.flexServerConfig(apiVersion, 'FlexibleServerConfiguration') + c.spec.providerConfigRef.name = 'my-provider-cfg' + c.spec.forProvider.serverId = s.status.atProvider.id diff --git a/examples/function-go-templating/functions/getCompositeResource/README.md b/examples/function-go-templating/functions/getCompositeResource/README.md deleted file mode 100644 index 661efce..0000000 --- a/examples/function-go-templating/functions/getCompositeResource/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# getCompositeResource -The getCompositeResource function is a utility function used to facilitate the retrieval of a composite resources (XR) within templated configurations. Upon successful retrieval, the function returns a map containing the observed composite resource's manifest. If the resource cannot be located or is unreachable, it returns nil, indicating the absence or inaccessibility of the composite resource. -## Usage - - -Examples: -Given the following XR spec -```yaml -apiVersion: example.crossplane.io/v1beta1 -kind: XR -metadata: - name: example -spec: - name: "example" - location: "eastus" - -``` -```golang -// Retrieve the observed composite resource (XR) from the function request -{{ $xr := getCompositeResource . }} - -apiVersion: example.crossplane.io/v1beta1 -kind: ExampleResource -// 'Patch' values from the composite resource into the composed resource -spec: - forProvider: - name: {{ get $xr.spec "name" }} - location: {{ get $xr.spec "location" }} - -``` diff --git a/examples/function-go-templating/functions/getResourceCondition/README.md b/examples/function-go-templating/functions/getResourceCondition/README.md deleted file mode 100644 index 56616eb..0000000 --- a/examples/function-go-templating/functions/getResourceCondition/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# getResourceCondition - -## Usage - -```golang -{{ getResourceCondition $conditionType $resource }} -{{ $resource | getResourceCondition $conditionType }} -``` - -Examples: - -```golang -// Print whole condition -{{ .observed.resources.project | getResourceCondition "Ready" | toYaml }} - -// Check status -{{ if eq (.observed.resources.project | getResourceCondition "Ready").Status "True" }} - // do something -{{ end }} -``` - -See example composition for more usage examples - -## Example Outputs - -Requested resource does not exist or does not have the requested condition - -```yaml -lasttransitiontime: "0001-01-01T00:00:00Z" -message: "" -reason: "" -status: Unknown -type: Ready -``` - -Requested resource does have the requested condition - -```yaml -lasttransitiontime: "2023-11-03T10:07:31+01:00" -message: "custom message" -reason: foo -status: "True" -type: Ready -``` diff --git a/examples/function-go-templating/functions/include/README.md b/examples/function-go-templating/functions/include/README.md deleted file mode 100644 index c69b19e..0000000 --- a/examples/function-go-templating/functions/include/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# getResourceCondition - -## Usage - -```golang -{{ include $name $context }} -``` - -Examples: - -```golang -// Print whole condition -{{ include "template-name" . | nindent 4 }} -{{ $output:= include "template-name" . }} - -``` - -See example composition for more usage examples \ No newline at end of file diff --git a/examples/function-go-templating/functions/include/composition.yaml b/examples/function-go-templating/functions/include/composition.yaml index 88d0b31..2700f54 100644 --- a/examples/function-go-templating/functions/include/composition.yaml +++ b/examples/function-go-templating/functions/include/composition.yaml @@ -8,7 +8,6 @@ spec: kind: XR mode: Pipeline pipeline: - - step: render-templates functionRef: name: function-pythonic diff --git a/examples/helm-copy-secret/composition.yaml b/examples/helm-copy-secret/composition.yaml index 4a43145..f71b73a 100644 --- a/examples/helm-copy-secret/composition.yaml +++ b/examples/helm-copy-secret/composition.yaml @@ -18,9 +18,10 @@ spec: def argcd_secret_config(secret): config = Map() config.tlsClientConfig.insecure = True - config.tlsClientConfig.caData = B64Decode(secret.data['certificate-authority']) - config.tlsClientConfig.certData = B64Decode(secret.data['client-certificate']) - config.tlsClientConfig.keyData = B64Decode(secret.data['client-key']) + # ArgoCD wants these fields to be B64 encoded, so don't decode them + config.tlsClientConfig.caData = secret.data['certificate-authority'] + config.tlsClientConfig.certData = secret.data['client-certificate'] + config.tlsClientConfig.keyData = secret.data['client-key'] return config class Composite(BaseComposite): @@ -38,18 +39,17 @@ spec: secret_name = f'vc-{name}' # This will work once crossplane-sdk-python is updated to the v2 function api - #vcluster_secrets = self.requireds.Secret('v1', 'Secret', namespace, secret_name) - vcluster_secrets = self.requireds.Secret('v1', 'Secret', labels={'vcluster-name':name}) + #vcluster_secrets = self.requireds.secrets('v1', 'Secret', namespace, secret_name) + vcluster_secrets = self.requireds.secrets('v1', 'Secret', labels={'vcluster-name':name}) for secret in vcluster_secrets: - if secret.metadata.name != secret_name: - continue - argocd_secret = self.resources.secret('v1', 'Secret', 'argocd', secret_name) - argocd_secret.metadata.labels['argocd.argoproj.io/secret-type'] = 'cluster' - argocd_secret.type = 'Opaque' - argocd_secret.data.name = B64Encode(name) - argocd_secret.data.server = B64Encode(f'https://{name}.{namespace}:443') - argocd_secret.data.config = B64Encode(format(argcd_secret_config(secret), 'json')) - argocd_secret.ready = argocd_secret.observed.data - break + if secret.metadata.namespace == namespace and secret.metadata.name == secret_name: + argocd_secret = self.resources.secret('v1', 'Secret', 'argocd', secret_name) + argocd_secret.metadata.labels['argocd.argoproj.io/secret-type'] = 'cluster' + argocd_secret.type = 'Opaque' + argocd_secret.data.name = B64Encode(name) + argocd_secret.data.server = B64Encode(f'https://{name}.{namespace}:443') + argocd_secret.data.config = B64Encode(format(argcd_secret_config(secret), 'json')) + argocd_secret.ready = argocd_secret.observed.data + break else: self.ready = False diff --git a/examples/helm-copy-secret/xrd.yaml b/examples/helm-copy-secret/xrd.yaml new file mode 100644 index 0000000..ebb9eca --- /dev/null +++ b/examples/helm-copy-secret/xrd.yaml @@ -0,0 +1,21 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: CompositeResourceDefinition +metadata: + name: xclusters.example.joebowbeer.com +spec: + group: example.joebowbeer.com + names: + kind: XCluster + plural: xclusters + scope: Cluster + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: {} diff --git a/function/composite.py b/function/composite.py index 3813dce..47cdd58 100644 --- a/function/composite.py +++ b/function/composite.py @@ -469,7 +469,7 @@ def _protobuf_value(self): 'message': self.message or '', } time = self.lastTransitionTime - if time is not None: + if time: value['lastTransitionTime'] = time.isoformat().replace('+00:00', 'Z') return value @@ -535,7 +535,7 @@ def lastTransitionTime(self): for observed in self._conditions._observed.resource.status.conditions: if observed.type == self.type: time = observed.lastTransitionTime - if time is not None: + if time: return datetime.datetime.fromisoformat(time) return None diff --git a/function/fn.py b/function/fn.py index 802afaf..6e0de12 100644 --- a/function/fn.py +++ b/function/fn.py @@ -100,12 +100,12 @@ async def RunFunction( return response - class Module: def __init__(self): self.BaseComposite = function.composite.BaseComposite self.Map = function.protobuf.Map self.List = function.protobuf.List + self.Unknown = function.protobuf.Unknown self.Yaml = function.protobuf.Yaml self.Json = function.protobuf.Json self.B64Encode = lambda s: base64.b64encode(s.encode('utf-8')).decode('utf-8') diff --git a/function/protobuf.py b/function/protobuf.py index ccd64a9..a142df9 100644 --- a/function/protobuf.py +++ b/function/protobuf.py @@ -22,16 +22,13 @@ def Map(**kwargs): - values = Values(None, None, google.protobuf.struct_pb2.Struct(), Values.Type.MAP) - for key, value in kwargs.items(): - values[key] = value - return values + return Values(None, None, google.protobuf.struct_pb2.Struct(), Values.Type.MAP)(**kwargs) def List(*args): - values = Values(None, None, google.protobuf.struct_pb2.ListValue(), Values.Type.LIST) - for ix, value in enumerate(args): - values[ix] = value - return values + return Values(None, None, google.protobuf.struct_pb2.ListValue(), Values.Type.LIST)(*args) + +def Unknown(): + return Values(None, None, None, Values.Type.UNKNOWN) def Yaml(string, readOnly=None): return _Object(yaml.safe_load(string), readOnly) @@ -102,7 +99,7 @@ def __contains__(self, key): return key in self._descriptor.fields_by_name def __iter__(self): - for key in self._descriptor.fields_by_name: + for key in sorted(self._descriptor.fields_by_name): yield key, self[key] def __hash__(self): @@ -241,7 +238,7 @@ def __contains__(self, key): def __iter__(self): if self._messages is not None: - for key in self._messages: + for key in sorted(self._messages): yield key, self[key] def __hash__(self): @@ -544,16 +541,12 @@ def __contains__(self, item): for value in self: if item == value: return True - if isinstance(item, Values) and item._isUnknown: - return bool(self._unknowns) return False def __iter__(self): if self._values is not None: if self._isMap: - for key in self._values: - yield key, self[key] - for key in self._unknowns: + for key in sorted(set(self._values) | self._unknowns): yield key, self[key] elif self._isList: for ix in range(len(self._values)): @@ -618,7 +611,10 @@ def _create_child(self, key, type): raise ValueError('Invalid key, must be a str for maps') self.__dict__['_type'] = self.Type.MAP if self._values is None: - self.__dict__['_values'] = self._parent._create_child(self._key, self._type) + if self._parent is None: + self.__dict__['_values'] = google.protobuf.struct_pb2.Struct() + else: + self.__dict__['_values'] = self._parent._create_child(self._key, self._type) struct_value = self._values.fields[key] elif isinstance(key, int): if not self._isList: @@ -626,7 +622,10 @@ def _create_child(self, key, type): raise ValueError('Invalid key, must be an int for lists') self.__dict__['_type'] = self.Type.LIST if self._values is None: - self.__dict__['_values'] = self._parent._create_child(self._key, self._type) + if self._parent is None: + self.__dict__['_values'] = google.protobuf.struct_pb2.ListValue() + else: + self.__dict__['_values'] = self._parent._create_child(self._key, self._type) while key >= len(self._values.values): self._values.values.add() struct_value = self._values.values[key] @@ -671,13 +670,6 @@ def __call__(self, *args, **kwargs): self._values.Clear() for key in range(len(args)): self[key] = args[key] - else: - if not self._isMap: - if not self._isUnknown: - self.__dict__['_type'] = self.Type.MAP # Assume a map is wanted - if self._values is None: - self.__dict__['_values'] = self._parent._create_child(self._key, self._type) - self._values.Clear() return self def __setattr__(self, key, value): @@ -692,7 +684,10 @@ def __setitem__(self, key, value): raise ValueError('Invalid key, must be a str for maps') self.__dict__['_type'] = self.Type.MAP if self._values is None: - self.__dict__['_values'] = self._parent._create_child(self._key, self._type) + if self._parent is None: + self.__dict__['_values'] = google.protobuf.struct_pb2.Struct() + else: + self.__dict__['_values'] = self._parent._create_child(self._key, self._type) values = self._values.fields elif isinstance(key, int): if not self._isList: @@ -700,7 +695,10 @@ def __setitem__(self, key, value): raise ValueError('Invalid key, must be an int for lists') self.__dict__['_type'] = self.Type.LIST if self._values is None: - self.__dict__['_values'] = self._parent._create_child(self._key, self._type) + if self._parent is None: + self.__dict__['_values'] = google.protobuf.struct_pb2.ListValue() + else: + self.__dict__['_values'] = self._parent._create_child(self._key, self._type) values = self._values.values while key >= len(values): values.add() @@ -720,25 +718,17 @@ def __setitem__(self, key, value): values[key].number_value = value elif isinstance(value, dict): values[key].struct_value.Clear() - sv = self[key] - for k, v in value.items(): - sv[k] = v + self[key](**value) elif isinstance(value, (list, tuple)): values[key].list_value.Clear() - sv = self[key] - for k, v in enumerate(value): - sv[k] = v + self[key](*value) elif isinstance(value, Values): if value._isMap: values[key].struct_value.Clear() - sv = self[key] - for k, v in value: - sv[k] = v + self[key](**{k:v for k,v in value}) elif value._isList: values[key].list_value.Clear() - sv = self[key] - for k, v in enumerate(value): - sv[k] = v + self[key](*[v for v in value]) else: self._unknowns.add(key) if self._isMap: @@ -780,7 +770,7 @@ def __delitem__(self, key): del self._values[key] self._cache.pop(key, None) self._unknowns.discard(key) - for ix in sorted([ix in self._unknowns]): + for ix in sorted(self._unknowns): if ix > key: self._cache.pop(ix, None) self._unknowns.add(ix - 1) diff --git a/pyproject.toml b/pyproject.toml index e028a3b..ce591b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,20 +43,29 @@ validate-bump = false # Allow going from 0.0.0.dev0+x to 0.0.0.dev0+y type = "virtual" path = ".venv-default" dependencies = ["ipython==9.1.0"] -scripts = { development = "python function/main.py --insecure --debug" } +[tool.hatch.envs.default.scripts] +development = "python function/main.py --insecure --debug" [tool.hatch.envs.lint] type = "virtual" detached = true path = ".venv-lint" dependencies = ["ruff==0.11.5"] -scripts = { check = "ruff check function tests && ruff format --diff function tests" } +[tool.hatch.envs.lint.scripts] +check = "ruff check function tests && ruff format --diff function tests" [tool.hatch.envs.test] type = "virtual" path = ".venv-test" -dependencies = ["pytest==8.4.1", "pytest-asyncio==1.1.0"] -scripts = { unit = "python -m pytest tests" } +dependencies = [ + "pytest==8.4.1", + "pytest-asyncio==1.1.0", + "pytest-cov==6.2.1", +] +[tool.hatch.envs.test.scripts] +all = "pytest tests/ --verbose --verbose --cov --cov-report=term --cov-report=html:reports" +protobuf = "pytest tests/test_protobuf*.py --verbose --verbose --cov --cov-report=term --cov-report=html:reports" +ci = "pytest tests --junitxml=reports/pytest-junit.xml --cov --cov-report=xml:reports/pytest-coverage.xml" [tool.ruff] target-version = "py311" diff --git a/tests/fn_cases/composed.yaml b/tests/fn_cases/composed.yaml new file mode 100644 index 0000000..ebaf2f0 --- /dev/null +++ b/tests/fn_cases/composed.yaml @@ -0,0 +1,57 @@ +request: + observed: + composite: + resource: + metadata: + name: my-app + spec: + image: nginx + resources: + flexServer: + resource: + status: + atProvider: + id: abcdef + conditions: + - type: Ready + reason: foo + status: 'True' + lastTransitionTime: '2023-11-03T09:07:31Z' + input: + composite: | + apiVersion = 'dbforpostgresql.azure.upbound.io/v1beta1' + class Composite(BaseComposite): + def compose(self): + self.ttl = (5 * 60) + 30 + s = self.resources.flexServer(apiVersion, 'FlexibleServer') + s.spec.providerConfigRef.name = 'my-provider-cfg' + s.spec.forProvider.storageMb = 32768 + c = self.resources.flexServerConfig(apiVersion, 'FlexibleServerConfiguration') + c.spec.providerConfigRef.name = 'my-provider-cfg' + c.spec.forProvider.serverId = s.status.atProvider.id + +response: + meta: + ttl: + seconds: 330 + desired: + resources: + flexServer: + resource: + apiVersion: dbforpostgresql.azure.upbound.io/v1beta1 + kind: FlexibleServer + spec: + providerConfigRef: + name: my-provider-cfg + forProvider: + storageMb: 32768 + ready: 1 + flexServerConfig: + resource: + apiVersion: dbforpostgresql.azure.upbound.io/v1beta1 + kind: FlexibleServerConfiguration + spec: + providerConfigRef: + name: my-provider-cfg + forProvider: + serverId: abcdef diff --git a/tests/fn_cases/conditions.yaml b/tests/fn_cases/conditions.yaml new file mode 100644 index 0000000..a16a51a --- /dev/null +++ b/tests/fn_cases/conditions.yaml @@ -0,0 +1,85 @@ +request: + observed: + composite: + resource: + status: + conditions: + - type: Ready + reason: xr foo + status: 'True' + lastTransitionTime: '2023-11-03T09:07:31Z' + resources: + project: + resource: + status: + atProvider: + id: abcdef + conditions: + - type: Ready + reason: foo + status: 'True' + lastTransitionTime: '2023-11-03T09:07:31Z' + input: + composite: | + class Composite(BaseComposite): + def compose(self): + self.status.compositeCondition = self.conditions.Ready + self.status.compositeNotFound = self.conditions.Other + self.status.composedCondition = self.resources.project.conditions.Ready + if self.resources.project.conditions.Ready.status: + self.status.projectId = self.resources.project.status.atProvider.id + self.status.pipeline = self.resources.project.conditions.Ready + self.status.nonResource = self.resources.whatever.conditions.Ready + + self.conditions.TestCondition(False, 'InstallFail', 'failed to install') + self.conditions.ConditionTrue(True, 'TrueCondition', 'we are true', True) + self.conditions.DatabaseReady(True, 'Ready', 'Database is ready') + +response: + desired: + composite: + resource: + status: + compositeCondition: + type: Ready + reason: xr foo + message: '' + status: 'True' + lastTransitionTime: '2023-11-03T09:07:31Z' + compositeNotFound: + type: Other + message: '' + reason: '' + status: Unknown + composedCondition: + type: Ready + reason: foo + message: '' + status: 'True' + lastTransitionTime: '2023-11-03T09:07:31Z' + projectId: abcdef + pipeline: + type: Ready + reason: foo + message: '' + status: 'True' + lastTransitionTime: '2023-11-03T09:07:31Z' + nonResource: + type: Ready + reason: '' + message: '' + status: Unknown + conditions: + - type: TestCondition + reason: InstallFail + message: failed to install + status: 3 + - type: ConditionTrue + reason: TrueCondition + message: we are true + status: 2 + target: 2 + - type: DatabaseReady + reason: Ready + message: Database is ready + status: 2 diff --git a/tests/fn_cases/context.yaml b/tests/fn_cases/context.yaml new file mode 100644 index 0000000..d6a24d3 --- /dev/null +++ b/tests/fn_cases/context.yaml @@ -0,0 +1,46 @@ +request: + context: + apiextensions.crossplane.io/environment: + complex: + a: b + c: + d: e + f: '1' + input: + composite: | + class Composite(BaseComposite): + def compose(self): + self.environment.update = 'environment' + self.environment.nestedEnvUpdate.hello = 'world' + self.environment.array = ['1', '2'] + self.context['other-context-key'].complex = self.environment.complex + self.context.newKey.hello = 'world' + self.status.fromEnv = self.environment.complex.c.d + +response: + context: + apiextensions.crossplane.io/environment: + complex: + a: b + c: + d: e + f: '1' + update: environment + nestedEnvUpdate: + hello: world + array: + - '1' + - '2' + other-context-key: + complex: + a: b + c: + d: e + f: '1' + newKey: + hello: world + desired: + composite: + resource: + status: + fromEnv: e diff --git a/tests/fn_cases/do-nothing.yaml b/tests/fn_cases/do-nothing.yaml new file mode 100644 index 0000000..947f80f --- /dev/null +++ b/tests/fn_cases/do-nothing.yaml @@ -0,0 +1,6 @@ +request: + input: + composite: | + class Composite(BaseComposite): + def compose(self): + pass diff --git a/tests/fn_cases/extra-resources.yaml b/tests/fn_cases/extra-resources.yaml new file mode 100644 index 0000000..ecc8371 --- /dev/null +++ b/tests/fn_cases/extra-resources.yaml @@ -0,0 +1,67 @@ +request: + observed: + composite: + resource: + spec: + environment: dev + extra_resources: + bucket: + items: + - resource: + apiVersion: s3.aws.upbound.io/v1beta1 + kind: Bucket + metadata: + labels: + testing.upbound.io/example-name: bucket-notification + name: my-awesome-dev-bucket + spec: + forProvider: + region: us-west-1 + status: + atProvider: + id: random-bucket-id + input: + composite: | + class Composite(BaseComposite): + def compose(self): + buckets = self.requireds.bucket('s3.aws.upbound.io/v1beta1', 'Bucket', name=f"my-awesome-{self.spec.environment}-bucket") + for ix, bucket in enumerate(buckets): + r = self.resources[f"bucket-configmap-{ix}"]('kubernetes.crossplane.io/v1alpha1', 'Object') + r.spec.providerConfigRef.name = 'kubernetes' + manifest = r.spec.forProvider.manifest + manifest.apiVersion = 'v1' + manifest.kind = 'ConfigMap' + manifest.metadata.namespace = 'examples' + manifest.metadata.name = f"{bucket.metadata.name}-bucket" + manifest.data.bucket = bucket.status.atProvider.id + self.status.dummy = 'cool-status' + +response: + requirements: + extra_resources: + bucket: + api_version: s3.aws.upbound.io/v1beta1 + kind: Bucket + match_name: my-awesome-dev-bucket + desired: + composite: + resource: + status: + dummy: cool-status + resources: + bucket-configmap-0: + resource: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + spec: + providerConfigRef: + name: kubernetes + forProvider: + manifest: + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: examples + name: my-awesome-dev-bucket-bucket + data: + bucket: random-bucket-id diff --git a/tests/fn_cases/get-started-app.yaml b/tests/fn_cases/get-started-app.yaml new file mode 100644 index 0000000..5273c5e --- /dev/null +++ b/tests/fn_cases/get-started-app.yaml @@ -0,0 +1,95 @@ +request: + observed: + composite: + resource: + metadata: + name: my-app + spec: + image: nginx + resources: + deployment: + resource: + status: + availableReplicas: 2 + conditions: + - type: Available + status: 'True' + reason: MinimumReplicasAvailable + message: Deployment has minimum availability + service: + resource: + spec: + clusterIP: 10.96.196.65 + input: + composite: | + class Composite(BaseComposite): + def compose(self): + labels = {'example.crossplane.io/app': self.metadata.name} + + d = self.resources.deployment('apps/v1', 'Deployment') + d.metadata.labels = labels + d.spec.replicas = 2 + d.spec.selector.matchLabels = labels + d.spec.template.metadata.labels = labels + d.spec.template.spec.containers[0].name = 'app' + d.spec.template.spec.containers[0].image = self.spec.image + d.spec.template.spec.containers[0].ports[0].containerPort = 80 + d.ready = d.conditions.Available.status + + s = self.resources.service('v1', 'Service') + s.metadata.labels = labels + s.spec.selector = labels + s.spec.ports[0].protocol = 'TCP' + s.spec.ports[0].port = 8080 + s.spec.ports[0].targetPort = 80 + s.ready = s.observed.spec.clusterIP + + self.status.replicas = d.status.availableReplicas + self.status.address = s.observed.spec.clusterIP + +response: + desired: + composite: + resource: + status: + replicas: 2 + address: 10.96.196.65 + resources: + deployment: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + example.crossplane.io/app: my-app + spec: + replicas: 2 + selector: + matchLabels: + example.crossplane.io/app: my-app + template: + metadata: + labels: + example.crossplane.io/app: my-app + spec: + containers: + - image: nginx + name: app + ports: + - containerPort: 80 + ready: 1 + service: + resource: + apiVersion: v1 + kind: Service + metadata: + labels: + example.crossplane.io/app: my-app + spec: + selector: + example.crossplane.io/app: my-app + ports: + - protocol: TCP + port: 8080 + targetPort: 80 + ready: 1 diff --git a/tests/fn_cases/yaml.yaml b/tests/fn_cases/yaml.yaml new file mode 100644 index 0000000..3c7f13c --- /dev/null +++ b/tests/fn_cases/yaml.yaml @@ -0,0 +1,36 @@ +request: + observed: + composite: + resource: + spec: + yamlBlob: | + key1: value1 + key2: value2 + key3: value3 + complexDictionary: + list: + - abc + - def + scalar1: true + scalar2: text + scalar3: 123 + input: + composite: | + class Composite(BaseComposite): + def compose(self): + self.status.fromYaml = Yaml(self.spec.yamlBlob).key2 + self.status.toYaml = format(self.spec.complexDictionary, 'yaml') + +response: + desired: + composite: + resource: + status: + fromYaml: value2 + toYaml: | + list: + - abc + - def + scalar1: true + scalar2: text + scalar3: 123 diff --git a/tests/test_fn.py b/tests/test_fn.py index 641b5c9..5000c45 100644 --- a/tests/test_fn.py +++ b/tests/test_fn.py @@ -4,7 +4,6 @@ import pathlib import pytest -import yaml from crossplane.function.proto.v1 import run_function_pb2 as fnv1 from google.protobuf import json_format @@ -22,7 +21,7 @@ ) @pytest.mark.asyncio async def test_run_function(fn_case): - test = yaml.safe_load(fn_case.read_text()) + test = utils.yaml_load(fn_case.read_text()) request = fnv1.RunFunctionRequest( observed=fnv1.State( @@ -38,20 +37,20 @@ async def test_run_function(fn_case): ), ) utils.message_merge(request, test['request']) + if 'response' not in test: + test['response'] = {} utils.map_defaults(test['response'], { 'meta': { 'ttl': { 'seconds': 60, }, }, - 'context': {} + 'context': {}, + 'desired': {}, }) response = utils.message_dict( await fn.FunctionRunner().RunFunction(request, None) ) - #print(yaml.dump(response)) - #assert False - assert response == test['response'] diff --git a/tests/test_protobuf_values.py b/tests/test_protobuf_values.py new file mode 100644 index 0000000..dbac55f --- /dev/null +++ b/tests/test_protobuf_values.py @@ -0,0 +1,187 @@ + +from function import protobuf + + +def test_map(): + value = protobuf.Map(a=1, b=2) + assert len(value) == 2 + assert value.a == 1 + assert value['a'] == 1 + assert value.b == 2 + value.c = 3 + assert len(value) == 3 + assert value.c == 3 + +def test_list(): + value = protobuf.List(1, 2) + assert len(value) == 2 + assert value[0] == 1 + assert value[1] == 2 + value[2] = 3 + assert len(value) == 3 + assert value[2] == 3 + +def test_unkown(): + value = protobuf.Unknown() + assert not value + assert value._isUnknown + value.a = 1 + assert value + assert not value._isUnknown + assert value._isMap + value = protobuf.Unknown() + assert not value + assert value._isUnknown + value[0] = 1 + assert value + assert not value._isUnknown + assert value._isList + +def test_yaml(): + value = protobuf.Yaml(''' +a: 1 +b: 2 +''') + assert isinstance(value, protobuf.Values) + assert value._isMap + assert len(value) == 2 + value = protobuf.Yaml(''' +- 1 +- 2 +''') + assert isinstance(value, protobuf.Values) + assert value._isList + assert len(value) == 2 + value = protobuf.Yaml('test') + assert isinstance(value, str) + value = protobuf.Yaml('1') + assert isinstance(value, int) + value = protobuf.Yaml('1.2') + assert isinstance(value, float) + +def test_json(): + value = protobuf.Json(''' +{ + "a": 1, + "b": 2 +} +''') + assert isinstance(value, protobuf.Values) + assert value._isMap + assert len(value) == 2 + value = protobuf.Json(''' +[ + 1, + 2 +] +''') + assert isinstance(value, protobuf.Values) + assert value._isList + assert len(value) == 2 + value = protobuf.Json('"test"') + assert isinstance(value, str) + value = protobuf.Json('1') + assert isinstance(value, int) + value = protobuf.Json('1.2') + assert isinstance(value, float) + +def test_values_map(): + values = protobuf.Map( + map=protobuf.Map(), + map2={}, + list=protobuf.List(), + list2=[], + unknown=protobuf.Unknown(), + none=None, + str='test', + int=1, + float=1.1, + bool=True, + ) + assert isinstance(values.map, protobuf.Values) + assert values.map._isMap + assert isinstance(values.map2, protobuf.Values) + assert values.map2._isMap + assert isinstance(values.list, protobuf.Values) + assert values.list._isList + assert isinstance(values.list2, protobuf.Values) + assert values.list2._isList + assert isinstance(values.unknown, protobuf.Values) + assert values.unknown._isUnknown + assert values.none is None + assert isinstance(values.str, str) + assert values.str == 'test' + assert isinstance(values.int, int) + assert values.int == 1 + assert isinstance(values.float, float) + assert values.float == 1.1 + assert isinstance(values.bool, bool) + assert values.bool is True + assert isinstance(values.placeholder, protobuf.Values) + assert values.placeholder._isUnknown + assert 'map' in values + assert 'nope' not in values + assert values == values + assert hash(values) == hash(values) + assert values._hasUnknowns + +def test_values_list(): + values = protobuf.List( + protobuf.Map(), # 0 + {}, # 1 + protobuf.List(), # 2 + [], # 3 + protobuf.Unknown(), # 4 + None, # 5 + 'test', # 6 + 1, # 7 + 1.1, # 8 + True, # 9 + ) + assert isinstance(values[0], protobuf.Values) + assert values[0]._isMap + assert isinstance(values[1], protobuf.Values) + assert values[1]._isMap + assert isinstance(values[2], protobuf.Values) + assert values[2]._isList + assert isinstance(values[3], protobuf.Values) + assert values[3]._isList + assert isinstance(values[4], protobuf.Values) + assert values[4]._isUnknown + assert values[5] is None + assert isinstance(values[6], str) + assert values[6] == 'test' + assert isinstance(values[7], int) + assert values[7] == 1 + assert isinstance(values[8], float) + assert values[8] == 1.1 + assert isinstance(values[9], bool) + assert values[9] is True + assert isinstance(values[10], protobuf.Values) + assert values[10]._isUnknown + assert 'test' in values + assert 'nope' not in values + assert protobuf.Unknown() in values + assert values == values + assert hash(values) == hash(values) + assert values._hasUnknowns + +def test_create_child(): + map = protobuf.Unknown() + list = protobuf.Unknown() + map.a.b = 'c' + assert 'b' in map.a + assert map.a.b == 'c' + del map.a.b + assert 'b' not in map.a + assert not map._hasUnknowns + map.a.b = protobuf.Unknown() + assert map._hasUnknowns + list[0][0] = 'c' + assert 'c' in list[0] + assert list[0][0] == 'c' + del list[0][0] + assert 'c' not in list[0] + assert not list._hasUnknowns + list[0][0] = protobuf.Unknown() + assert list._hasUnknowns diff --git a/tests/utils.py b/tests/utils.py index e7b1032..7f5e9e8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,7 +1,24 @@ +import datetime +import yaml from google.protobuf.struct_pb2 import Struct, ListValue +def yaml_load(text): + return _yaml_clean(yaml.safe_load(text)) + +def _yaml_clean(values): + if isinstance(values, dict): + for field, value in values.items(): + values[field] = _yaml_clean(value) + elif isinstance(values, (list, tuple)): + for ix, value in enumerate(values): + values[ix] = _yaml_clean(value) + elif isinstance(values, datetime.datetime): + values = values.isoformat().replace('+00:00', 'Z') + return values + + def message_merge(message, values): for field, value in values.items(): if isinstance(value, (dict, list, tuple)): @@ -19,7 +36,17 @@ def message_merge(message, values): else: message_merge(current, value) else: - list_merge(current, value) + if isinstance(current, ListValue): + list_merge(current, value) + else: + descriptor = message.DESCRIPTOR.fields_by_name.get(field) + if descriptor.label == descriptor.LABEL_REPEATED: + if descriptor.message_type.GetOptions().map_entry: + message_map_merge(descriptor, current, value) + else: + message_list_merge(current, value) + else: + message_merge(current, value) continue setattr(message, field, value) @@ -154,6 +181,11 @@ def message_dict(message): value = message_list_list(field, value) else: value = message_dict(value) + elif field.type in (field.TYPE_DOUBLE, field.TYPE_FLOAT): + if value.is_integer(): + value = int(value) + elif field.type == field.TYPE_BYTES: + value = value.decode() result[field.name] = value return result @@ -166,13 +198,11 @@ def message_map_dict(descriptor, message): value = map_dict(value) elif descriptor.message_type.name == 'ListValue': value = list_list(value) - elif descriptor.label == descriptor.LABEL_REPEATED: - if descriptor.message_type.GetOptions().map_entry: - value = message_map_dict(value) - else: - value = message_list_list(value) else: value = message_dict(value) + elif descriptor.type in (descriptor.TYPE_DOUBLE, descriptor.TYPE_FLOAT): + if value.is_integer(): + value = int(value) elif descriptor.type == descriptor.TYPE_BYTES: value = value.decode() result[field] = value @@ -181,18 +211,16 @@ def message_map_dict(descriptor, message): def message_list_list(descriptor, message): result = [] for value in message: - if descriptor.type == descrptor.TYPE_MESSAGE: + if descriptor.type == descriptor.TYPE_MESSAGE: if descriptor.message_type.name == 'Struct': value = map_dict(value) elif descriptor.message_type.name == 'ListValue': value = list_list(value) - elif descriptor.label == descriptor.LABEL_REPEATED: - if descriptor.message_type.GetOptions().map_entry: - value = message_map_dict(value) - else: - value = message_list_list(value) else: value = message_dict(value) + elif descriptor.type in (descriptor.TYPE_DOUBLE, descriptor.TYPE_FLOAT): + if value.is_integer(): + value = int(value) elif descriptor.type == descriptor.TYPE_BYTES: value = value.decode() result.append(value) @@ -205,6 +233,9 @@ def map_dict(message): value = map_dict(value) elif isinstance(value, ListValue): value = list_list(value) + elif isinstance(value, float): + if value.is_integer(): + value = int(value) result[field] = value return result @@ -215,5 +246,8 @@ def list_list(message): value = map_dict(value) elif isinstance(value, ListValue): value = list_list(value) + elif isinstance(value, float): + if value.is_integer(): + value = int(value) result.append(value) return result