From 3246efc4da7d0536f14015fabf880306f57342b5 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 14 Nov 2024 11:14:13 -0600 Subject: [PATCH 001/122] Moving to workload identity --- .../templates/deployment.yaml | 1 + software/applications/osdu-core/legal.yaml | 34 +++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/charts/osdu-developer-service/templates/deployment.yaml b/charts/osdu-developer-service/templates/deployment.yaml index f27f743f..43b85bd6 100644 --- a/charts/osdu-developer-service/templates/deployment.yaml +++ b/charts/osdu-developer-service/templates/deployment.yaml @@ -42,6 +42,7 @@ spec: {{- end }} spec: {{- if $nodePool }} + serviceAccountName: {{ .serviceAccountName | default "workload-identity-sa" }} nodeSelector: nodepool: {{ $nodePool }} {{- end }} diff --git a/software/applications/osdu-core/legal.yaml b/software/applications/osdu-core/legal.yaml index 54f72934..582a05dd 100644 --- a/software/applications/osdu-core/legal.yaml +++ b/software/applications/osdu-core/legal.yaml @@ -66,22 +66,22 @@ spec: - "/api/legal/v1/api-docs*" - "/api/legal/v1/webjars/*" env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -101,7 +101,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_SERVLET_CONTEXTPATH value: "/api/legal/v1/" - name: SERVER_PORT From 7aca360b1041a52b1f19f5234bb6df41853b796b Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 14 Nov 2024 13:43:19 -0600 Subject: [PATCH 002/122] Updated Chart --- charts/osdu-developer-service/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-service/templates/deployment.yaml b/charts/osdu-developer-service/templates/deployment.yaml index 43b85bd6..2973a912 100644 --- a/charts/osdu-developer-service/templates/deployment.yaml +++ b/charts/osdu-developer-service/templates/deployment.yaml @@ -23,6 +23,7 @@ metadata: labels: {{ $labels }} app: {{ or .service .scaledObject }} appVersion: {{ $osduVersion }} + azure.workload.identity/use: "true" spec: {{- if not .autoscale }} replicas: {{ .replicaCount | default $replicaCount }} @@ -90,7 +91,6 @@ spec: persistentVolumeClaim: claimName: {{ .name }}-pvc {{- end }} - serviceAccountName: workload-identity-sa containers: - name: {{ or .service .scaledObject }} image: {{ if hasSuffix "-" .repository }}{{ .repository }}{{ $osduVersion }}:{{ .tag }}{{ else }}{{ .repository }}:{{ .tag }}{{ end }} From 2f417c42c54c5799290e3eb9b088a6761422fb7f Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 14 Nov 2024 13:55:48 -0600 Subject: [PATCH 003/122] Updated Chart --- charts/osdu-developer-service/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-service/templates/deployment.yaml b/charts/osdu-developer-service/templates/deployment.yaml index 2973a912..7f53e62a 100644 --- a/charts/osdu-developer-service/templates/deployment.yaml +++ b/charts/osdu-developer-service/templates/deployment.yaml @@ -42,8 +42,8 @@ spec: version: {{ $subset }} {{- end }} spec: - {{- if $nodePool }} serviceAccountName: {{ .serviceAccountName | default "workload-identity-sa" }} + {{- if $nodePool }} nodeSelector: nodepool: {{ $nodePool }} {{- end }} From 5aa97d20dc7163d777f36487a626e0c40df0c027 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 14 Nov 2024 14:12:27 -0600 Subject: [PATCH 004/122] Updated Chart --- charts/osdu-developer-service/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-service/templates/deployment.yaml b/charts/osdu-developer-service/templates/deployment.yaml index 7f53e62a..430ff1da 100644 --- a/charts/osdu-developer-service/templates/deployment.yaml +++ b/charts/osdu-developer-service/templates/deployment.yaml @@ -23,7 +23,6 @@ metadata: labels: {{ $labels }} app: {{ or .service .scaledObject }} appVersion: {{ $osduVersion }} - azure.workload.identity/use: "true" spec: {{- if not .autoscale }} replicas: {{ .replicaCount | default $replicaCount }} @@ -38,6 +37,7 @@ spec: {{- end }} labels: {{ $selectorLabels }} app: {{ or .service .scaledObject }} + azure.workload.identity/use: "true" {{- if $subset }} version: {{ $subset }} {{- end }} From f0909deb64f4abbf75161c6bdade1ea366694af3 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 14 Nov 2024 14:30:54 -0600 Subject: [PATCH 005/122] Testing services with workload identity. --- .../applications/osdu-core/entitlements.yaml | 34 +++++++++---------- software/applications/osdu-core/file.yaml | 34 +++++++++---------- software/applications/osdu-core/indexer.yaml | 34 +++++++++---------- .../applications/osdu-core/partition.yaml | 34 +++++++++---------- software/applications/osdu-core/schema.yaml | 34 +++++++++---------- software/applications/osdu-core/search.yaml | 34 +++++++++---------- software/applications/osdu-core/storage.yaml | 34 +++++++++---------- software/applications/osdu-core/workflow.yaml | 34 +++++++++---------- .../osdu-reference/crs-catalog.yaml | 34 +++++++++---------- .../osdu-reference/crs-conversion.yaml | 34 +++++++++---------- .../applications/osdu-reference/unit.yaml | 34 +++++++++---------- 11 files changed, 187 insertions(+), 187 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index 72ae1809..3c482d81 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -66,22 +66,22 @@ spec: - "/api/entitlements/v2/api-docs*" - "/api/entitlements/v2/webjars/*" env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -101,7 +101,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_SERVLET_CONTEXTPATH value: "/api/entitlements/v2/" - name: SERVER_PORT diff --git a/software/applications/osdu-core/file.yaml b/software/applications/osdu-core/file.yaml index 01961dbe..71fea364 100644 --- a/software/applications/osdu-core/file.yaml +++ b/software/applications/osdu-core/file.yaml @@ -70,22 +70,22 @@ spec: - "/api/file/v2/api-docs*" - "/api/file/v2/webjars/*" env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URL secret: name: azure-resources @@ -109,7 +109,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_SERVLET_CONTEXTPATH value: "/api/file/" - name: SERVER_PORT diff --git a/software/applications/osdu-core/indexer.yaml b/software/applications/osdu-core/indexer.yaml index a3c544d7..b93625f9 100644 --- a/software/applications/osdu-core/indexer.yaml +++ b/software/applications/osdu-core/indexer.yaml @@ -67,22 +67,22 @@ spec: - '*/_dps/task-handlers' - '*/reindex' env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -102,7 +102,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_PORT value: "80" - name: ACCEPT_HTTP diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 95e33554..16ed1cab 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -65,22 +65,22 @@ spec: - "/api/partition/v1/webjars/*" - "/api/partition/v1/liveness_check*" env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -100,7 +100,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_SERVLET_CONTEXTPATH value: "/api/partition/v1/" - name: SERVER_PORT diff --git a/software/applications/osdu-core/schema.yaml b/software/applications/osdu-core/schema.yaml index d46993ca..6f4aafa6 100644 --- a/software/applications/osdu-core/schema.yaml +++ b/software/applications/osdu-core/schema.yaml @@ -65,22 +65,22 @@ spec: - "/api/schema-service/v1/api-docs*" - "/api/schema-service/v2/webjars/*" env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -100,7 +100,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_SERVLET_CONTEXTPATH value: "/api/schema-service/v1/" - name: SERVER_PORT diff --git a/software/applications/osdu-core/search.yaml b/software/applications/osdu-core/search.yaml index ffb69963..387513c3 100644 --- a/software/applications/osdu-core/search.yaml +++ b/software/applications/osdu-core/search.yaml @@ -67,22 +67,22 @@ spec: - "/api/search/v2/api-docs*" - "/api/search/v2/webjars/*" env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -102,7 +102,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_PORT value: "80" - name: ACCEPT_HTTP diff --git a/software/applications/osdu-core/storage.yaml b/software/applications/osdu-core/storage.yaml index 7111362b..6e53ffaa 100644 --- a/software/applications/osdu-core/storage.yaml +++ b/software/applications/osdu-core/storage.yaml @@ -72,22 +72,22 @@ spec: - "/api/storage/v2/api-docs*" - "/api/storage/v2/webjars/*" env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -107,7 +107,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_PORT value: "80" - name: ACCEPT_HTTP diff --git a/software/applications/osdu-core/workflow.yaml b/software/applications/osdu-core/workflow.yaml index 5eddb537..e93ba04d 100644 --- a/software/applications/osdu-core/workflow.yaml +++ b/software/applications/osdu-core/workflow.yaml @@ -66,22 +66,22 @@ spec: - "/api/workflow/v3/api-docs*" - "/api/workflow/v3/webjars/*" env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -101,7 +101,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_SERVLET_CONTEXTPATH value: "/api/workflow/" - name: SERVER_PORT diff --git a/software/applications/osdu-reference/crs-catalog.yaml b/software/applications/osdu-reference/crs-catalog.yaml index 95c8b0c4..2db15fd9 100644 --- a/software/applications/osdu-reference/crs-catalog.yaml +++ b/software/applications/osdu-reference/crs-catalog.yaml @@ -72,22 +72,22 @@ spec: path: /mnt/crs_catalogs subPath: crs env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -107,7 +107,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_PORT value: "80" - name: ACCEPT_HTTP diff --git a/software/applications/osdu-reference/crs-conversion.yaml b/software/applications/osdu-reference/crs-conversion.yaml index 5ccb3073..6ee7bff9 100644 --- a/software/applications/osdu-reference/crs-conversion.yaml +++ b/software/applications/osdu-reference/crs-conversion.yaml @@ -72,22 +72,22 @@ spec: path: /mnt/crs_conversion subPath: crs-conversion env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -107,7 +107,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_PORT value: "80" - name: ACCEPT_HTTP diff --git a/software/applications/osdu-reference/unit.yaml b/software/applications/osdu-reference/unit.yaml index 0f6c8819..83ddc082 100644 --- a/software/applications/osdu-reference/unit.yaml +++ b/software/applications/osdu-reference/unit.yaml @@ -72,22 +72,22 @@ spec: path: /mnt/unit_catalogs subPath: unit env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -107,7 +107,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_PORT value: "80" - name: ACCEPT_HTTP From 7d4858aef314c244fdd4852a9fcc7084d26f0303 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 14 Nov 2024 14:51:27 -0600 Subject: [PATCH 006/122] Moved entitlements back to sp --- .../applications/osdu-core/entitlements.yaml | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index 3c482d81..c27288ff 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -66,22 +66,22 @@ spec: - "/api/entitlements/v2/api-docs*" - "/api/entitlements/v2/webjars/*" env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword + - name: AZURE_TENANT_ID + secret: + name: active-directory + key: tenant-id + - name: AZURE_SUBSCRIPTION_ID + secret: + name: active-directory + key: subscription-id + - name: AZURE_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: AZURE_CLIENT_SECRET + secret: + name: active-directory + key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources From 84319859f2173a0580459727f5cb507a89f5df44 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 15 Nov 2024 09:04:59 -0600 Subject: [PATCH 007/122] Revert Storage --- software/applications/osdu-core/storage.yaml | 32 ++++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/software/applications/osdu-core/storage.yaml b/software/applications/osdu-core/storage.yaml index 6e53ffaa..1ba46dc5 100644 --- a/software/applications/osdu-core/storage.yaml +++ b/software/applications/osdu-core/storage.yaml @@ -72,22 +72,22 @@ spec: - "/api/storage/v2/api-docs*" - "/api/storage/v2/webjars/*" env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword + - name: AZURE_TENANT_ID + secret: + name: active-directory + key: tenant-id + - name: AZURE_SUBSCRIPTION_ID + secret: + name: active-directory + key: subscription-id + - name: AZURE_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: AZURE_CLIENT_SECRET + secret: + name: active-directory + key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources From c610aceb1e7f14b180165e28ccce7fc707f27393 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 15 Nov 2024 09:36:25 -0600 Subject: [PATCH 008/122] Revert Storage --- software/applications/osdu-core/storage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/applications/osdu-core/storage.yaml b/software/applications/osdu-core/storage.yaml index 1ba46dc5..7111362b 100644 --- a/software/applications/osdu-core/storage.yaml +++ b/software/applications/osdu-core/storage.yaml @@ -107,7 +107,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" + value: "false" - name: SERVER_PORT value: "80" - name: ACCEPT_HTTP From 1b3051fe8579faacbd66adb4baf802d2a020bdcd Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 09:22:10 -0600 Subject: [PATCH 009/122] backout entitlements --- software/applications/osdu-core/entitlements.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index c27288ff..72ae1809 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -101,7 +101,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" + value: "false" - name: SERVER_SERVLET_CONTEXTPATH value: "/api/entitlements/v2/" - name: SERVER_PORT From f020e842af77984b18b46db1f27c981e0dee61a6 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 09:49:39 -0600 Subject: [PATCH 010/122] migrated storage --- software/applications/osdu-core/file.yaml | 16 --------- software/applications/osdu-core/indexer.yaml | 16 --------- software/applications/osdu-core/legal.yaml | 16 --------- .../applications/osdu-core/partition.yaml | 16 --------- software/applications/osdu-core/schema.yaml | 16 --------- software/applications/osdu-core/search.yaml | 16 --------- software/applications/osdu-core/storage.yaml | 34 +++++++++---------- software/applications/osdu-core/workflow.yaml | 16 --------- .../osdu-reference/crs-catalog.yaml | 16 --------- .../osdu-reference/crs-conversion.yaml | 16 --------- .../applications/osdu-reference/unit.yaml | 16 --------- 11 files changed, 17 insertions(+), 177 deletions(-) diff --git a/software/applications/osdu-core/file.yaml b/software/applications/osdu-core/file.yaml index 71fea364..155d6da1 100644 --- a/software/applications/osdu-core/file.yaml +++ b/software/applications/osdu-core/file.yaml @@ -70,22 +70,6 @@ spec: - "/api/file/v2/api-docs*" - "/api/file/v2/webjars/*" env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword - name: KEYVAULT_URL secret: name: azure-resources diff --git a/software/applications/osdu-core/indexer.yaml b/software/applications/osdu-core/indexer.yaml index b93625f9..f332106a 100644 --- a/software/applications/osdu-core/indexer.yaml +++ b/software/applications/osdu-core/indexer.yaml @@ -67,22 +67,6 @@ spec: - '*/_dps/task-handlers' - '*/reindex' env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources diff --git a/software/applications/osdu-core/legal.yaml b/software/applications/osdu-core/legal.yaml index 582a05dd..b9fc4489 100644 --- a/software/applications/osdu-core/legal.yaml +++ b/software/applications/osdu-core/legal.yaml @@ -66,22 +66,6 @@ spec: - "/api/legal/v1/api-docs*" - "/api/legal/v1/webjars/*" env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 16ed1cab..9fca3cb9 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -65,22 +65,6 @@ spec: - "/api/partition/v1/webjars/*" - "/api/partition/v1/liveness_check*" env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources diff --git a/software/applications/osdu-core/schema.yaml b/software/applications/osdu-core/schema.yaml index 6f4aafa6..3c877ebe 100644 --- a/software/applications/osdu-core/schema.yaml +++ b/software/applications/osdu-core/schema.yaml @@ -65,22 +65,6 @@ spec: - "/api/schema-service/v1/api-docs*" - "/api/schema-service/v2/webjars/*" env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources diff --git a/software/applications/osdu-core/search.yaml b/software/applications/osdu-core/search.yaml index 387513c3..9e97dafa 100644 --- a/software/applications/osdu-core/search.yaml +++ b/software/applications/osdu-core/search.yaml @@ -67,22 +67,6 @@ spec: - "/api/search/v2/api-docs*" - "/api/search/v2/webjars/*" env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources diff --git a/software/applications/osdu-core/storage.yaml b/software/applications/osdu-core/storage.yaml index 7111362b..6e53ffaa 100644 --- a/software/applications/osdu-core/storage.yaml +++ b/software/applications/osdu-core/storage.yaml @@ -72,22 +72,22 @@ spec: - "/api/storage/v2/api-docs*" - "/api/storage/v2/webjars/*" env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -107,7 +107,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_PORT value: "80" - name: ACCEPT_HTTP diff --git a/software/applications/osdu-core/workflow.yaml b/software/applications/osdu-core/workflow.yaml index e93ba04d..aa38b414 100644 --- a/software/applications/osdu-core/workflow.yaml +++ b/software/applications/osdu-core/workflow.yaml @@ -66,22 +66,6 @@ spec: - "/api/workflow/v3/api-docs*" - "/api/workflow/v3/webjars/*" env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources diff --git a/software/applications/osdu-reference/crs-catalog.yaml b/software/applications/osdu-reference/crs-catalog.yaml index 2db15fd9..17c6f809 100644 --- a/software/applications/osdu-reference/crs-catalog.yaml +++ b/software/applications/osdu-reference/crs-catalog.yaml @@ -72,22 +72,6 @@ spec: path: /mnt/crs_catalogs subPath: crs env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources diff --git a/software/applications/osdu-reference/crs-conversion.yaml b/software/applications/osdu-reference/crs-conversion.yaml index 6ee7bff9..3f413cc6 100644 --- a/software/applications/osdu-reference/crs-conversion.yaml +++ b/software/applications/osdu-reference/crs-conversion.yaml @@ -72,22 +72,6 @@ spec: path: /mnt/crs_conversion subPath: crs-conversion env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources diff --git a/software/applications/osdu-reference/unit.yaml b/software/applications/osdu-reference/unit.yaml index 83ddc082..00785fce 100644 --- a/software/applications/osdu-reference/unit.yaml +++ b/software/applications/osdu-reference/unit.yaml @@ -72,22 +72,6 @@ spec: path: /mnt/unit_catalogs subPath: unit env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources From cd53ca11117f2989ca1be19bbc6396b606c5596e Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 10:06:44 -0600 Subject: [PATCH 011/122] regress storage --- software/applications/osdu-core/storage.yaml | 34 ++++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/software/applications/osdu-core/storage.yaml b/software/applications/osdu-core/storage.yaml index 6e53ffaa..7111362b 100644 --- a/software/applications/osdu-core/storage.yaml +++ b/software/applications/osdu-core/storage.yaml @@ -72,22 +72,22 @@ spec: - "/api/storage/v2/api-docs*" - "/api/storage/v2/webjars/*" env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword + - name: AZURE_TENANT_ID + secret: + name: active-directory + key: tenant-id + - name: AZURE_SUBSCRIPTION_ID + secret: + name: active-directory + key: subscription-id + - name: AZURE_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: AZURE_CLIENT_SECRET + secret: + name: active-directory + key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -107,7 +107,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" + value: "false" - name: SERVER_PORT value: "80" - name: ACCEPT_HTTP From 45a736fcd953d181bdb1036ac2c16d36dd5c7d44 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 16:25:27 -0600 Subject: [PATCH 012/122] Entitlements only. --- .../applications/osdu-core/entitlements.yaml | 98 ++-- software/applications/osdu-core/file.yaml | 276 +++++----- software/applications/osdu-core/indexer.yaml | 476 +++++++++--------- software/applications/osdu-core/legal.yaml | 236 ++++----- software/applications/osdu-core/schema.yaml | 334 ++++++------ software/applications/osdu-core/search.yaml | 242 ++++----- software/applications/osdu-core/storage.yaml | 308 ++++++------ .../applications/osdu-core/user-init.yaml | 104 ++-- software/applications/osdu-core/workflow.yaml | 390 +++++++------- tools/rest-scripts/admin.http | 6 +- 10 files changed, 1235 insertions(+), 1235 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index 72ae1809..a9117e77 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -125,52 +125,52 @@ spec: - name: PARTITION_SERVICE_ENDPOINT value: "http://partition/api/partition/v1" --- -# Retrigger: kubectl annotate helmrelease osdu-init-entitlements fluxcd.io/retrigger=$(date +%s) -n osdu-core -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-entitlements - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" - fluxcd.io/retrigger: "initial" -spec: - dependsOn: - - name: osdu-entitlements - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - installationType: osduCore - jobs: - partitionInit: false - entitlementInit: true - userInit: false - elasticInit: false - schemaInit: false - partition: opendes - clientSecret: - name: active-directory - key: principal-clientpassword - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id +# # Retrigger: kubectl annotate helmrelease osdu-init-entitlements fluxcd.io/retrigger=$(date +%s) -n osdu-core +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-entitlements +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# fluxcd.io/retrigger: "initial" +# spec: +# dependsOn: +# - name: osdu-entitlements +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# installationType: osduCore +# jobs: +# partitionInit: false +# entitlementInit: true +# userInit: false +# elasticInit: false +# schemaInit: false +# partition: opendes +# clientSecret: +# name: active-directory +# key: principal-clientpassword +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: clientId +# valuesKey: client_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: tenantId +# valuesKey: tenant_id diff --git a/software/applications/osdu-core/file.yaml b/software/applications/osdu-core/file.yaml index 155d6da1..054edcfd 100644 --- a/software/applications/osdu-core/file.yaml +++ b/software/applications/osdu-core/file.yaml @@ -1,138 +1,138 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-file - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-indexer-queue - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: file - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: file - path: /api/file/ - hosts: - - "*" - cors: - - "http://localhost:8080" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/system/file/file- - branch: release-0-26 - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - request: - cpu: 1000m - memory: 1Gi - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/file/v2/info" - - "/api/file/v2/swagger*" - - "/api/file/v2/api-docs*" - - "/api/file/v2/webjars/*" - env: - - name: KEYVAULT_URL - secret: - name: azure-resources - key: keyvault-uri - - name: AZURE_AD_APP_RESOURCE_ID - secret: - name: active-directory - key: principal-clientid - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" - - name: SERVER_SERVLET_CONTEXTPATH - value: "/api/file/" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: "file" - - name: SPRING_CONFIG_NAME - value: "common,application" - - name: LOG_PREFIX - value: file - - name: LOGGING_LEVEL - value: INFO - - name: APPLICATION_PORT - value: 80 - - name: COSMOSDB_DATABASE - value: osdu-db - - name: OSDU_ENTITLEMENTS_APP_KEY - value: OBSOLETE - - name: PARTITION_SERVICE_ENDPOINT - value: "http://partition/api/partition/v1" - - name: OSDU_ENTITLEMENTS_URL - value: http://entitlements/api/entitlements/v2 - - name: authorizeAPI - value: http://entitlements/api/entitlements/v2 - - name: OSDU_STORAGE_URL - value: http://storage/api/storage/v2 - - name: SEARCH_HOST - value: http://search/api/search/v2 - - name: AZURE_PUBSUB_PUBLISH - value: "true" - - name: SERVICE_BUS_ENABLED_STATUS - value: "true" - - name: SERVICE_BUS_TOPIC_STATUS - value: "statuschangedtopic" - - name: BATCH_SIZE - value: "100" - - name: SEARCH_QUERY_LIMIT - value: "1000" - - name: FILE_CHECKSUM_CALCULATION_LIMIT - value: "5368709120L" \ No newline at end of file +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-file +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-indexer-queue +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: file +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: file +# path: /api/file/ +# hosts: +# - "*" +# cors: +# - "http://localhost:8080" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/system/file/file- +# branch: release-0-26 +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# request: +# cpu: 1000m +# memory: 1Gi +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/file/v2/info" +# - "/api/file/v2/swagger*" +# - "/api/file/v2/api-docs*" +# - "/api/file/v2/webjars/*" +# env: +# - name: KEYVAULT_URL +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AZURE_AD_APP_RESOURCE_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_PODIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_SERVLET_CONTEXTPATH +# value: "/api/file/" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: "file" +# - name: SPRING_CONFIG_NAME +# value: "common,application" +# - name: LOG_PREFIX +# value: file +# - name: LOGGING_LEVEL +# value: INFO +# - name: APPLICATION_PORT +# value: 80 +# - name: COSMOSDB_DATABASE +# value: osdu-db +# - name: OSDU_ENTITLEMENTS_APP_KEY +# value: OBSOLETE +# - name: PARTITION_SERVICE_ENDPOINT +# value: "http://partition/api/partition/v1" +# - name: OSDU_ENTITLEMENTS_URL +# value: http://entitlements/api/entitlements/v2 +# - name: authorizeAPI +# value: http://entitlements/api/entitlements/v2 +# - name: OSDU_STORAGE_URL +# value: http://storage/api/storage/v2 +# - name: SEARCH_HOST +# value: http://search/api/search/v2 +# - name: AZURE_PUBSUB_PUBLISH +# value: "true" +# - name: SERVICE_BUS_ENABLED_STATUS +# value: "true" +# - name: SERVICE_BUS_TOPIC_STATUS +# value: "statuschangedtopic" +# - name: BATCH_SIZE +# value: "100" +# - name: SEARCH_QUERY_LIMIT +# value: "1000" +# - name: FILE_CHECKSUM_CALCULATION_LIMIT +# value: "5368709120L" \ No newline at end of file diff --git a/software/applications/osdu-core/indexer.yaml b/software/applications/osdu-core/indexer.yaml index f332106a..50841b2e 100644 --- a/software/applications/osdu-core/indexer.yaml +++ b/software/applications/osdu-core/indexer.yaml @@ -1,238 +1,238 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-indexer-service - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-legal - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: indexer - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: indexer - path: /api/indexer/v2/ - hosts: - - "*" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/system/indexer-service/indexer-service- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/indexer/v2/info" - - /api/indexer/v2/swagger* - - /api/indexer/v2/api-docs* - - "/api/indexer/v2/webjars/*" - - '*/index-worker' - - '*/_dps/task-handlers' - - '*/reindex' - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SECURITY_HTTPS_CERTIFICATE_TRUST - value: "true" - - name: SPRING_APPLICATION_NAME - value: indexer - - name: SERVER_SERVLET_CONTEXTPATH - value: /api/indexer/v2/ - - name: COSMOSDB_DATABASE - value: osdu-db - - name: REDIS_DATABASE - value: "4" - - name: REDIS_TTL_SECONDS - value: "3600" - - name: SERVICEBUS_TOPIC_NAME - value: indexing-progress - - name: REINDEX_TOPIC_NAME - value: recordstopic - - name: PARTITION_SERVICE_ENDPOINT - value: http://partition/api/partition/v1 - - name: ENTITLEMENTS_SERVICE_ENDPOINT - value: http://entitlements/api/entitlements/v2 - - name: ENTITLEMENTS_SERVICE_API_KEY - value: "OBSOLETE" - - name: SCHEMA_SERVICE_URL - value: http://schema/api/schema-service/v1 - - name: STORAGE_SERVICE_URL - value: http://storage/api/storage/v2 - - name: STORAGE_SCHEMA_HOST - value: http://storage/api/storage/v2/schemas - - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST - value: http://storage/api/storage/v2/query/records:batch - - name: STORAGE_QUERY_RECORD_HOST - value: http://storage/api/storage/v2/query/records - - name: SEARCH_SERVICE_URL - value: http://search/api/search/v2 ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-indexer-queue - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-legal - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: indexer-queue - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: indexer-queue - repository: community.opengroup.org:5555/osdu/platform/system/indexer-queue/indexer-queue- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_APP_RESOURCE_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_APPLICATION_INSIGHTS_INSTRUMENTATION_KEY - secret: - name: azure-resources - key: insights-key - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" - - name: SERVER_PORT - value: "80" - - name: SPRING_APPLICATION_NAME - value: indexer-queue - - name: AZURE_SERVICEBUS_TOPIC_NAME - value: recordstopic - - name: AZURE_REINDEX_TOPIC_NAME - value: reindextopic - - name: AZURE_SERVICEBUS_TOPIC_SUBSCRIPTION - value: recordstopicsubscription - - name: AZURE_REINDEX_TOPIC_SUBSCRIPTION - value: reindextopicsubscription - - name: AZURE_SCHEMACHANGED_TOPIC_NAME - value: schemachangedtopic - - name: AZURE_SCHEMACHANGED_TOPIC_SUBSCRIPTION - value: schemachangedtopiceg - - name: MAX_CONCURRENT_CALLS - value: "32" - - name: MAX_DELIVERY_COUNT - value: "5" - - name: EXECUTOR_N_THREADS - value: "32" - - name: MAX_LOCK_RENEW_DURATION_SECONDS - value: "600" - - name: PARTITION_API - value: http://partition/api/partition/v1 - - name: INDEXER_WORKER_URL - value: http://indexer/api/indexer/v2/_dps/task-handlers/index-worker - - name: schema_worker_url - value: http://indexer-service/api/indexer/v2/_dps/task-handlers/schema-worker \ No newline at end of file +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-indexer-service +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-legal +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: indexer +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: indexer +# path: /api/indexer/v2/ +# hosts: +# - "*" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/system/indexer-service/indexer-service- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/indexer/v2/info" +# - /api/indexer/v2/swagger* +# - /api/indexer/v2/api-docs* +# - "/api/indexer/v2/webjars/*" +# - '*/index-worker' +# - '*/_dps/task-handlers' +# - '*/reindex' +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_PODIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SECURITY_HTTPS_CERTIFICATE_TRUST +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: indexer +# - name: SERVER_SERVLET_CONTEXTPATH +# value: /api/indexer/v2/ +# - name: COSMOSDB_DATABASE +# value: osdu-db +# - name: REDIS_DATABASE +# value: "4" +# - name: REDIS_TTL_SECONDS +# value: "3600" +# - name: SERVICEBUS_TOPIC_NAME +# value: indexing-progress +# - name: REINDEX_TOPIC_NAME +# value: recordstopic +# - name: PARTITION_SERVICE_ENDPOINT +# value: http://partition/api/partition/v1 +# - name: ENTITLEMENTS_SERVICE_ENDPOINT +# value: http://entitlements/api/entitlements/v2 +# - name: ENTITLEMENTS_SERVICE_API_KEY +# value: "OBSOLETE" +# - name: SCHEMA_SERVICE_URL +# value: http://schema/api/schema-service/v1 +# - name: STORAGE_SERVICE_URL +# value: http://storage/api/storage/v2 +# - name: STORAGE_SCHEMA_HOST +# value: http://storage/api/storage/v2/schemas +# - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST +# value: http://storage/api/storage/v2/query/records:batch +# - name: STORAGE_QUERY_RECORD_HOST +# value: http://storage/api/storage/v2/query/records +# - name: SEARCH_SERVICE_URL +# value: http://search/api/search/v2 +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-indexer-queue +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-legal +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: indexer-queue +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: indexer-queue +# repository: community.opengroup.org:5555/osdu/platform/system/indexer-queue/indexer-queue- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# env: +# - name: AZURE_TENANT_ID +# secret: +# name: active-directory +# key: tenant-id +# - name: AZURE_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: AZURE_CLIENT_SECRET +# secret: +# name: active-directory +# key: principal-clientpassword +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: AZURE_APP_RESOURCE_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: AZURE_APPLICATION_INSIGHTS_INSTRUMENTATION_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_PODIDENTITY_ISENABLED +# value: "false" +# - name: SERVER_PORT +# value: "80" +# - name: SPRING_APPLICATION_NAME +# value: indexer-queue +# - name: AZURE_SERVICEBUS_TOPIC_NAME +# value: recordstopic +# - name: AZURE_REINDEX_TOPIC_NAME +# value: reindextopic +# - name: AZURE_SERVICEBUS_TOPIC_SUBSCRIPTION +# value: recordstopicsubscription +# - name: AZURE_REINDEX_TOPIC_SUBSCRIPTION +# value: reindextopicsubscription +# - name: AZURE_SCHEMACHANGED_TOPIC_NAME +# value: schemachangedtopic +# - name: AZURE_SCHEMACHANGED_TOPIC_SUBSCRIPTION +# value: schemachangedtopiceg +# - name: MAX_CONCURRENT_CALLS +# value: "32" +# - name: MAX_DELIVERY_COUNT +# value: "5" +# - name: EXECUTOR_N_THREADS +# value: "32" +# - name: MAX_LOCK_RENEW_DURATION_SECONDS +# value: "600" +# - name: PARTITION_API +# value: http://partition/api/partition/v1 +# - name: INDEXER_WORKER_URL +# value: http://indexer/api/indexer/v2/_dps/task-handlers/index-worker +# - name: schema_worker_url +# value: http://indexer-service/api/indexer/v2/_dps/task-handlers/schema-worker \ No newline at end of file diff --git a/software/applications/osdu-core/legal.yaml b/software/applications/osdu-core/legal.yaml index b9fc4489..10bb8f8e 100644 --- a/software/applications/osdu-core/legal.yaml +++ b/software/applications/osdu-core/legal.yaml @@ -1,118 +1,118 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-legal - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-partition - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: legal - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: legal - path: /api/legal/v1/ - hosts: - - "*" - cors: - - "http://localhost:8080" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/legal/legal- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/legal/v1/info" - - "/api/legal/v1/swagger*" - - "/api/legal/v1/api-docs*" - - "/api/legal/v1/webjars/*" - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" - - name: SERVER_SERVLET_CONTEXTPATH - value: "/api/legal/v1/" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: "legal" - - name: SPRING_CONFIG_NAME - value: "common,application" - - name: LOG_PREFIX - value: "legal" - - name: AZURE_STORAGE_ENABLE_HTTPS - value: "true" - - name: COSMOSDB_DATABASE - value: "osdu-db" - - name: AZURE_STORAGE_CONTAINER_NAME - value: "legal-service-azure-configuration" - - name: LEGAL_SERVICE_REGION - value: "us" - - name: SERVICEBUS_TOPIC_NAME - value: "legaltags" - - name: REDIS_DATABASE - value: "2" - - name: PARTITION_SERVICE_ENDPOINT - value: "http://partition/api/partition/v1" - - name: ENTITLEMENTS_SERVICE_ENDPOINT - value: "http://entitlements/api/entitlements/v2" - - name: ENTITLEMENTS_SERVICE_API_KEY - value: "OBSOLETE" \ No newline at end of file +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-legal +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-partition +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: legal +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: legal +# path: /api/legal/v1/ +# hosts: +# - "*" +# cors: +# - "http://localhost:8080" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/legal/legal- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/legal/v1/info" +# - "/api/legal/v1/swagger*" +# - "/api/legal/v1/api-docs*" +# - "/api/legal/v1/webjars/*" +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_PODIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_SERVLET_CONTEXTPATH +# value: "/api/legal/v1/" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: "legal" +# - name: SPRING_CONFIG_NAME +# value: "common,application" +# - name: LOG_PREFIX +# value: "legal" +# - name: AZURE_STORAGE_ENABLE_HTTPS +# value: "true" +# - name: COSMOSDB_DATABASE +# value: "osdu-db" +# - name: AZURE_STORAGE_CONTAINER_NAME +# value: "legal-service-azure-configuration" +# - name: LEGAL_SERVICE_REGION +# value: "us" +# - name: SERVICEBUS_TOPIC_NAME +# value: "legaltags" +# - name: REDIS_DATABASE +# value: "2" +# - name: PARTITION_SERVICE_ENDPOINT +# value: "http://partition/api/partition/v1" +# - name: ENTITLEMENTS_SERVICE_ENDPOINT +# value: "http://entitlements/api/entitlements/v2" +# - name: ENTITLEMENTS_SERVICE_API_KEY +# value: "OBSOLETE" \ No newline at end of file diff --git a/software/applications/osdu-core/schema.yaml b/software/applications/osdu-core/schema.yaml index 3c877ebe..6a162bbb 100644 --- a/software/applications/osdu-core/schema.yaml +++ b/software/applications/osdu-core/schema.yaml @@ -1,167 +1,167 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-schema - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-indexer-service - namespace: osdu-core - - name: osdu-indexer-queue - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: schema - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: schema - path: /api/schema-service/v1/ - hosts: - - "*" - cors: - - "http://localhost:8080" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/system/schema-service/schema-service- - tag: latest - probe: - path: /actuator/health - port: 8081 - keyvault: true - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/schema-service/v1/info" - - "/api/schema-service/v1/swagger*" - - "/api/schema-service/v1/api-docs*" - - "/api/schema-service/v2/webjars/*" - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" - - name: SERVER_SERVLET_CONTEXTPATH - value: "/api/schema-service/v1/" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: "schema" - - name: COSMOSDB_DATABASE - value: "osdu-db" - - name: LOG_PREFIX - value: "schema" - - name: AZURE_SYSTEM_STORAGECONTAINERNAME - value: "system" - - name: SERVICEBUS_TOPIC_NAME - value: "schemachangedtopic" - - name: EVENT_GRID_ENABLED - value: 'false' - - name: EVENT_GRID_TOPIC - value: "schemachangedtopic" - - name: SERVICE_BUS_ENABLED - value: 'true' - - name: PARTITION_SERVICE_ENDPOINT - value: "http://partition/api/partition/v1" - - name: ENTITLEMENTS_SERVICE_ENDPOINT - value: "http://entitlements/api/entitlements/v2" - - name: ENTITLEMENTS_SERVICE_API_KEY - value: "OBSOLETE" ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-schema - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-schema - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - installationType: osduCore - jobs: - partitionInit: false - entitlementInit: false - userInit: false - schemaInit: true - elasticInit: false - partition: opendes - clientSecret: - name: active-directory - key: principal-clientpassword - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id - - kind: ConfigMap - name: configmap-services - targetPath: serviceBus - valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-schema +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-indexer-service +# namespace: osdu-core +# - name: osdu-indexer-queue +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: schema +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: schema +# path: /api/schema-service/v1/ +# hosts: +# - "*" +# cors: +# - "http://localhost:8080" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/system/schema-service/schema-service- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# keyvault: true +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/schema-service/v1/info" +# - "/api/schema-service/v1/swagger*" +# - "/api/schema-service/v1/api-docs*" +# - "/api/schema-service/v2/webjars/*" +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_PODIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_SERVLET_CONTEXTPATH +# value: "/api/schema-service/v1/" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: "schema" +# - name: COSMOSDB_DATABASE +# value: "osdu-db" +# - name: LOG_PREFIX +# value: "schema" +# - name: AZURE_SYSTEM_STORAGECONTAINERNAME +# value: "system" +# - name: SERVICEBUS_TOPIC_NAME +# value: "schemachangedtopic" +# - name: EVENT_GRID_ENABLED +# value: 'false' +# - name: EVENT_GRID_TOPIC +# value: "schemachangedtopic" +# - name: SERVICE_BUS_ENABLED +# value: 'true' +# - name: PARTITION_SERVICE_ENDPOINT +# value: "http://partition/api/partition/v1" +# - name: ENTITLEMENTS_SERVICE_ENDPOINT +# value: "http://entitlements/api/entitlements/v2" +# - name: ENTITLEMENTS_SERVICE_API_KEY +# value: "OBSOLETE" +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-schema +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-schema +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# installationType: osduCore +# jobs: +# partitionInit: false +# entitlementInit: false +# userInit: false +# schemaInit: true +# elasticInit: false +# partition: opendes +# clientSecret: +# name: active-directory +# key: principal-clientpassword +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: clientId +# valuesKey: client_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: tenantId +# valuesKey: tenant_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: serviceBus +# valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file diff --git a/software/applications/osdu-core/search.yaml b/software/applications/osdu-core/search.yaml index 9e97dafa..c2958375 100644 --- a/software/applications/osdu-core/search.yaml +++ b/software/applications/osdu-core/search.yaml @@ -1,121 +1,121 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-search - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-indexer-queue - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: search - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: search - path: /api/search/v2/ - hosts: - - "*" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/system/search-service/search-service- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - request: - cpu: 1000m - memory: 1Gi - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/search/v2/info" - - "/api/search/v2/swagger*" - - "/api/search/v2/api-docs*" - - "/api/search/v2/webjars/*" - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: search - - name: SERVER_SERVLET_CONTEXTPATH - value: /api/search/v2/ - - name: LOG_PREFIX - value: "search" - - name: SEARCH_SERVICE_SPRING_LOGGING_LEVEL - value: "DEBUG" - - name: COSMOSDB_DATABASE - value: "osdu-db" - - name: REDIS_DATABASE - value: "5" - - name: ENVIRONMENT - value: "evt" - - name: ELASTIC_CACHE_EXPIRATION - value: 1 - - name: MAX_CACHE_VALUE_SIZE - value: 60 - - name: POLICY_SERVICE_ENABLED - value: "false" - - name: ENTITLEMENTS_SERVICE_API_KEY - value: "OBSOLETE" - - name: PARTITION_SERVICE_ENDPOINT - value: "http://partition/api/partition/v1" - - name: ENTITLEMENTS_SERVICE_ENDPOINT - value: "http://entitlements/api/entitlements/v2" - - name: POLICY_SERVICE_ENDPOINT - value: http://policy/api/policy/v1 \ No newline at end of file +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-search +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-indexer-queue +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: search +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: search +# path: /api/search/v2/ +# hosts: +# - "*" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/system/search-service/search-service- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# request: +# cpu: 1000m +# memory: 1Gi +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/search/v2/info" +# - "/api/search/v2/swagger*" +# - "/api/search/v2/api-docs*" +# - "/api/search/v2/webjars/*" +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_PODIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: search +# - name: SERVER_SERVLET_CONTEXTPATH +# value: /api/search/v2/ +# - name: LOG_PREFIX +# value: "search" +# - name: SEARCH_SERVICE_SPRING_LOGGING_LEVEL +# value: "DEBUG" +# - name: COSMOSDB_DATABASE +# value: "osdu-db" +# - name: REDIS_DATABASE +# value: "5" +# - name: ENVIRONMENT +# value: "evt" +# - name: ELASTIC_CACHE_EXPIRATION +# value: 1 +# - name: MAX_CACHE_VALUE_SIZE +# value: 60 +# - name: POLICY_SERVICE_ENABLED +# value: "false" +# - name: ENTITLEMENTS_SERVICE_API_KEY +# value: "OBSOLETE" +# - name: PARTITION_SERVICE_ENDPOINT +# value: "http://partition/api/partition/v1" +# - name: ENTITLEMENTS_SERVICE_ENDPOINT +# value: "http://entitlements/api/entitlements/v2" +# - name: POLICY_SERVICE_ENDPOINT +# value: http://policy/api/policy/v1 \ No newline at end of file diff --git a/software/applications/osdu-core/storage.yaml b/software/applications/osdu-core/storage.yaml index 7111362b..e3eae9ab 100644 --- a/software/applications/osdu-core/storage.yaml +++ b/software/applications/osdu-core/storage.yaml @@ -1,155 +1,155 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-storage - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-indexer-queue - namespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - targetNamespace: osdu-core - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: storage - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: storage - path: /api/storage/v2/ - hosts: - - "*" - cors: - - "http://localhost:8080" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/system/storage/storage- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - request: - cpu: 800m - memory: 1Gi - # limit: - # cpu: 1000m - # memory: 4Gi - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/storage/v2/info" - - "/api/storage/v2/swagger*" - - "/api/storage/v2/api-docs*" - - "/api/storage/v2/webjars/*" - env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: storage - - name: SERVER_SERVLET_CONTEXTPATH - value: /api/storage/v2/ - - name: COSMOSDB_DATABASE - value: osdu-db - - name: AZURE_EVENTGRID_ENABLED - value: "false" - - name: AZURE_SERVICEBUS_ENABLED - value: "true" - - name: SERVICEBUS_TOPIC_NAME - value: recordstopic - - name: SERVICEBUS_V2_TOPIC_NAME - value: recordstopic-v2 - - name: REDIS_DATABASE - value: "4" - - name: PARTITION_SERVICE_ENDPOINT - value: http://partition/api/partition/v1 - - name: ENTITLEMENTS_SERVICE_ENDPOINT - value: http://entitlements/api/entitlements/v2 - - name: ENTITLEMENTS_SERVICE_API_KEY - value: "OBSOLETE" - - name: LEGAL_SERVICE_ENDPOINT - value: http://legal/api/legal/v1 - - name: LEGAL_SERVICE_REGION - value: southcentralus - - name: LEGAL_SERVICEBUS_TOPIC_NAME - value: legaltagschangedtopiceg - - name: LEGAL_SERVICEBUS_TOPIC_SUBSCRIPTION - value: eg_sb_legaltagchangedsubscription - - name: CRS_CONVERSION_SERVICE_ENDPOINT - value: http://crs-conversion/api/crs/converter/v2 - - name: POLICY_SERVICE_ENDPOINT - value: http://policy/api/policy/v1 - - name: OPA_ENABLED - value: "false" - - name: REDIS_HOST_KEY - value: redis-hostname - - name: REDIS_PASSWORD_KEY - value: redis-password +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-storage +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-indexer-queue +# namespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# targetNamespace: osdu-core +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: storage +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: storage +# path: /api/storage/v2/ +# hosts: +# - "*" +# cors: +# - "http://localhost:8080" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/system/storage/storage- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# request: +# cpu: 800m +# memory: 1Gi +# # limit: +# # cpu: 1000m +# # memory: 4Gi +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/storage/v2/info" +# - "/api/storage/v2/swagger*" +# - "/api/storage/v2/api-docs*" +# - "/api/storage/v2/webjars/*" +# env: +# - name: AZURE_TENANT_ID +# secret: +# name: active-directory +# key: tenant-id +# - name: AZURE_SUBSCRIPTION_ID +# secret: +# name: active-directory +# key: subscription-id +# - name: AZURE_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: AZURE_CLIENT_SECRET +# secret: +# name: active-directory +# key: principal-clientpassword +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_PODIDENTITY_ISENABLED +# value: "false" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: storage +# - name: SERVER_SERVLET_CONTEXTPATH +# value: /api/storage/v2/ +# - name: COSMOSDB_DATABASE +# value: osdu-db +# - name: AZURE_EVENTGRID_ENABLED +# value: "false" +# - name: AZURE_SERVICEBUS_ENABLED +# value: "true" +# - name: SERVICEBUS_TOPIC_NAME +# value: recordstopic +# - name: SERVICEBUS_V2_TOPIC_NAME +# value: recordstopic-v2 +# - name: REDIS_DATABASE +# value: "4" +# - name: PARTITION_SERVICE_ENDPOINT +# value: http://partition/api/partition/v1 +# - name: ENTITLEMENTS_SERVICE_ENDPOINT +# value: http://entitlements/api/entitlements/v2 +# - name: ENTITLEMENTS_SERVICE_API_KEY +# value: "OBSOLETE" +# - name: LEGAL_SERVICE_ENDPOINT +# value: http://legal/api/legal/v1 +# - name: LEGAL_SERVICE_REGION +# value: southcentralus +# - name: LEGAL_SERVICEBUS_TOPIC_NAME +# value: legaltagschangedtopiceg +# - name: LEGAL_SERVICEBUS_TOPIC_SUBSCRIPTION +# value: eg_sb_legaltagchangedsubscription +# - name: CRS_CONVERSION_SERVICE_ENDPOINT +# value: http://crs-conversion/api/crs/converter/v2 +# - name: POLICY_SERVICE_ENDPOINT +# value: http://policy/api/policy/v1 +# - name: OPA_ENABLED +# value: "false" +# - name: REDIS_HOST_KEY +# value: redis-hostname +# - name: REDIS_PASSWORD_KEY +# value: redis-password \ No newline at end of file diff --git a/software/applications/osdu-core/user-init.yaml b/software/applications/osdu-core/user-init.yaml index 6c448de4..4b9adfb6 100644 --- a/software/applications/osdu-core/user-init.yaml +++ b/software/applications/osdu-core/user-init.yaml @@ -1,52 +1,52 @@ ---- -# kubectl create job --namespace osdu-core --from=cronjob/user-init user-init-run-$(date +%s) -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-users - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-init-entitlements - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - installationType: osduCore - jobs: - partitionInit: false - entitlementInit: false - userInit: true - elasticInit: false - schemaInit: false - clientSecret: - name: active-directory - key: principal-clientpassword - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id - - kind: ConfigMap - name: configmap-services - targetPath: emailAddress - valuesKey: first_user_id \ No newline at end of file +# --- +# # kubectl create job --namespace osdu-core --from=cronjob/user-init user-init-run-$(date +%s) +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-users +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-init-entitlements +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# installationType: osduCore +# jobs: +# partitionInit: false +# entitlementInit: false +# userInit: true +# elasticInit: false +# schemaInit: false +# clientSecret: +# name: active-directory +# key: principal-clientpassword +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: clientId +# valuesKey: client_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: tenantId +# valuesKey: tenant_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: emailAddress +# valuesKey: first_user_id \ No newline at end of file diff --git a/software/applications/osdu-core/workflow.yaml b/software/applications/osdu-core/workflow.yaml index aa38b414..8e67b2a2 100644 --- a/software/applications/osdu-core/workflow.yaml +++ b/software/applications/osdu-core/workflow.yaml @@ -1,195 +1,195 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-workflow - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-partition - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: workflow - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: workflow - path: /api/workflow/ - hosts: - - "*" - cors: - - "http://localhost:8080" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/data-flow/ingestion/ingestion-workflow/ingestion-workflow- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/workflow/v3/info" - - "/api/workflow/v3/swagger*" - - "/api/workflow/v3/api-docs*" - - "/api/workflow/v3/webjars/*" - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" - - name: SERVER_SERVLET_CONTEXTPATH - value: "/api/workflow/" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: "workflow" - - name: SPRING_CONFIG_NAME - value: "common,application" - - name: LOG_PREFIX - value: "workflow" - - name: AZURE_STORAGE_ENABLE_HTTPS - value: "true" - - name: COSMOSDB_DATABASE - value: "osdu-db" - - name: COSMOSDB_SYSTEM_DATABASE - value: osdu-system-db - - name: AIRFLOW_STORAGE_ACCOUNT_NAME - secret: - name: azure-resources - key: azurestorageaccountname - - name: AIRFLOW_STORAGE_ACCOUNT_KEY - secret: - name: azure-resources - key: azurestorageaccountkey - - name: OSDU_AIRFLOW_USERNAME - secret: - name: azure-resources - key: airflow-username - - name: OSDU_AIRFLOW_PASSWORD - secret: - name: azure-resources - key: airflow-password - - name: AUTHORIZEAPI - value: http://entitlements/api/entitlements/v2 - - name: AUTHORIZEAPIKEY - value: "OBSOLETE" - - name: PARTITION_SERVICE_ENDPOINT - value: "http://partition/api/partition/v1" - - name: OSDU_ENTITLEMENTS_URL - value: "http://entitlements/api/entitlements/v2" - - name: OSDU_AIRFLOW_URL - value: "http://airflow-web.airflow.svc.cluster.local:8080/airflow" - - name: OSDU_ENTITLEMENTS_APPKEY - value: "OBSOLETE" - - name: OSDU_AIRFLOW_VERSION2_ENABLED - value: true - - name: DP_AIRFLOW_FOR_SYSTEM_DAG - value: "false" - - name: IGNORE_DAGCONTENT - value: "true" - - name: IGNORE_CUSTOMOPERATORCONTENT - value: "true" ---- -# Retrigger: kubectl annotate helmrelease osdu-init-workflow fluxcd.io/retrigger=$(date +%s) -n osdu-core -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-workflow - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" - fluxcd.io/retrigger: "initial" -spec: - dependsOn: - - name: osdu-workflow - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - installationType: osduCore - jobs: - workflowInit: true - workflows: - - name: "Osdu_ingest" - description: "Manifest Ingest workflow for OSDU" - - name: "Osdu_ingest_by_reference" - description: "Manifest Ingest by reference workflow for OSDU" - - name: 'csv-parser' - description: 'CSV Parser workflow for OSDU' - partition: opendes - clientSecret: - name: active-directory - key: principal-clientpassword - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-workflow +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-partition +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: workflow +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: workflow +# path: /api/workflow/ +# hosts: +# - "*" +# cors: +# - "http://localhost:8080" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/data-flow/ingestion/ingestion-workflow/ingestion-workflow- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/workflow/v3/info" +# - "/api/workflow/v3/swagger*" +# - "/api/workflow/v3/api-docs*" +# - "/api/workflow/v3/webjars/*" +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_PODIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_SERVLET_CONTEXTPATH +# value: "/api/workflow/" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: "workflow" +# - name: SPRING_CONFIG_NAME +# value: "common,application" +# - name: LOG_PREFIX +# value: "workflow" +# - name: AZURE_STORAGE_ENABLE_HTTPS +# value: "true" +# - name: COSMOSDB_DATABASE +# value: "osdu-db" +# - name: COSMOSDB_SYSTEM_DATABASE +# value: osdu-system-db +# - name: AIRFLOW_STORAGE_ACCOUNT_NAME +# secret: +# name: azure-resources +# key: azurestorageaccountname +# - name: AIRFLOW_STORAGE_ACCOUNT_KEY +# secret: +# name: azure-resources +# key: azurestorageaccountkey +# - name: OSDU_AIRFLOW_USERNAME +# secret: +# name: azure-resources +# key: airflow-username +# - name: OSDU_AIRFLOW_PASSWORD +# secret: +# name: azure-resources +# key: airflow-password +# - name: AUTHORIZEAPI +# value: http://entitlements/api/entitlements/v2 +# - name: AUTHORIZEAPIKEY +# value: "OBSOLETE" +# - name: PARTITION_SERVICE_ENDPOINT +# value: "http://partition/api/partition/v1" +# - name: OSDU_ENTITLEMENTS_URL +# value: "http://entitlements/api/entitlements/v2" +# - name: OSDU_AIRFLOW_URL +# value: "http://airflow-web.airflow.svc.cluster.local:8080/airflow" +# - name: OSDU_ENTITLEMENTS_APPKEY +# value: "OBSOLETE" +# - name: OSDU_AIRFLOW_VERSION2_ENABLED +# value: true +# - name: DP_AIRFLOW_FOR_SYSTEM_DAG +# value: "false" +# - name: IGNORE_DAGCONTENT +# value: "true" +# - name: IGNORE_CUSTOMOPERATORCONTENT +# value: "true" +# --- +# # Retrigger: kubectl annotate helmrelease osdu-init-workflow fluxcd.io/retrigger=$(date +%s) -n osdu-core +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-workflow +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# fluxcd.io/retrigger: "initial" +# spec: +# dependsOn: +# - name: osdu-workflow +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# installationType: osduCore +# jobs: +# workflowInit: true +# workflows: +# - name: "Osdu_ingest" +# description: "Manifest Ingest workflow for OSDU" +# - name: "Osdu_ingest_by_reference" +# description: "Manifest Ingest by reference workflow for OSDU" +# - name: 'csv-parser' +# description: 'CSV Parser workflow for OSDU' +# partition: opendes +# clientSecret: +# name: active-directory +# key: principal-clientpassword +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: clientId +# valuesKey: client_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: tenantId +# valuesKey: tenant_id diff --git a/tools/rest-scripts/admin.http b/tools/rest-scripts/admin.http index 1d6ab49c..34ad39f5 100644 --- a/tools/rest-scripts/admin.http +++ b/tools/rest-scripts/admin.http @@ -48,13 +48,14 @@ grant_type=client_credentials # @name info GET {{ENTITLEMENTS_HOST}}/info Authorization: Bearer {{access_token}} -# x-payload: Bearer {{access_token}} Accept: application/json +# x-payload: Bearer {{access_token}} # --------------------------------- # Group Name # --------------------------------- +### # This is the identifier for the group you are adding. @group_name = app.trusted @@ -72,7 +73,6 @@ Accept: application/json @group_type_email = {{admins_group_type}}@{{DATA_PARTITION}}.{{domain}} - # ----------------------------------------------------------------------------------------------------------------- # These actions are part of deployment processes. # ----------------------------------------------------------------------------------------------------------------- @@ -162,7 +162,7 @@ Content-Type: application/json # This is the identifier for the user you are adding. # When using AAD, this is the email for the user if adding an AAD user. # When using ADD and a Service principal this is the OID of the Service Principal. -@member_email = daniel.scholl@microsoft.com +@member_email = # ----------------------- From 09509c2105db72787b7baab665caf51d667ade26 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 17:39:48 -0600 Subject: [PATCH 013/122] change --- bicep/main.bicep | 9 +- bicep/modules/keyvault_secrets.bicep | 18 +++ .../applications/osdu-core/entitlements.yaml | 34 +++--- .../applications/osdu-core/partition.yaml | 108 +++++++++--------- 4 files changed, 94 insertions(+), 75 deletions(-) diff --git a/bicep/main.bicep b/bicep/main.bicep index dc0b73a0..4c2419c8 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -553,10 +553,10 @@ var vaultSecrets = [ secretName: 'app-dev-sp-password' secretValue: applicationClientSecret == '' ? 'dummy' : applicationClientSecret } - { - secretName: 'app-dev-sp-id' - secretValue: applicationClientId - } + // { + // secretName: 'app-dev-sp-id' + // secretValue: applicationClientSecret == '' ? stampIdentity.outputs.clientId : applicationClientId + // } { secretName: 'cpng-user-name' secretValue: 'dbuser' @@ -675,6 +675,7 @@ module keyvaultSecrets 'modules/keyvault_secrets.bicep' = { workspaceName: logAnalytics.outputs.name insightsName: insights.outputs.name cacheName: redis.outputs.name + identityName: stampIdentity.outputs.name } dependsOn: [ insights diff --git a/bicep/modules/keyvault_secrets.bicep b/bicep/modules/keyvault_secrets.bicep index 099d171c..3c246add 100644 --- a/bicep/modules/keyvault_secrets.bicep +++ b/bicep/modules/keyvault_secrets.bicep @@ -14,6 +14,11 @@ param insightsName string param cacheName string +@description('The name of the identity.') +@minLength(0) +param identityName string + + resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = { name: keyVaultName } @@ -30,6 +35,10 @@ resource redis 'Microsoft.Cache/redis@2022-06-01' existing = { name: cacheName } +resource identity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-07-31-preview' existing = { + name: identityName +} + resource cachePassword 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { name: 'redis-password' parent: keyVault @@ -93,4 +102,13 @@ resource insightsConnection 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { } } +resource identityClientIdSecret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { + name: 'app-dev-sp-id' + parent: keyVault + + properties: { + value: identity.properties.clientId + } +} + output keyVaultName string = keyVault.name diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index a9117e77..5fa79ee4 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -66,22 +66,22 @@ spec: - "/api/entitlements/v2/api-docs*" - "/api/entitlements/v2/webjars/*" env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -101,7 +101,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_SERVLET_CONTEXTPATH value: "/api/entitlements/v2/" - name: SERVER_PORT diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 9fca3cb9..47a8ac5b 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -97,57 +97,57 @@ spec: value: "1" - name: PARTITION_SPRING_LOGGING_LEVEL value: "DEBUG" ---- -# Retrigger: kubectl annotate helmrelease osdu-init-partition fluxcd.io/retrigger=$(date +%s) -n osdu-core -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-partition - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" - fluxcd.io/retrigger: "initial" # Update this to a new value each time -spec: - dependsOn: - - name: osdu-partition - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - installationType: osduCore - jobs: - partitionInit: true - entitlementInit: false - userInit: false - elasticInit: false - schemaInit: false - partition: opendes - clientSecret: - name: active-directory - key: principal-clientpassword - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id - - kind: ConfigMap - name: configmap-services - targetPath: serviceBus - valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file +# --- +# # Retrigger: kubectl annotate helmrelease osdu-init-partition fluxcd.io/retrigger=$(date +%s) -n osdu-core +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-partition +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# fluxcd.io/retrigger: "initial" # Update this to a new value each time +# spec: +# dependsOn: +# - name: osdu-partition +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# installationType: osduCore +# jobs: +# partitionInit: true +# entitlementInit: false +# userInit: false +# elasticInit: false +# schemaInit: false +# partition: opendes +# clientSecret: +# name: active-directory +# key: principal-clientpassword +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: clientId +# valuesKey: client_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: tenantId +# valuesKey: tenant_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: serviceBus +# valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file From 615d04132ed9ba5a542aa80c5d63aa363c7adcf7 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 17:47:55 -0600 Subject: [PATCH 014/122] change --- .../applications/osdu-core/entitlements.yaml | 254 +++++++++--------- .../applications/osdu-core/partition.yaml | 198 +++++++------- 2 files changed, 226 insertions(+), 226 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index 5fa79ee4..63e475f4 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -1,130 +1,130 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-entitlements - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-partition - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: entitlements - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: entitlements - path: /api/entitlements/v2/ - hosts: - - "*" - cors: - - "http://localhost:8080" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/entitlements/entitlements- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/entitlements/v2/info" - - "/api/entitlements/v2/swagger*" - - "/api/entitlements/v2/api-docs*" - - "/api/entitlements/v2/webjars/*" - env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" - - name: SERVER_SERVLET_CONTEXTPATH - value: "/api/entitlements/v2/" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: "entitlements" - - name: SPRING_CONFIG_NAME - value: "common,application" - - name: LOG_PREFIX - value: "entitlements" - - name: LOGGING_LEVEL - value: "DEBUG" - - name: SERVICE_DOMAIN_NAME - value: "dataservices.energy" - - name: ROOT_DATA_GROUP_QUOTA - value: "5000" - - name: REDIS_TTL_SECONDS - value: "1" - - name: PARTITION_SERVICE_ENDPOINT - value: "http://partition/api/partition/v1" ---- +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-entitlements +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-partition +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: entitlements +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: entitlements +# path: /api/entitlements/v2/ +# hosts: +# - "*" +# cors: +# - "http://localhost:8080" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/entitlements/entitlements- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/entitlements/v2/info" +# - "/api/entitlements/v2/swagger*" +# - "/api/entitlements/v2/api-docs*" +# - "/api/entitlements/v2/webjars/*" +# env: +# # - name: AZURE_TENANT_ID +# # secret: +# # name: active-directory +# # key: tenant-id +# # - name: AZURE_SUBSCRIPTION_ID +# # secret: +# # name: active-directory +# # key: subscription-id +# # - name: AZURE_CLIENT_ID +# # secret: +# # name: active-directory +# # key: principal-clientid +# # - name: AZURE_CLIENT_SECRET +# # secret: +# # name: active-directory +# # key: principal-clientpassword +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_PODIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_SERVLET_CONTEXTPATH +# value: "/api/entitlements/v2/" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: "entitlements" +# - name: SPRING_CONFIG_NAME +# value: "common,application" +# - name: LOG_PREFIX +# value: "entitlements" +# - name: LOGGING_LEVEL +# value: "DEBUG" +# - name: SERVICE_DOMAIN_NAME +# value: "dataservices.energy" +# - name: ROOT_DATA_GROUP_QUOTA +# value: "5000" +# - name: REDIS_TTL_SECONDS +# value: "1" +# - name: PARTITION_SERVICE_ENDPOINT +# value: "http://partition/api/partition/v1" +# --- # # Retrigger: kubectl annotate helmrelease osdu-init-entitlements fluxcd.io/retrigger=$(date +%s) -n osdu-core # apiVersion: helm.toolkit.fluxcd.io/v2beta1 # kind: HelmRelease diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 47a8ac5b..6a3ab5f3 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -1,102 +1,102 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-partition - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-developer-base-core - namespace: default - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: partition - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: partition - path: /api/partition/v1/ - hosts: - - "*" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/system/partition/partition- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/partition/v1/info" - - "/api/partition/v1/swagger*" - - "/api/partition/v1/api-docs*" - - "/api/partition/v1/webjars/*" - - "/api/partition/v1/liveness_check*" - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" - - name: SERVER_SERVLET_CONTEXTPATH - value: "/api/partition/v1/" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: "partition" - - name: REDIS_DATABASE - value: "1" - - name: PARTITION_SPRING_LOGGING_LEVEL - value: "DEBUG" +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-partition +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-developer-base-core +# namespace: default +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: partition +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: partition +# path: /api/partition/v1/ +# hosts: +# - "*" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/system/partition/partition- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/partition/v1/info" +# - "/api/partition/v1/swagger*" +# - "/api/partition/v1/api-docs*" +# - "/api/partition/v1/webjars/*" +# - "/api/partition/v1/liveness_check*" +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_PODIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_SERVLET_CONTEXTPATH +# value: "/api/partition/v1/" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: "partition" +# - name: REDIS_DATABASE +# value: "1" +# - name: PARTITION_SPRING_LOGGING_LEVEL +# value: "DEBUG" # --- # # Retrigger: kubectl annotate helmrelease osdu-init-partition fluxcd.io/retrigger=$(date +%s) -n osdu-core # apiVersion: helm.toolkit.fluxcd.io/v2beta1 From 5f2a4d4ca0b584ea65eaa25f941c145f4b0985c3 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 18:34:32 -0600 Subject: [PATCH 015/122] change --- .../templates/partition-init.yaml | 2 + .../applications/osdu-core/partition.yaml | 306 +++++++++--------- 2 files changed, 155 insertions(+), 153 deletions(-) diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index aef03b35..637e0ffb 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -211,6 +211,8 @@ data: echo "==================================================================" echo " Identity Client Id: ${AZURE_CLIENT_ID}" + sleep 300000 + OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url https://login.microsoftonline.com/${AZURE_TENANT_ID}/oauth2/token \ --header "content-type: application/x-www-form-urlencoded" \ diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 6a3ab5f3..9fca3cb9 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -1,153 +1,153 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-partition -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-developer-base-core -# namespace: default -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: partition -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: partition -# path: /api/partition/v1/ -# hosts: -# - "*" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/system/partition/partition- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/partition/v1/info" -# - "/api/partition/v1/swagger*" -# - "/api/partition/v1/api-docs*" -# - "/api/partition/v1/webjars/*" -# - "/api/partition/v1/liveness_check*" -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_PODIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_SERVLET_CONTEXTPATH -# value: "/api/partition/v1/" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: "partition" -# - name: REDIS_DATABASE -# value: "1" -# - name: PARTITION_SPRING_LOGGING_LEVEL -# value: "DEBUG" -# --- -# # Retrigger: kubectl annotate helmrelease osdu-init-partition fluxcd.io/retrigger=$(date +%s) -n osdu-core -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-partition -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# fluxcd.io/retrigger: "initial" # Update this to a new value each time -# spec: -# dependsOn: -# - name: osdu-partition -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# installationType: osduCore -# jobs: -# partitionInit: true -# entitlementInit: false -# userInit: false -# elasticInit: false -# schemaInit: false -# partition: opendes -# clientSecret: -# name: active-directory -# key: principal-clientpassword -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: clientId -# valuesKey: client_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: tenantId -# valuesKey: tenant_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: serviceBus -# valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-partition + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-developer-base-core + namespace: default + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: partition + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: partition + path: /api/partition/v1/ + hosts: + - "*" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/system/partition/partition- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/partition/v1/info" + - "/api/partition/v1/swagger*" + - "/api/partition/v1/api-docs*" + - "/api/partition/v1/webjars/*" + - "/api/partition/v1/liveness_check*" + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_PODIDENTITY_ISENABLED + value: "true" + - name: SERVER_SERVLET_CONTEXTPATH + value: "/api/partition/v1/" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: "partition" + - name: REDIS_DATABASE + value: "1" + - name: PARTITION_SPRING_LOGGING_LEVEL + value: "DEBUG" +--- +# Retrigger: kubectl annotate helmrelease osdu-init-partition fluxcd.io/retrigger=$(date +%s) -n osdu-core +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-partition + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" + fluxcd.io/retrigger: "initial" # Update this to a new value each time +spec: + dependsOn: + - name: osdu-partition + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + installationType: osduCore + jobs: + partitionInit: true + entitlementInit: false + userInit: false + elasticInit: false + schemaInit: false + partition: opendes + clientSecret: + name: active-directory + key: principal-clientpassword + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: clientId + valuesKey: client_id + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id + - kind: ConfigMap + name: configmap-services + targetPath: serviceBus + valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file From 0fd556f4f40c037efd569fa76753f64b4d51def1 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 18:43:49 -0600 Subject: [PATCH 016/122] change --- charts/osdu-developer-init/templates/partition-init.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index 637e0ffb..16c6056f 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -18,7 +18,7 @@ spec: defaultMode: 0500 initContainers: - name: data-seed - image: alpine + image: mcr.microsoft.com/azure-cli:cbl-mariner2.0 command: - script/init.sh volumeMounts: @@ -204,7 +204,7 @@ data: set -euo pipefail set -o nounset - apk add --no-cache curl jq + tdnf install -y curl jq echo "==================================================================" echo " Creating Bearer Token for Application: ${AZURE_CLIENT_ID} " From f9a46c653445e0e083d729dc8b49838dfe37b850 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 18:58:37 -0600 Subject: [PATCH 017/122] Changed --- charts/osdu-developer-init/templates/partition-init.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index 16c6056f..e08efd9a 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -11,6 +11,7 @@ spec: ttlSecondsAfterFinished: 120 template: spec: + serviceAccountName: workload-identity-sa volumes: - name: script configMap: From 2f491661e2e6ad60571f2d295fd207d58387f349 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 19:12:04 -0600 Subject: [PATCH 018/122] Changed --- .../templates/partition-init.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index e08efd9a..fb19ce92 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -34,13 +34,13 @@ spec: value: {{ .Values.serviceBus | quote }} - name: AZURE_TENANT_ID value: {{ .Values.tenantId | quote }} - - name: AZURE_CLIENT_ID - value: {{ .Values.clientId | quote }} - - name: AZURE_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.clientSecret.name | quote }} - key: {{ .Values.clientSecret.key | quote }} + # - name: AZURE_CLIENT_ID + # value: {{ .Values.clientId | quote }} + # - name: AZURE_CLIENT_SECRET + # valueFrom: + # secretKeyRef: + # name: {{ .Values.clientSecret.name | quote }} + # key: {{ .Values.clientSecret.key | quote }} containers: - name: sleep image: istio/base From 051edc7df160b621d4f044844163bbfe2de9d6ea Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 19:28:39 -0600 Subject: [PATCH 019/122] Changed --- .../templates/partition-init.yaml | 4 +++- .../applications/osdu-core/partition.yaml | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index fb19ce92..11321c6c 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -207,12 +207,14 @@ data: tdnf install -y curl jq + sleep 300000 + echo "==================================================================" echo " Creating Bearer Token for Application: ${AZURE_CLIENT_ID} " echo "==================================================================" echo " Identity Client Id: ${AZURE_CLIENT_ID}" - sleep 300000 + OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url https://login.microsoftonline.com/${AZURE_TENANT_ID}/oauth2/token \ diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 9fca3cb9..6f059470 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -132,17 +132,22 @@ spec: elasticInit: false schemaInit: false partition: opendes - clientSecret: - name: active-directory - key: principal-clientpassword + - name: AZURE_CLIENT_ID + secret: + name: active-directory + key: msi-clientid + # clientSecret: + # name: active-directory + # key: principal-clientpassword valuesFrom: - kind: ConfigMap name: configmap-software valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id + + # - kind: ConfigMap + # name: configmap-services + # targetPath: clientId + # valuesKey: client_id - kind: ConfigMap name: configmap-services targetPath: tenantId From ad07fc4504662c44d9021c7992976c24f106c811 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 19:40:28 -0600 Subject: [PATCH 020/122] Changed --- .../templates/partition-init.yaml | 4 ++-- software/applications/osdu-core/partition.yaml | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index 11321c6c..da4b0e12 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -34,8 +34,8 @@ spec: value: {{ .Values.serviceBus | quote }} - name: AZURE_TENANT_ID value: {{ .Values.tenantId | quote }} - # - name: AZURE_CLIENT_ID - # value: {{ .Values.clientId | quote }} + - name: AZURE_CLIENT_ID + value: {{ .Values.clientId | quote }} # - name: AZURE_CLIENT_SECRET # valueFrom: # secretKeyRef: diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 6f059470..24935777 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -132,10 +132,10 @@ spec: elasticInit: false schemaInit: false partition: opendes - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: msi-clientid + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: msi-clientid # clientSecret: # name: active-directory # key: principal-clientpassword @@ -144,10 +144,10 @@ spec: name: configmap-software valuesKey: value.yaml - # - kind: ConfigMap - # name: configmap-services - # targetPath: clientId - # valuesKey: client_id + - kind: Secret + name: active-directory + targetPath: clientId + valuesKey: msi-clientid - kind: ConfigMap name: configmap-services targetPath: tenantId From d9f45bd918c92e1ad33cab9de91f42a31d2152eb Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 19:50:20 -0600 Subject: [PATCH 021/122] Changed --- .../osdu-developer-init/templates/partition-init.yaml | 10 +++++----- software/applications/osdu-core/partition.yaml | 7 +------ 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index da4b0e12..1bdb0fcd 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -209,12 +209,12 @@ data: sleep 300000 - echo "==================================================================" - echo " Creating Bearer Token for Application: ${AZURE_CLIENT_ID} " - echo "==================================================================" - echo " Identity Client Id: ${AZURE_CLIENT_ID}" + # echo "==================================================================" + # echo " Creating Bearer Token for Application: ${AZURE_CLIENT_ID} " + # echo "==================================================================" + # echo " Identity Client Id: ${AZURE_CLIENT_ID}" - + sleep 300000 OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url https://login.microsoftonline.com/${AZURE_TENANT_ID}/oauth2/token \ diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 24935777..4bd212d6 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -132,18 +132,13 @@ spec: elasticInit: false schemaInit: false partition: opendes - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: msi-clientid # clientSecret: # name: active-directory # key: principal-clientpassword valuesFrom: - kind: ConfigMap name: configmap-software - valuesKey: value.yaml - + valuesKey: value.yaml - kind: Secret name: active-directory targetPath: clientId From cd1a4c2fb5900969488ee701b7cc68c36814a8e8 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 19:59:00 -0600 Subject: [PATCH 022/122] Changed --- .../osdu-developer-init/templates/partition-init.yaml | 10 +++++----- software/applications/osdu-core/partition.yaml | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index 1bdb0fcd..8d5cfdbd 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -36,11 +36,11 @@ spec: value: {{ .Values.tenantId | quote }} - name: AZURE_CLIENT_ID value: {{ .Values.clientId | quote }} - # - name: AZURE_CLIENT_SECRET - # valueFrom: - # secretKeyRef: - # name: {{ .Values.clientSecret.name | quote }} - # key: {{ .Values.clientSecret.key | quote }} + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.clientSecret.name | quote }} + key: {{ .Values.clientSecret.key | quote }} containers: - name: sleep image: istio/base diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 4bd212d6..3fd8bcbf 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -132,9 +132,9 @@ spec: elasticInit: false schemaInit: false partition: opendes - # clientSecret: - # name: active-directory - # key: principal-clientpassword + clientSecret: + name: active-directory + key: principal-clientpassword valuesFrom: - kind: ConfigMap name: configmap-software From 0eef16287852da567c41927634b457f16eb8997b Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 19:59:34 -0600 Subject: [PATCH 023/122] Changed --- charts/osdu-developer-init/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-init/Chart.yaml b/charts/osdu-developer-init/Chart.yaml index af8bcbbf..2f97a4f5 100644 --- a/charts/osdu-developer-init/Chart.yaml +++ b/charts/osdu-developer-init/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-init type: application description: Installs the OSDU developer Initialization resources -version: 0.0.1 +version: 0.0.2 appVersion: 0.0.1 maintainers: - name: danielscholl From 9afc880ae5a2b897c4026d8c79e5e1ed46b3a977 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 20:17:34 -0600 Subject: [PATCH 024/122] Changed --- charts/osdu-developer-init/README.md | 4 +- .../templates/partition-init.yaml | 6 +- .../applications/osdu-core/partition.yaml | 106 +++++++++--------- 3 files changed, 56 insertions(+), 60 deletions(-) diff --git a/charts/osdu-developer-init/README.md b/charts/osdu-developer-init/README.md index c5dd7e4f..b27e6c1f 100644 --- a/charts/osdu-developer-init/README.md +++ b/charts/osdu-developer-init/README.md @@ -24,7 +24,7 @@ Install the helm chart. ```bash # Create Namespace NAMESPACE=osdu-core -helm template osdu-core -f custom_values.yaml . +helm template osdu-core-osdu-init-partition -f custom_values.yaml . -helm upgrade --install osdu-core . -n $NAMESPACE -f custom_values.yaml +helm upgrade --install osdu-core-osdu-init-partition . -n $NAMESPACE -f custom_values.yaml ``` diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index 8d5cfdbd..2d590aad 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -36,11 +36,6 @@ spec: value: {{ .Values.tenantId | quote }} - name: AZURE_CLIENT_ID value: {{ .Values.clientId | quote }} - - name: AZURE_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.clientSecret.name | quote }} - key: {{ .Values.clientSecret.key | quote }} containers: - name: sleep image: istio/base @@ -207,6 +202,7 @@ data: tdnf install -y curl jq + sleep 300000 # echo "==================================================================" diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 3fd8bcbf..6c74dc81 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -98,56 +98,56 @@ spec: - name: PARTITION_SPRING_LOGGING_LEVEL value: "DEBUG" --- -# Retrigger: kubectl annotate helmrelease osdu-init-partition fluxcd.io/retrigger=$(date +%s) -n osdu-core -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-partition - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" - fluxcd.io/retrigger: "initial" # Update this to a new value each time -spec: - dependsOn: - - name: osdu-partition - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - installationType: osduCore - jobs: - partitionInit: true - entitlementInit: false - userInit: false - elasticInit: false - schemaInit: false - partition: opendes - clientSecret: - name: active-directory - key: principal-clientpassword - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: Secret - name: active-directory - targetPath: clientId - valuesKey: msi-clientid - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id - - kind: ConfigMap - name: configmap-services - targetPath: serviceBus - valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file +# # Retrigger: kubectl annotate helmrelease osdu-init-partition fluxcd.io/retrigger=$(date +%s) -n osdu-core +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-partition +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# fluxcd.io/retrigger: "initial" # Update this to a new value each time +# spec: +# dependsOn: +# - name: osdu-partition +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# installationType: osduCore +# jobs: +# partitionInit: true +# entitlementInit: false +# userInit: false +# elasticInit: false +# schemaInit: false +# partition: opendes +# clientSecret: +# name: active-directory +# key: principal-clientpassword +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: Secret +# name: active-directory +# targetPath: clientId +# valuesKey: msi-clientid +# - kind: ConfigMap +# name: configmap-services +# targetPath: tenantId +# valuesKey: tenant_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: serviceBus +# valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file From 37272858a35cbd9ec457baabaa7b8395bf6c2a49 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 21:39:42 -0600 Subject: [PATCH 025/122] Updated Chart --- .../templates/partition-init.yaml | 40 +++---- .../applications/osdu-core/partition.yaml | 106 +++++++++--------- 2 files changed, 67 insertions(+), 79 deletions(-) diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index 2d590aad..7f7c3776 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -10,6 +10,9 @@ metadata: spec: ttlSecondsAfterFinished: 120 template: + metadata: + labels: + azure.workload.identity/use: "true" spec: serviceAccountName: workload-identity-sa volumes: @@ -34,7 +37,7 @@ spec: value: {{ .Values.serviceBus | quote }} - name: AZURE_TENANT_ID value: {{ .Values.tenantId | quote }} - - name: AZURE_CLIENT_ID + - name: AZURE_AD_APPLICATION_ID value: {{ .Values.clientId | quote }} containers: - name: sleep @@ -202,33 +205,18 @@ data: tdnf install -y curl jq + echo "==================================================================" + echo " Logging in using Workload Identity" + echo "==================================================================" - sleep 300000 + # Login using the federated token from the environment variable + az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ + --service-principal \ + -u ${AZURE_CLIENT_ID} \ + -t ${AZURE_TENANT_ID} - # echo "==================================================================" - # echo " Creating Bearer Token for Application: ${AZURE_CLIENT_ID} " - # echo "==================================================================" - # echo " Identity Client Id: ${AZURE_CLIENT_ID}" - - sleep 300000 - - OUTPUT=$(curl -s -w "%{http_code}" --request POST \ - --url https://login.microsoftonline.com/${AZURE_TENANT_ID}/oauth2/token \ - --header "content-type: application/x-www-form-urlencoded" \ - --data "grant_type=client_credentials" \ - --data "client_id=${AZURE_CLIENT_ID}" \ - --data "client_secret=${AZURE_CLIENT_SECRET}" \ - --data "resource=${AZURE_CLIENT_ID}") - - HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$') - BODY=${OUTPUT%???} - - if [[ "$HTTP_STATUS_CODE" != "200" ]]; then - echo "Error: Unexpected HTTP status code $HTTP_STATUS_CODE" - exit 1 - fi - - TOKEN=$(echo "$BODY" | jq .access_token | tr -d '"') + # Get token (no resource needed) + TOKEN=$(az account get-access-token --resource ${AZURE_AD_APPLICATION_ID} --query accessToken -o tsv) OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://partition.{{ $namespace }}/api/partition/v1/partitions/${PARTITION} \ diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 6c74dc81..bacd3f83 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -98,56 +98,56 @@ spec: - name: PARTITION_SPRING_LOGGING_LEVEL value: "DEBUG" --- -# # Retrigger: kubectl annotate helmrelease osdu-init-partition fluxcd.io/retrigger=$(date +%s) -n osdu-core -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-partition -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# fluxcd.io/retrigger: "initial" # Update this to a new value each time -# spec: -# dependsOn: -# - name: osdu-partition -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# installationType: osduCore -# jobs: -# partitionInit: true -# entitlementInit: false -# userInit: false -# elasticInit: false -# schemaInit: false -# partition: opendes -# clientSecret: -# name: active-directory -# key: principal-clientpassword -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: Secret -# name: active-directory -# targetPath: clientId -# valuesKey: msi-clientid -# - kind: ConfigMap -# name: configmap-services -# targetPath: tenantId -# valuesKey: tenant_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: serviceBus -# valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file +# Retrigger: kubectl annotate helmrelease osdu-init-partition fluxcd.io/retrigger=$(date +%s) -n osdu-core +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-partition + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" + fluxcd.io/retrigger: "initial" # Update this to a new value each time +spec: + dependsOn: + - name: osdu-partition + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + installationType: osduCore + jobs: + partitionInit: true + entitlementInit: false + userInit: false + elasticInit: false + schemaInit: false + partition: opendes + # clientSecret: + # name: active-directory + # key: principal-clientpassword + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: Secret + name: active-directory + targetPath: clientId + valuesKey: msi-clientid + # - kind: ConfigMap + # name: configmap-services + # targetPath: tenantId + # valuesKey: tenant_id + - kind: ConfigMap + name: configmap-services + targetPath: serviceBus + valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file From 89306638f6d8db961692b8429a2e9732b366c3e4 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 21:51:51 -0600 Subject: [PATCH 026/122] Updated Chart --- charts/osdu-developer-init/Chart.yaml | 2 +- software/applications/osdu-core/partition.yaml | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/charts/osdu-developer-init/Chart.yaml b/charts/osdu-developer-init/Chart.yaml index 2f97a4f5..af8bcbbf 100644 --- a/charts/osdu-developer-init/Chart.yaml +++ b/charts/osdu-developer-init/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-init type: application description: Installs the OSDU developer Initialization resources -version: 0.0.2 +version: 0.0.1 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index bacd3f83..10cedaf1 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -132,9 +132,6 @@ spec: elasticInit: false schemaInit: false partition: opendes - # clientSecret: - # name: active-directory - # key: principal-clientpassword valuesFrom: - kind: ConfigMap name: configmap-software @@ -143,10 +140,6 @@ spec: name: active-directory targetPath: clientId valuesKey: msi-clientid - # - kind: ConfigMap - # name: configmap-services - # targetPath: tenantId - # valuesKey: tenant_id - kind: ConfigMap name: configmap-services targetPath: serviceBus From 2a637a8de5216f727cc6153b7ccd23377764a598 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 18 Nov 2024 22:04:07 -0600 Subject: [PATCH 027/122] Updated Chart --- software/applications/osdu-core/partition.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 10cedaf1..99819efe 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -140,6 +140,10 @@ spec: name: active-directory targetPath: clientId valuesKey: msi-clientid + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id - kind: ConfigMap name: configmap-services targetPath: serviceBus From 2874189de8d32c7ea8ef2a2859e65227685f9471 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Tue, 19 Nov 2024 08:01:17 -0600 Subject: [PATCH 028/122] Add Entitlements --- .../applications/osdu-core/entitlements.yaml | 352 +++++++++--------- 1 file changed, 176 insertions(+), 176 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index 63e475f4..3c482d81 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -1,176 +1,176 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-entitlements -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-partition -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: entitlements -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: entitlements -# path: /api/entitlements/v2/ -# hosts: -# - "*" -# cors: -# - "http://localhost:8080" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/entitlements/entitlements- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/entitlements/v2/info" -# - "/api/entitlements/v2/swagger*" -# - "/api/entitlements/v2/api-docs*" -# - "/api/entitlements/v2/webjars/*" -# env: -# # - name: AZURE_TENANT_ID -# # secret: -# # name: active-directory -# # key: tenant-id -# # - name: AZURE_SUBSCRIPTION_ID -# # secret: -# # name: active-directory -# # key: subscription-id -# # - name: AZURE_CLIENT_ID -# # secret: -# # name: active-directory -# # key: principal-clientid -# # - name: AZURE_CLIENT_SECRET -# # secret: -# # name: active-directory -# # key: principal-clientpassword -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_PODIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_SERVLET_CONTEXTPATH -# value: "/api/entitlements/v2/" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: "entitlements" -# - name: SPRING_CONFIG_NAME -# value: "common,application" -# - name: LOG_PREFIX -# value: "entitlements" -# - name: LOGGING_LEVEL -# value: "DEBUG" -# - name: SERVICE_DOMAIN_NAME -# value: "dataservices.energy" -# - name: ROOT_DATA_GROUP_QUOTA -# value: "5000" -# - name: REDIS_TTL_SECONDS -# value: "1" -# - name: PARTITION_SERVICE_ENDPOINT -# value: "http://partition/api/partition/v1" -# --- -# # Retrigger: kubectl annotate helmrelease osdu-init-entitlements fluxcd.io/retrigger=$(date +%s) -n osdu-core -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-entitlements -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# fluxcd.io/retrigger: "initial" -# spec: -# dependsOn: -# - name: osdu-entitlements -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# installationType: osduCore -# jobs: -# partitionInit: false -# entitlementInit: true -# userInit: false -# elasticInit: false -# schemaInit: false -# partition: opendes -# clientSecret: -# name: active-directory -# key: principal-clientpassword -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: clientId -# valuesKey: client_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: tenantId -# valuesKey: tenant_id +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-entitlements + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-partition + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: entitlements + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: entitlements + path: /api/entitlements/v2/ + hosts: + - "*" + cors: + - "http://localhost:8080" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/entitlements/entitlements- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/entitlements/v2/info" + - "/api/entitlements/v2/swagger*" + - "/api/entitlements/v2/api-docs*" + - "/api/entitlements/v2/webjars/*" + env: + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_PODIDENTITY_ISENABLED + value: "true" + - name: SERVER_SERVLET_CONTEXTPATH + value: "/api/entitlements/v2/" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: "entitlements" + - name: SPRING_CONFIG_NAME + value: "common,application" + - name: LOG_PREFIX + value: "entitlements" + - name: LOGGING_LEVEL + value: "DEBUG" + - name: SERVICE_DOMAIN_NAME + value: "dataservices.energy" + - name: ROOT_DATA_GROUP_QUOTA + value: "5000" + - name: REDIS_TTL_SECONDS + value: "1" + - name: PARTITION_SERVICE_ENDPOINT + value: "http://partition/api/partition/v1" +--- +# Retrigger: kubectl annotate helmrelease osdu-init-entitlements fluxcd.io/retrigger=$(date +%s) -n osdu-core +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-entitlements + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" + fluxcd.io/retrigger: "initial" +spec: + dependsOn: + - name: osdu-entitlements + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + installationType: osduCore + jobs: + partitionInit: false + entitlementInit: true + userInit: false + elasticInit: false + schemaInit: false + partition: opendes + clientSecret: + name: active-directory + key: principal-clientpassword + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: clientId + valuesKey: client_id + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id From 8207cc29dd861644d6aa28daa00a87e6e7352b99 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Tue, 19 Nov 2024 08:05:43 -0600 Subject: [PATCH 029/122] Add Entitlements --- charts/osdu-developer-init/templates/entitlement-init.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/osdu-developer-init/templates/entitlement-init.yaml b/charts/osdu-developer-init/templates/entitlement-init.yaml index 73392af2..1b3cad80 100644 --- a/charts/osdu-developer-init/templates/entitlement-init.yaml +++ b/charts/osdu-developer-init/templates/entitlement-init.yaml @@ -58,6 +58,7 @@ data: set -euo pipefail set -o nounset + exit 0 apk add --no-cache curl jq echo "==================================================================" From ebde890193209a58ff0c34bc4adb56d71d5a7430 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Tue, 19 Nov 2024 08:36:44 -0600 Subject: [PATCH 030/122] Add Entitlements --- .../applications/osdu-core/entitlements.yaml | 100 +++++++++--------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index 3c482d81..b4a1725f 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -124,53 +124,53 @@ spec: value: "1" - name: PARTITION_SERVICE_ENDPOINT value: "http://partition/api/partition/v1" ---- -# Retrigger: kubectl annotate helmrelease osdu-init-entitlements fluxcd.io/retrigger=$(date +%s) -n osdu-core -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-entitlements - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" - fluxcd.io/retrigger: "initial" -spec: - dependsOn: - - name: osdu-entitlements - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - installationType: osduCore - jobs: - partitionInit: false - entitlementInit: true - userInit: false - elasticInit: false - schemaInit: false - partition: opendes - clientSecret: - name: active-directory - key: principal-clientpassword - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id +# --- +# # Retrigger: kubectl annotate helmrelease osdu-init-entitlements fluxcd.io/retrigger=$(date +%s) -n osdu-core +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-entitlements +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# fluxcd.io/retrigger: "initial" +# spec: +# dependsOn: +# - name: osdu-entitlements +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# installationType: osduCore +# jobs: +# partitionInit: false +# entitlementInit: true +# userInit: false +# elasticInit: false +# schemaInit: false +# partition: opendes +# clientSecret: +# name: active-directory +# key: principal-clientpassword +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: clientId +# valuesKey: client_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: tenantId +# valuesKey: tenant_id From 828ec14a6b81022992b9e5982c4fd3ddb37193d5 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Tue, 19 Nov 2024 11:25:52 -0600 Subject: [PATCH 031/122] Updated the Envoy Filter. --- .../templates/envoy-filter.yaml | 129 ++++++++++++++++-- .../templates/entitlement-init.yaml | 46 +++---- .../templates/partition-init.yaml | 8 +- 3 files changed, 140 insertions(+), 43 deletions(-) diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index 281cb0bd..0c237802 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -1,8 +1,11 @@ -# This command increases logging --> istioctl proxy-config log --level lua:debug +{{- $namespace := .Release.Namespace }} +{{- $managedIdentityClientId := .Values.azure.clientId }} + apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: name: microsoft-identity-filter + namespace: {{ $namespace }} spec: configPatches: - applyTo: HTTP_FILTER @@ -22,12 +25,12 @@ spec: "@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua" inlineCode: | function envoy_on_request(request_handle) - -- Remove the Well Known Headers + -- Remove headers first request_handle:headers():remove("x-user-id") request_handle:headers():remove("x-app-id") request_handle:logInfo("x-user-id and x-app-id headers removed") - -- Retrieve the JWT Payload + -- Get JWT metadata local meta = request_handle:streamInfo():dynamicMetadata():get("envoy.filters.http.jwt_authn") if meta and meta["payload"] then @@ -44,18 +47,26 @@ spec: -- Check issuer if string.find(payload["iss"], "sts.windows.net") then - -- Set Well Known Header with an order of preference: upn, unique_name, appid - if payload["upn"] then + -- Handle workload identity scenario first + if payload["appid"] == "{{ $managedIdentityClientId }}" then -- Your managed identity client ID + local on_behalf_of = request_handle:headers():get("x-on-behalf-of") + if on_behalf_of then + request_handle:headers():add("x-user-id", on_behalf_of) + request_handle:logWarn("x-user-id set from on-behalf-of header") + else + request_handle:headers():add("x-user-id", payload["appid"]) + request_handle:logWarn("x-user-id set from appid (workload identity)") + end + -- Regular AAD v1 token handling + elseif payload["upn"] then request_handle:headers():add("x-user-id", payload["upn"]) - request_handle:logWarn("x-user-id set from 'upn' claim: " .. payload["upn"]) + request_handle:logWarn("x-user-id set from 'upn' claim") elseif payload["unique_name"] then request_handle:headers():add("x-user-id", payload["unique_name"]) - request_handle:logWarn("x-user-id set from 'unique_name' claim: " .. payload["unique_name"]) + request_handle:logWarn("x-user-id set from 'unique_name' claim") elseif payload["appid"] then request_handle:headers():add("x-user-id", payload["appid"]) - request_handle:logWarn("x-user-id set from 'appid' claim: " .. payload["appid"]) - else - request_handle:logError("No valid user ID claim (upn, unique_name, appid) found for sts.windows.net") + request_handle:logWarn("x-user-id set from 'appid' claim") end elseif string.find(payload["iss"], "login.microsoftonline.com") then @@ -92,4 +103,100 @@ spec: end end return table.concat(lines, "\n") - end \ No newline at end of file + end + +# # This command increases logging --> istioctl proxy-config log --level lua:debug +# apiVersion: networking.istio.io/v1alpha3 +# kind: EnvoyFilter +# metadata: +# name: microsoft-identity-filter +# spec: +# configPatches: +# - applyTo: HTTP_FILTER +# match: +# context: SIDECAR_INBOUND +# listener: +# filterChain: +# filter: +# name: envoy.filters.network.http_connection_manager +# subFilter: +# name: envoy.filters.http.router +# patch: +# operation: INSERT_BEFORE +# value: +# name: envoy.lua.remove-user-appid-header +# typed_config: +# "@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua" +# inlineCode: | +# function envoy_on_request(request_handle) +# -- Remove the Well Known Headers +# request_handle:headers():remove("x-user-id") +# request_handle:headers():remove("x-app-id") +# request_handle:logInfo("x-user-id and x-app-id headers removed") + +# -- Retrieve the JWT Payload +# local meta = request_handle:streamInfo():dynamicMetadata():get("envoy.filters.http.jwt_authn") + +# if meta and meta["payload"] then +# local payload = meta["payload"] +# request_handle:logDebug("JWT Payload: " .. tableToString(payload)) + +# -- Set the x-app-id Well Known Header if 'aud' claim is present +# if payload["aud"] then +# request_handle:headers():add("x-app-id", payload["aud"]) +# request_handle:logWarn("x-app-id set from 'aud' claim: " .. payload["aud"]) +# else +# request_handle:logError("JWT Payload does not contain 'aud' claim; cannot set x-app-id") +# end + +# -- Check issuer +# if string.find(payload["iss"], "sts.windows.net") then +# -- Set Well Known Header with an order of preference: upn, unique_name, appid +# if payload["upn"] then +# request_handle:headers():add("x-user-id", payload["upn"]) +# request_handle:logWarn("x-user-id set from 'upn' claim: " .. payload["upn"]) +# elseif payload["unique_name"] then +# request_handle:headers():add("x-user-id", payload["unique_name"]) +# request_handle:logWarn("x-user-id set from 'unique_name' claim: " .. payload["unique_name"]) +# elseif payload["appid"] then +# request_handle:headers():add("x-user-id", payload["appid"]) +# request_handle:logWarn("x-user-id set from 'appid' claim: " .. payload["appid"]) +# else +# request_handle:logError("No valid user ID claim (upn, unique_name, appid) found for sts.windows.net") +# end + +# elseif string.find(payload["iss"], "login.microsoftonline.com") then +# -- Set Well Known Header with an order of preference: azp, oid +# if payload["azp"] then +# request_handle:headers():add("x-user-id", payload["azp"]) +# request_handle:logWarn("x-user-id set from 'azp' claim: " .. payload["azp"]) +# elseif payload["oid"] then +# request_handle:headers():add("x-user-id", payload["oid"]) +# request_handle:logWarn("x-user-id set from 'oid' claim: " .. payload["oid"]) +# else +# request_handle:logError("No valid user ID claim (azp, oid) found for login.microsoftonline.com") +# end +# else +# request_handle:logError("Issuer does not match known issuers") +# end +# else +# request_handle:logError("No JWT metadata found or payload is malformed") +# end +# end + +# -- Helper function to convert a table to a string for logging +# function tableToString(tbl, indent) +# if not indent then indent = 0 end +# if type(tbl) ~= 'table' then return tostring(tbl) end +# local lines = {} +# for k, v in pairs(tbl) do +# local formatting = string.rep(" ", indent) .. k .. ": " +# if type(v) == "table" then +# table.insert(lines, formatting) +# table.insert(lines, tableToString(v, indent+1)) +# else +# table.insert(lines, formatting .. tostring(v)) +# end +# end +# return table.concat(lines, "\n") +# end \ No newline at end of file diff --git a/charts/osdu-developer-init/templates/entitlement-init.yaml b/charts/osdu-developer-init/templates/entitlement-init.yaml index 1b3cad80..c03c7e19 100644 --- a/charts/osdu-developer-init/templates/entitlement-init.yaml +++ b/charts/osdu-developer-init/templates/entitlement-init.yaml @@ -10,7 +10,11 @@ metadata: spec: ttlSecondsAfterFinished: 120 template: + metadata: + labels: + azure.workload.identity/use: "true" spec: + serviceAccountName: workload-identity-sa volumes: - name: script configMap: @@ -18,7 +22,7 @@ spec: defaultMode: 0500 initContainers: - name: data-seed - image: alpine + image: mcr.microsoft.com/azure-cli:cbl-mariner2.0 command: - script/init.sh volumeMounts: @@ -27,17 +31,12 @@ spec: env: - name: NAMESPACE value: {{ $namespace }} - - name: PARTITION - value: {{ .Values.partition | quote }} - name: AZURE_TENANT_ID value: {{ .Values.tenantId | quote }} - - name: AZURE_CLIENT_ID + - name: AZURE_AD_APPLICATION_ID value: {{ .Values.clientId | quote }} - - name: AZURE_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.clientSecret.name | quote }} - key: {{ .Values.clientSecret.key | quote }} + - name: PARTITION + value: {{ .Values.partition | quote }} containers: - name: sleep image: istio/base @@ -58,31 +57,22 @@ data: set -euo pipefail set -o nounset - exit 0 - apk add --no-cache curl jq + tdnf install -y curl jq echo "==================================================================" - echo " Creating Bearer Token for Application: ${AZURE_CLIENT_ID} " + echo " Logging in using Workload Identity" echo "==================================================================" - echo " Identity Client Id: ${AZURE_CLIENT_ID}" - - OUTPUT=$(curl -s -w "%{http_code}" --request POST \ - --url https://login.microsoftonline.com/${AZURE_TENANT_ID}/oauth2/token \ - --header "content-type: application/x-www-form-urlencoded" \ - --data "grant_type=client_credentials" \ - --data "client_id=${AZURE_CLIENT_ID}" \ - --data "client_secret=${AZURE_CLIENT_SECRET}" \ - --data "resource=${AZURE_CLIENT_ID}") - HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$') - BODY=${OUTPUT%???} + # Login using the federated token from the environment variable + az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ + --service-principal \ + -u ${AZURE_CLIENT_ID} \ + -t ${AZURE_TENANT_ID} - if [[ "$HTTP_STATUS_CODE" != "200" ]]; then - echo "Error: Unexpected HTTP status code $HTTP_STATUS_CODE" - exit 1 - fi + # Get token (no resource needed) + TOKEN=$(az account get-access-token --resource ${AZURE_AD_APPLICATION_ID} --query accessToken -o tsv) - TOKEN=$(echo "$BODY" | jq .access_token | tr -d '"') + sleep 300000 OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://entitlements.{{ $namespace }}/api/entitlements/v2/tenant-provisioning \ diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index 7f7c3776..1419d0cd 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -31,14 +31,14 @@ spec: env: - name: NAMESPACE value: {{ $namespace }} - - name: PARTITION - value: {{ .Values.partition | quote }} - - name: SERVICE_BUS_NAME - value: {{ .Values.serviceBus | quote }} - name: AZURE_TENANT_ID value: {{ .Values.tenantId | quote }} - name: AZURE_AD_APPLICATION_ID value: {{ .Values.clientId | quote }} + - name: PARTITION + value: {{ .Values.partition | quote }} + - name: SERVICE_BUS_NAME + value: {{ .Values.serviceBus | quote }} containers: - name: sleep image: istio/base From f285d3c866fa6ec161674ee2e06d5a6aedd06e48 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 21 Nov 2024 08:52:40 -0600 Subject: [PATCH 032/122] Updated the Envoy Filter. --- .../templates/envoy-filter.yaml | 120 ++++++++++++++---- 1 file changed, 93 insertions(+), 27 deletions(-) diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index 0c237802..82e48bc0 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -24,6 +24,39 @@ spec: typed_config: "@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua" inlineCode: | + --[[ Token Scenarios Handled: + 1. AAD v1 User Token (sts.windows.net): + - With OID: x-user-id = oid + - Without OID: x-user-id = upn/unique_name + + 2. AAD v1 Service-to-Service Delegation: + - Service identity (appid) matches managedIdentityClientId + - Delegation flow: x-user-id = x-on-behalf-of (delegated user) + - Fallback: x-user-id = appid (direct service call) + + 3. AAD v1 Application (non-delegated): + - x-user-id = appid + + 4. AAD v2 User Token (login.microsoftonline.com): + - Has 'scp' claim: x-user-id = oid + + 5. AAD v2 Service-to-Service Delegation: + - Service identity (azp) matches managedIdentityClientId + - Delegation flow: x-user-id = x-on-behalf-of (delegated user) + - Fallback: x-user-id = azp (direct service call) + + 6. AAD v2 Application (non-delegated): + - x-user-id = azp or oid + + Note: All scenarios set x-app-id from 'aud' claim when present + + OAuth Delegation (On-Behalf-Of) Flow: + - When a service calls another service on behalf of a user + - The original user's identity is passed via x-on-behalf-of header + - Maintains user context through the service chain + - Enables proper authorization based on user identity + ]] + function envoy_on_request(request_handle) -- Remove headers first request_handle:headers():remove("x-user-id") @@ -37,48 +70,81 @@ spec: local payload = meta["payload"] request_handle:logDebug("JWT Payload: " .. tableToString(payload)) - -- Set the x-app-id Well Known Header if 'aud' claim is present + -- Always set x-app-id from aud claim if present if payload["aud"] then request_handle:headers():add("x-app-id", payload["aud"]) request_handle:logWarn("x-app-id set from 'aud' claim: " .. payload["aud"]) - else - request_handle:logError("JWT Payload does not contain 'aud' claim; cannot set x-app-id") end -- Check issuer if string.find(payload["iss"], "sts.windows.net") then - -- Handle workload identity scenario first - if payload["appid"] == "{{ $managedIdentityClientId }}" then -- Your managed identity client ID - local on_behalf_of = request_handle:headers():get("x-on-behalf-of") - if on_behalf_of then - request_handle:headers():add("x-user-id", on_behalf_of) - request_handle:logWarn("x-user-id set from on-behalf-of header") + -- AAD v1 token handling + if payload["upn"] then + -- Scenario 1: AAD v1 User Token + if payload["oid"] then + request_handle:headers():add("x-user-id", payload["oid"]) + request_handle:logWarn("x-user-id set from 'oid' claim with upn present") else - request_handle:headers():add("x-user-id", payload["appid"]) - request_handle:logWarn("x-user-id set from appid (workload identity)") + request_handle:headers():add("x-user-id", payload["upn"]) + request_handle:logWarn("x-user-id set from 'upn' claim") end - -- Regular AAD v1 token handling - elseif payload["upn"] then - request_handle:headers():add("x-user-id", payload["upn"]) - request_handle:logWarn("x-user-id set from 'upn' claim") elseif payload["unique_name"] then - request_handle:headers():add("x-user-id", payload["unique_name"]) - request_handle:logWarn("x-user-id set from 'unique_name' claim") + -- Scenario 1: AAD v1 User Token (alternate claim) + if payload["oid"] then + request_handle:headers():add("x-user-id", payload["oid"]) + request_handle:logWarn("x-user-id set from 'oid' claim with unique_name present") + else + request_handle:headers():add("x-user-id", payload["unique_name"]) + request_handle:logWarn("x-user-id set from 'unique_name' claim") + end elseif payload["appid"] then - request_handle:headers():add("x-user-id", payload["appid"]) - request_handle:logWarn("x-user-id set from 'appid' claim") + -- Check for service identity + if payload["appid"] == "{{ $managedIdentityClientId }}" then + -- Scenario 2: AAD v1 Service-to-Service Delegation + local on_behalf_of = request_handle:headers():get("x-on-behalf-of") + if on_behalf_of and on_behalf_of ~= "" then + request_handle:headers():add("x-user-id", on_behalf_of) + request_handle:logWarn("x-user-id set from on-behalf-of header (delegation flow)") + else + request_handle:headers():add("x-user-id", payload["appid"]) + request_handle:logWarn("x-user-id set from appid (direct service call)") + end + else + -- Scenario 3: AAD v1 Application Token + request_handle:headers():add("x-user-id", payload["appid"]) + request_handle:logWarn("x-user-id set from 'appid' claim (non-delegated)") + end end elseif string.find(payload["iss"], "login.microsoftonline.com") then - -- Set Well Known Header with an order of preference: azp, oid - if payload["azp"] then - request_handle:headers():add("x-user-id", payload["azp"]) - request_handle:logWarn("x-user-id set from 'azp' claim: " .. payload["azp"]) - elseif payload["oid"] then - request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logWarn("x-user-id set from 'oid' claim: " .. payload["oid"]) + -- AAD v2 token handling + if payload["scp"] then + -- Scenario 4: AAD v2 User Token + if payload["oid"] then + request_handle:headers():add("x-user-id", payload["oid"]) + request_handle:logWarn("x-user-id set from 'oid' claim with scp present") + end else - request_handle:logError("No valid user ID claim (azp, oid) found for login.microsoftonline.com") + if payload["azp"] then + if payload["azp"] == "{{ $managedIdentityClientId }}" then + -- Scenario 5: AAD v2 Service-to-Service Delegation + local on_behalf_of = request_handle:headers():get("x-on-behalf-of") + if on_behalf_of and on_behalf_of ~= "" then + request_handle:headers():add("x-user-id", on_behalf_of) + request_handle:logWarn("x-user-id set from on-behalf-of header (v2 delegation flow)") + else + request_handle:headers():add("x-user-id", payload["azp"]) + request_handle:logWarn("x-user-id set from azp (direct service call)") + end + else + -- Scenario 6: AAD v2 Application Token + request_handle:headers():add("x-user-id", payload["azp"]) + request_handle:logWarn("x-user-id set from 'azp' claim (non-delegated)") + end + elseif payload["oid"] then + request_handle:headers():add("x-user-id", payload["oid"]) + request_handle:logWarn("x-user-id set from 'oid' claim") + end end else request_handle:logError("Issuer does not match known issuers") From ebc5e86626c6429fec835769dcc55dcf463edf6c Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 25 Nov 2024 07:41:04 -0600 Subject: [PATCH 033/122] Moved to internal registry. --- .../templates/envoy-filter.yaml | 2 + scripts/envrc_template | 1 + .../applications/osdu-core/entitlements.yaml | 3 +- .../applications/osdu-core/partition.yaml | 3 +- src/Dockerfile-java | 76 +++++++++++++++++++ 5 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 src/Dockerfile-java diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index 82e48bc0..efe8289d 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -1,3 +1,5 @@ +# istioctl proxy-config log --level lua:debug + {{- $namespace := .Release.Namespace }} {{- $managedIdentityClientId := .Values.azure.clientId }} diff --git a/scripts/envrc_template b/scripts/envrc_template index 0775971c..d3caf8c5 100644 --- a/scripts/envrc_template +++ b/scripts/envrc_template @@ -1,5 +1,6 @@ # Common Name Pattern export COMMON_NAME="%COMMON_NAME%" +export REGISTRY="${COMMON_NAME}.azurecr.io/" # Run export AZURE_TENANT_ID="%AZURE_TENANT_ID%" diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index b4a1725f..f3466c68 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -45,7 +45,8 @@ spec: gateways: - istio-system/internal-gateway - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/entitlements/entitlements- + repository: mainrctu55xpre6lo.azurecr.io/entitlements + # repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/entitlements/entitlements- tag: latest probe: path: /actuator/health diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 99819efe..8e279aad 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -43,7 +43,8 @@ spec: gateways: - istio-system/internal-gateway - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/system/partition/partition- + repository: mainrctu55xpre6lo.azurecr.io/partition + # repository: community.opengroup.org:5555/osdu/platform/system/partition/partition- tag: latest probe: path: /actuator/health diff --git a/src/Dockerfile-java b/src/Dockerfile-java new file mode 100644 index 00000000..6bb36b03 --- /dev/null +++ b/src/Dockerfile-java @@ -0,0 +1,76 @@ +# Define default values for ARGs +ARG IFX_AUDIT_PACKAGE=libifxaudit-1.0-1525.x86_64 +ARG AZUL_JDK_PACKAGE="zulu17.48.15-ca-jdk17.0.10-linux_x64" +ARG BASE_IMAGE="mcr.microsoft.com/openjdk/jdk:17-mariner" +ARG EXTRA_JAVA_OPTS="" + +FROM ${BASE_IMAGE} as builder +ARG EXTRA_FILES +ARG EXTRA_FILES_DEST_DIR="extra-files" +ARG INCLUDE_MODULES_OPT="" +ARG SKIP_TESTS=false +ARG SERVICE_PATH + +WORKDIR /app + +# Install required packages +RUN tdnf update -y && \ + tdnf install -y maven && \ + rm -rf /var/cache/tdnf/* + +# Copy local source code instead of git clone +COPY ${SERVICE_PATH} src/ + +# Maven Build Service +RUN mvn -f src/pom.xml validate ${INCLUDE_MODULES_OPT} --settings src/.mvn/community-maven.settings.xml && \ + if [ "$SKIP_TESTS" = "true" ]; then \ + mvn -f src/pom.xml clean install ${INCLUDE_MODULES_OPT} --settings src/.mvn/community-maven.settings.xml -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DskipTests; \ + else \ + mvn -f src/pom.xml clean install ${INCLUDE_MODULES_OPT} --settings src/.mvn/community-maven.settings.xml -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn; \ + fi + +RUN find src -type f \( -name '*aks*' -o -name '*Enqueue*' -o -name '*azure*' \) -a -name '*-spring-boot.jar' -exec cp {} app.jar \; +# RUN find src -type f -name 'git.sha' -exec cp {} git.sha \; + +# Copy extra files only if EXTRA_FILES is set, otherwise it would copy src folder with other java artifacts +RUN mkdir -p $EXTRA_FILES_DEST_DIR + +RUN if [ -n "$EXTRA_FILES" ]; then \ + FILE_NAME=$(basename "$EXTRA_FILES") && \ + RELATIVE_PATH=$(dirname "$EXTRA_FILES") && \ + mkdir -p "$EXTRA_FILES_DEST_DIR/$RELATIVE_PATH" && \ + cp -r "src/$EXTRA_FILES" "$EXTRA_FILES_DEST_DIR/$RELATIVE_PATH"; \ + fi + +FROM --platform=linux/amd64 mcr.microsoft.com/cbl-mariner/base/core:2.0 +ARG AZUL_JDK_PACKAGE +ARG EXTRA_JAVA_OPTS +ARG JAR_FILE +ARG IFX_AUDIT_PACKAGE +ARG EXTRA_FILES +ARG EXTRA_FILES_DEST_DIR="extra-files" + +# Install required packages +RUN tdnf update -y && \ + tdnf install -y curl tar ca-certificates && \ + rm -rf /var/cache/tdnf/* + +RUN curl -LO https://cdn.azul.com/zulu/bin/${AZUL_JDK_PACKAGE}.tar.gz \ + && mkdir -p /usr/lib/jvm \ + && tar -xf ./${AZUL_JDK_PACKAGE}.tar.gz -C /usr/lib/jvm \ + && rm -f ${AZUL_JDK_PACKAGE}.tar.gz + +RUN curl -LO https://packages.microsoft.com/centos/7/prod/Packages/l/${IFX_AUDIT_PACKAGE}.rpm \ + && tdnf install -y ${IFX_AUDIT_PACKAGE}.rpm \ + && rm -rf ${IFX_AUDIT_PACKAGE}.rpm + +COPY --from=builder /app/app.jar app.jar +COPY --from=builder /app/${EXTRA_FILES_DEST_DIR}/${EXTRA_FILES} ${EXTRA_FILES} +# COPY --from=builder /app/git.sha git.sha + +ENV PATH="/usr/lib/jvm/${AZUL_JDK_PACKAGE}/bin:$PATH" +ENV JAVA_HOME="/usr/lib/jvm/${AZUL_JDK_PACKAGE}" +ENV DEFAULT_JVM_OPTS="-XX:+UseG1GC -XX:InitialRAMPercentage=25.0 -XX:MaxRAMPercentage=50.0 -XX:+HeapDumpOnOutOfMemoryError" +ENV LOGGING_JVM_OPTS="-DAPPINSIGHTS_LOGGING_ENABLED=false -Dlog4j2.formatMsgNoLookups=true -Djna.library.path=/usr/lib -DAZURE_AUDIT_ENABLED=true" +ENV JAVA_OPTS="${DEFAULT_JVM_OPTS} ${LOGGING_JVM_OPTS} ${EXTRA_JAVA_OPTS} ${JAVA_OPTS}" +ENTRYPOINT java ${JAVA_OPTS} -jar /app.jar \ No newline at end of file From 27ead9d71cf51222534b6347dd772b9d58cb46b8 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 25 Nov 2024 09:28:30 -0600 Subject: [PATCH 034/122] added ACR pull to AKS --- bicep/main.bicep | 5 +++++ bicep/modules/blade_cluster.bicep | 3 +++ 2 files changed, 8 insertions(+) diff --git a/bicep/main.bicep b/bicep/main.bicep index 4c2419c8..eb85ea30 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -518,6 +518,11 @@ module registry 'br/public:avm/res/container-registry/registry:0.1.1' = { principalType: 'ServicePrincipal' roleDefinitionIdOrName: 'AcrPull' } + { + principalId: clusterBlade.outputs.kubeletIdentityId + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'AcrPull' + } ] } } diff --git a/bicep/modules/blade_cluster.bicep b/bicep/modules/blade_cluster.bicep index df41ae9e..caeec3e8 100644 --- a/bicep/modules/blade_cluster.bicep +++ b/bicep/modules/blade_cluster.bicep @@ -394,6 +394,9 @@ output natClusterIP string = natClusterIP.outputs.ipAddress @description('The OIDC Issuer URL for the cluster.') output oidcIssuerUrl string = cluster.outputs.oidcIssuerUrl +@description('The Object ID of the Kubelet Identity.') +output kubeletIdentityId string = cluster.outputs.kubeletIdentityObjectId + // =============== // // Definitions // // =============== // From b63368d72aa05dce3ad09087c6e6b1d954a6b0e7 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 25 Nov 2024 11:53:21 -0600 Subject: [PATCH 035/122] added ACR pull to AKS --- software/applications/osdu-core/partition.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 8e279aad..f5e02b7e 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -43,7 +43,7 @@ spec: gateways: - istio-system/internal-gateway - istio-system/external-gateway - repository: mainrctu55xpre6lo.azurecr.io/partition + repository: maindwqgbkzaiijzc.azurecr.io/partition # repository: community.opengroup.org:5555/osdu/platform/system/partition/partition- tag: latest probe: From 271bc831bae1e1ccf8b38f42440459c7beea714e Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 27 Nov 2024 14:00:45 -0600 Subject: [PATCH 036/122] Enabled local ACR pulling. --- .../applications/osdu-core/namespace.yaml | 2 +- software/applications/osdu-core/override.yaml | 11 ++ .../applications/osdu-core/partition.yaml | 8 +- src/Dockerfile-java | 8 +- src/docker-bake.hcl | 166 ++++++++++++++++++ 5 files changed, 188 insertions(+), 7 deletions(-) create mode 100644 software/applications/osdu-core/override.yaml create mode 100644 src/docker-bake.hcl diff --git a/software/applications/osdu-core/namespace.yaml b/software/applications/osdu-core/namespace.yaml index 814ee83c..42b4e963 100644 --- a/software/applications/osdu-core/namespace.yaml +++ b/software/applications/osdu-core/namespace.yaml @@ -5,4 +5,4 @@ metadata: name: osdu-core labels: toolkit.fluxcd.io/tenant: dev-team - istio-injection: enabled + istio-injection: enabled \ No newline at end of file diff --git a/software/applications/osdu-core/override.yaml b/software/applications/osdu-core/override.yaml new file mode 100644 index 00000000..a365c02d --- /dev/null +++ b/software/applications/osdu-core/override.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: repository-override + namespace: osdu-core +data: + repository.yaml: | + configuration: + - service: sample + repository: acr_name.azurecr.io/sample \ No newline at end of file diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index f5e02b7e..f8496905 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -27,6 +27,10 @@ spec: - kind: ConfigMap name: configmap-software valuesKey: value.yaml + - kind: ConfigMap + name: repository-override + optional: true # Makes this ConfigMap optional + valuesKey: repository.yaml values: nameOverride: partition installationType: osduCore @@ -43,8 +47,8 @@ spec: gateways: - istio-system/internal-gateway - istio-system/external-gateway - repository: maindwqgbkzaiijzc.azurecr.io/partition - # repository: community.opengroup.org:5555/osdu/platform/system/partition/partition- + # repository: maindwqgbkzaiijzc.azurecr.io/partition + repository: community.opengroup.org:5555/osdu/platform/system/partition/partition- tag: latest probe: path: /actuator/health diff --git a/src/Dockerfile-java b/src/Dockerfile-java index 6bb36b03..c8bd71ec 100644 --- a/src/Dockerfile-java +++ b/src/Dockerfile-java @@ -4,7 +4,7 @@ ARG AZUL_JDK_PACKAGE="zulu17.48.15-ca-jdk17.0.10-linux_x64" ARG BASE_IMAGE="mcr.microsoft.com/openjdk/jdk:17-mariner" ARG EXTRA_JAVA_OPTS="" -FROM ${BASE_IMAGE} as builder +FROM ${BASE_IMAGE} AS builder ARG EXTRA_FILES ARG EXTRA_FILES_DEST_DIR="extra-files" ARG INCLUDE_MODULES_OPT="" @@ -42,7 +42,7 @@ RUN if [ -n "$EXTRA_FILES" ]; then \ cp -r "src/$EXTRA_FILES" "$EXTRA_FILES_DEST_DIR/$RELATIVE_PATH"; \ fi -FROM --platform=linux/amd64 mcr.microsoft.com/cbl-mariner/base/core:2.0 +FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 ARG AZUL_JDK_PACKAGE ARG EXTRA_JAVA_OPTS ARG JAR_FILE @@ -72,5 +72,5 @@ ENV PATH="/usr/lib/jvm/${AZUL_JDK_PACKAGE}/bin:$PATH" ENV JAVA_HOME="/usr/lib/jvm/${AZUL_JDK_PACKAGE}" ENV DEFAULT_JVM_OPTS="-XX:+UseG1GC -XX:InitialRAMPercentage=25.0 -XX:MaxRAMPercentage=50.0 -XX:+HeapDumpOnOutOfMemoryError" ENV LOGGING_JVM_OPTS="-DAPPINSIGHTS_LOGGING_ENABLED=false -Dlog4j2.formatMsgNoLookups=true -Djna.library.path=/usr/lib -DAZURE_AUDIT_ENABLED=true" -ENV JAVA_OPTS="${DEFAULT_JVM_OPTS} ${LOGGING_JVM_OPTS} ${EXTRA_JAVA_OPTS} ${JAVA_OPTS}" -ENTRYPOINT java ${JAVA_OPTS} -jar /app.jar \ No newline at end of file +ENV JAVA_OPTS="${DEFAULT_JVM_OPTS} ${LOGGING_JVM_OPTS} ${EXTRA_JAVA_OPTS}" +ENTRYPOINT ["java", "-jar", "/app.jar"] \ No newline at end of file diff --git a/src/docker-bake.hcl b/src/docker-bake.hcl new file mode 100644 index 00000000..40da4077 --- /dev/null +++ b/src/docker-bake.hcl @@ -0,0 +1,166 @@ +# docker buildx bake +# Set to "true" to build for arm64 +variable "BUILD_ARM" { + default = "auto" +} + +function "platforms" { + params = [] + result = equal(BUILD_ARM, "true") ? ["linux/amd64", "linux/arm64"] : ["linux/amd64"] +} + +variable "REGISTRY" {} + +group "default" { + targets = [ + "partition", + "entitlements", + "legal", + "schema", + "storage", + "file", + "indexer", + "indexer-queue", + "search", + "crs-catalog", + "crs-conversion", + "unit" + ] +} + +target "partition" { + context = "." + dockerfile = "Dockerfile-java" + args = { + SERVICE_PATH = "./core/partition" + } + platforms = platforms() + tags = ["${REGISTRY}partition"] + output = ["type=image,push=true"] +} + +target "entitlements" { + context = "." + dockerfile = "Dockerfile-java" + args = { + SERVICE_PATH = "./core/entitlements" + SKIP_TESTS = "true" + } + platforms = platforms() + tags = ["${REGISTRY}entitlements"] + output = ["type=image,push=true"] +} + +target "legal" { + context = "." + dockerfile = "Dockerfile-java" + args = { + SERVICE_PATH = "./core/legal" + } + platforms = platforms() + tags = ["${REGISTRY}legal"] + output = ["type=image,push=true"] +} + +target "schema" { + context = "." + dockerfile = "Dockerfile-java" + args = { + SERVICE_PATH = "./core/schema" + } + platforms = platforms() + tags = ["${REGISTRY}schema"] + output = ["type=image,push=true"] +} + +target "storage" { + context = "." + dockerfile = "Dockerfile-java" + args = { + SERVICE_PATH = "./core/storage" + } + platforms = platforms() + tags = ["${REGISTRY}storage"] + output = ["type=image,push=true"] +} + +target "file" { + context = "." + dockerfile = "Dockerfile-java" + args = { + SERVICE_PATH = "./core/file" + } + platforms = platforms() + tags = ["${REGISTRY}file"] + output = ["type=image,push=true"] +} + +target "indexer" { + context = "." + dockerfile = "Dockerfile-java" + args = { + SERVICE_PATH = "./core/indexer" + } + platforms = platforms() + tags = ["${REGISTRY}indexer"] + output = ["type=image,push=true"] +} + +target "indexer-queue" { + context = "." + dockerfile = "Dockerfile-java" + args = { + SERVICE_PATH = "./core/indexer-queue" + } + platforms = platforms() + tags = ["${REGISTRY}indexer-queue"] + output = ["type=image,push=true"] +} + +target "search" { + context = "." + dockerfile = "Dockerfile-java" + args = { + SERVICE_PATH = "./core/search" + INCLUDE_MODULES_OPT = "-pl search-core,provider/search-azure" + } + platforms = platforms() + tags = ["${REGISTRY}search"] + output = ["type=image,push=true"] +} + +target "crs-catalog" { + context = "." + dockerfile = "Dockerfile-java" + args = { + SERVICE_PATH = "./core/crs-catalog" + EXTRA_FILES = "data/crs_catalog_v2.json" + } + platforms = platforms() + tags = ["${REGISTRY}crs-catalog"] + output = ["type=image,push=true"] +} + +target "crs-conversion" { + context = "." + dockerfile = "Dockerfile-java" + args = { + SERVICE_PATH = "./core/crs-conversion" + EXTRA_FILES = "apachesis_setup" + } + platforms = platforms() + tags = ["${REGISTRY}crs-conversion"] + output = ["type=image,push=true"] +} + +target "unit" { + context = "." + dockerfile = "Dockerfile-java" + args = { + SERVICE_PATH = "./core/unit" + EXTRA_FILES = "data/unit_catalog_v2.json" + } + platforms = platforms() + tags = ["${REGISTRY}unit"] + output = ["type=image,push=true"] +} \ No newline at end of file From f69ffd6aae80dc1567fb6a3d688c22820a995dfe Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 27 Nov 2024 14:03:55 -0600 Subject: [PATCH 037/122] Regress to OSDU repo --- software/applications/osdu-core/entitlements.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index f3466c68..b4a1725f 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -45,8 +45,7 @@ spec: gateways: - istio-system/internal-gateway - istio-system/external-gateway - repository: mainrctu55xpre6lo.azurecr.io/entitlements - # repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/entitlements/entitlements- + repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/entitlements/entitlements- tag: latest probe: path: /actuator/health From 22b45ef20bc22acf2d4f1dcdb2e83e3bd0e73e0f Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 27 Nov 2024 14:18:51 -0600 Subject: [PATCH 038/122] Updated entitlements --- software/applications/osdu-core/entitlements.yaml | 4 ++++ software/applications/osdu-core/partition.yaml | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index b4a1725f..1409e0f2 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -27,6 +27,10 @@ spec: - kind: ConfigMap name: configmap-software valuesKey: value.yaml + - kind: ConfigMap + name: repository-override + optional: true + valuesKey: repository.yaml values: nameOverride: entitlements installationType: osduCore diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index f8496905..d3c07d42 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -29,7 +29,7 @@ spec: valuesKey: value.yaml - kind: ConfigMap name: repository-override - optional: true # Makes this ConfigMap optional + optional: true valuesKey: repository.yaml values: nameOverride: partition @@ -47,7 +47,6 @@ spec: gateways: - istio-system/internal-gateway - istio-system/external-gateway - # repository: maindwqgbkzaiijzc.azurecr.io/partition repository: community.opengroup.org:5555/osdu/platform/system/partition/partition- tag: latest probe: From 8ce39eff80f5656e414e41eb7209b1127fcbb9e5 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 27 Nov 2024 14:46:50 -0600 Subject: [PATCH 039/122] Add in the job. --- .../applications/osdu-core/entitlements.yaml | 100 +++++++++--------- software/applications/osdu-core/override.yaml | 4 +- 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index 1409e0f2..b970a5da 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -128,53 +128,53 @@ spec: value: "1" - name: PARTITION_SERVICE_ENDPOINT value: "http://partition/api/partition/v1" -# --- -# # Retrigger: kubectl annotate helmrelease osdu-init-entitlements fluxcd.io/retrigger=$(date +%s) -n osdu-core -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-entitlements -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# fluxcd.io/retrigger: "initial" -# spec: -# dependsOn: -# - name: osdu-entitlements -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# installationType: osduCore -# jobs: -# partitionInit: false -# entitlementInit: true -# userInit: false -# elasticInit: false -# schemaInit: false -# partition: opendes -# clientSecret: -# name: active-directory -# key: principal-clientpassword -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: clientId -# valuesKey: client_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: tenantId -# valuesKey: tenant_id +--- +# Retrigger: kubectl annotate helmrelease osdu-init-entitlements fluxcd.io/retrigger=$(date +%s) -n osdu-core +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-entitlements + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" + fluxcd.io/retrigger: "initial" +spec: + dependsOn: + - name: osdu-entitlements + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + installationType: osduCore + jobs: + partitionInit: false + entitlementInit: true + userInit: false + elasticInit: false + schemaInit: false + partition: opendes + clientSecret: + name: active-directory + key: principal-clientpassword + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: clientId + valuesKey: client_id + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id diff --git a/software/applications/osdu-core/override.yaml b/software/applications/osdu-core/override.yaml index a365c02d..a45ee361 100644 --- a/software/applications/osdu-core/override.yaml +++ b/software/applications/osdu-core/override.yaml @@ -7,5 +7,5 @@ metadata: data: repository.yaml: | configuration: - - service: sample - repository: acr_name.azurecr.io/sample \ No newline at end of file + - service: entitlements + repository: maindwqgbkzaiijzc.azurecr.io/entitlements:latest \ No newline at end of file From 144f18a49d127932ddee967f1c4d093f90f73bef Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 27 Nov 2024 16:15:20 -0600 Subject: [PATCH 040/122] Added app insights --- charts/osdu-developer-init/Chart.yaml | 2 +- .../templates/entitlement-init.yaml | 2 -- software/applications/osdu-core/entitlements.yaml | 1 + software/applications/osdu-core/override.yaml | 11 ----------- software/components/global/repository-override.yaml | 11 +++++++++++ src/Dockerfile-java | 9 +++++++-- 6 files changed, 20 insertions(+), 16 deletions(-) delete mode 100644 software/applications/osdu-core/override.yaml create mode 100644 software/components/global/repository-override.yaml diff --git a/charts/osdu-developer-init/Chart.yaml b/charts/osdu-developer-init/Chart.yaml index af8bcbbf..2f97a4f5 100644 --- a/charts/osdu-developer-init/Chart.yaml +++ b/charts/osdu-developer-init/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-init type: application description: Installs the OSDU developer Initialization resources -version: 0.0.1 +version: 0.0.2 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/osdu-developer-init/templates/entitlement-init.yaml b/charts/osdu-developer-init/templates/entitlement-init.yaml index c03c7e19..e25bcb73 100644 --- a/charts/osdu-developer-init/templates/entitlement-init.yaml +++ b/charts/osdu-developer-init/templates/entitlement-init.yaml @@ -72,8 +72,6 @@ data: # Get token (no resource needed) TOKEN=$(az account get-access-token --resource ${AZURE_AD_APPLICATION_ID} --query accessToken -o tsv) - sleep 300000 - OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://entitlements.{{ $namespace }}/api/entitlements/v2/tenant-provisioning \ --header "Host: entitlements.{{ $namespace }}" \ diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index b970a5da..00336feb 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -29,6 +29,7 @@ spec: valuesKey: value.yaml - kind: ConfigMap name: repository-override + namespace: default optional: true valuesKey: repository.yaml values: diff --git a/software/applications/osdu-core/override.yaml b/software/applications/osdu-core/override.yaml deleted file mode 100644 index a45ee361..00000000 --- a/software/applications/osdu-core/override.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: repository-override - namespace: osdu-core -data: - repository.yaml: | - configuration: - - service: entitlements - repository: maindwqgbkzaiijzc.azurecr.io/entitlements:latest \ No newline at end of file diff --git a/software/components/global/repository-override.yaml b/software/components/global/repository-override.yaml new file mode 100644 index 00000000..392c7d5c --- /dev/null +++ b/software/components/global/repository-override.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: repository-override + namespace: default +data: + repository.yaml: | + configuration: + - service: service_name + repository: acr_name.azurecr.io/service_name:latest \ No newline at end of file diff --git a/src/Dockerfile-java b/src/Dockerfile-java index c8bd71ec..a5967b17 100644 --- a/src/Dockerfile-java +++ b/src/Dockerfile-java @@ -49,12 +49,17 @@ ARG JAR_FILE ARG IFX_AUDIT_PACKAGE ARG EXTRA_FILES ARG EXTRA_FILES_DEST_DIR="extra-files" +ARG APPLICATIONINSIGHTS_VERSION="3.5.4" # Install required packages RUN tdnf update -y && \ tdnf install -y curl tar ca-certificates && \ rm -rf /var/cache/tdnf/* +# Download Application Insights agent +RUN curl -LO https://github.com/microsoft/ApplicationInsights-Java/releases/download/${APPLICATIONINSIGHTS_VERSION}/applicationinsights-agent-${APPLICATIONINSIGHTS_VERSION}.jar \ + && mv applicationinsights-agent-${APPLICATIONINSIGHTS_VERSION}.jar applicationinsights-agent.jar + RUN curl -LO https://cdn.azul.com/zulu/bin/${AZUL_JDK_PACKAGE}.tar.gz \ && mkdir -p /usr/lib/jvm \ && tar -xf ./${AZUL_JDK_PACKAGE}.tar.gz -C /usr/lib/jvm \ @@ -71,6 +76,6 @@ COPY --from=builder /app/${EXTRA_FILES_DEST_DIR}/${EXTRA_FILES} ${EXTRA_FILES} ENV PATH="/usr/lib/jvm/${AZUL_JDK_PACKAGE}/bin:$PATH" ENV JAVA_HOME="/usr/lib/jvm/${AZUL_JDK_PACKAGE}" ENV DEFAULT_JVM_OPTS="-XX:+UseG1GC -XX:InitialRAMPercentage=25.0 -XX:MaxRAMPercentage=50.0 -XX:+HeapDumpOnOutOfMemoryError" -ENV LOGGING_JVM_OPTS="-DAPPINSIGHTS_LOGGING_ENABLED=false -Dlog4j2.formatMsgNoLookups=true -Djna.library.path=/usr/lib -DAZURE_AUDIT_ENABLED=true" +ENV LOGGING_JVM_OPTS="-DAPPINSIGHTS_LOGGING_ENABLED=true -Dlog4j2.formatMsgNoLookups=true -Djna.library.path=/usr/lib -DAZURE_AUDIT_ENABLED=true" ENV JAVA_OPTS="${DEFAULT_JVM_OPTS} ${LOGGING_JVM_OPTS} ${EXTRA_JAVA_OPTS}" -ENTRYPOINT ["java", "-jar", "/app.jar"] \ No newline at end of file +ENTRYPOINT ["java", "-javaagent:/applicationinsights-agent.jar", "-jar", "/app.jar"] \ No newline at end of file From b7b97e1daa5e4fad30aef07b2ee94ce426e446d2 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 27 Nov 2024 16:24:05 -0600 Subject: [PATCH 041/122] Updated entitlements --- software/applications/osdu-core/entitlements.yaml | 1 - .../osdu-core/repo-override.yaml} | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) rename software/{components/global/repository-override.yaml => applications/osdu-core/repo-override.yaml} (76%) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index 00336feb..b970a5da 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -29,7 +29,6 @@ spec: valuesKey: value.yaml - kind: ConfigMap name: repository-override - namespace: default optional: true valuesKey: repository.yaml values: diff --git a/software/components/global/repository-override.yaml b/software/applications/osdu-core/repo-override.yaml similarity index 76% rename from software/components/global/repository-override.yaml rename to software/applications/osdu-core/repo-override.yaml index 392c7d5c..6e3b6bf4 100644 --- a/software/components/global/repository-override.yaml +++ b/software/applications/osdu-core/repo-override.yaml @@ -1,3 +1,4 @@ +## This file can be used to override the repository for service images. --- apiVersion: v1 kind: ConfigMap From 772f4454ddc5ac7115618e5f0faa87fed72cd8e7 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 27 Nov 2024 16:29:37 -0600 Subject: [PATCH 042/122] Updated override repo --- software/applications/osdu-core/repo-override.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/applications/osdu-core/repo-override.yaml b/software/applications/osdu-core/repo-override.yaml index 6e3b6bf4..a37e876b 100644 --- a/software/applications/osdu-core/repo-override.yaml +++ b/software/applications/osdu-core/repo-override.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: repository-override - namespace: default + namespace: osdu-core data: repository.yaml: | configuration: From fd8b708c0e832ff83e7953751187620c641173ce Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 27 Nov 2024 16:54:33 -0600 Subject: [PATCH 043/122] Added registry to configmap --- bicep/main.bicep | 7 +++++++ .../osdu-core/{repo-override.yaml => configmap-repo.yaml} | 2 +- software/applications/osdu-core/entitlements.yaml | 2 +- software/applications/osdu-core/partition.yaml | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) rename software/applications/osdu-core/{repo-override.yaml => configmap-repo.yaml} (89%) diff --git a/bicep/main.bicep b/bicep/main.bicep index eb85ea30..d5a80db0 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -1087,11 +1087,18 @@ module configBlade 'modules/blade_configuration.bicep' = { contentType: 'text/plain' label: 'configmap-services' } + { + name: 'registry' + value: registry.outputs.loginServer + contentType: 'text/plain' + label: 'configmap-services' + } ] } dependsOn: [ clusterBlade partitionBlade + registry fluxExtension ] } diff --git a/software/applications/osdu-core/repo-override.yaml b/software/applications/osdu-core/configmap-repo.yaml similarity index 89% rename from software/applications/osdu-core/repo-override.yaml rename to software/applications/osdu-core/configmap-repo.yaml index a37e876b..2a9a43a0 100644 --- a/software/applications/osdu-core/repo-override.yaml +++ b/software/applications/osdu-core/configmap-repo.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: repository-override + name: configmap-repo-override namespace: osdu-core data: repository.yaml: | diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index b970a5da..db6d2815 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -28,7 +28,7 @@ spec: name: configmap-software valuesKey: value.yaml - kind: ConfigMap - name: repository-override + name: configmap-repo-override optional: true valuesKey: repository.yaml values: diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index d3c07d42..2d251212 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -28,7 +28,7 @@ spec: name: configmap-software valuesKey: value.yaml - kind: ConfigMap - name: repository-override + name: configmap-repo-override optional: true valuesKey: repository.yaml values: From 71ccbee9f4324a2eed403c7c348403f414fb9086 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sat, 30 Nov 2024 17:17:03 -0600 Subject: [PATCH 044/122] Added configuration for workload identity. --- bicep/modules/blade_configuration.bicep | 58 ++++++++++--------- .../applications/osdu-core/entitlements.yaml | 25 ++++---- 2 files changed, 47 insertions(+), 36 deletions(-) diff --git a/bicep/modules/blade_configuration.bicep b/bicep/modules/blade_configuration.bicep index 3b0feba8..0f2cdc60 100644 --- a/bicep/modules/blade_configuration.bicep +++ b/bicep/modules/blade_configuration.bicep @@ -1,5 +1,5 @@ ///////////////// -// Configuration Blade +// Configuration Blade ///////////////// @description('The configuration for the blade section.') @@ -12,10 +12,10 @@ param location string param tags object = {} @description('The name of the Key Vault where the secret exists') -param kvName string +param kvName string @description('The Uri of the Key Vault where the secret exists') -param kvUri string +param kvUri string @description('The name of the cluster.') param clusterName string @@ -169,12 +169,12 @@ var federatedIdentityCredentials = [ ] /* - _______ _______ _______ _______ .______ ___ .___________. __ ______ .__ __. -| ____|| ____|| \ | ____|| _ \ / \ | || | / __ \ | \ | | -| |__ | |__ | .--. || |__ | |_) | / ^ \ `---| |----`| | | | | | | \| | -| __| | __| | | | || __| | / / /_\ \ | | | | | | | | | . ` | -| | | |____ | '--' || |____ | |\ \----./ _____ \ | | | | | `--' | | |\ | -|__| |_______||_______/ |_______|| _| `._____/__/ \__\ |__| |__| \______/ |__| \__| + _______ _______ _______ _______ .______ ___ .___________. __ ______ .__ __. +| ____|| ____|| \ | ____|| _ \ / \ | || | / __ \ | \ | | +| |__ | |__ | .--. || |__ | |_) | / ^ \ `---| |----`| | | | | | | \| | +| __| | __| | | | || __| | / / /_\ \ | | | | | | | | | . ` | +| | | |____ | '--' || |____ | |\ \----./ _____ \ | | | | | `--' | | |\ | +|__| |_______||_______/ |_______|| _| `._____/__/ \__\ |__| |__| \______/ |__| \__| */ @batchSize(1) module federatedCredentials './federated_identity.bicep' = [for (cred, index) in federatedIdentityCredentials: { @@ -207,6 +207,12 @@ var common_helm_values = [ contentType: 'text/plain' label: 'configmap-common-values' } + { + name: 'AZURE_PAAS_WORKLOADIDENTITY_ISENABLED' + value: 'true' + contentType: 'text/plain' + label: 'configmap-common-values' + } { name: 'ACCEPT_HTTP' value: 'true' @@ -323,12 +329,12 @@ var partitionStorageSettings = [for (name, i) in partitionStorageNames: { }] /* - ___ .______ .______ ______ ______ .__ __. _______ __ _______ + ___ .______ .______ ______ ______ .__ __. _______ __ _______ / \ | _ \ | _ \ / | / __ \ | \ | | | ____|| | / _____| - / ^ \ | |_) | | |_) | | ,----'| | | | | \| | | |__ | | | | __ - / /_\ \ | ___/ | ___/ | | | | | | | . ` | | __| | | | | |_ | - / _____ \ | | | | | `----.| `--' | | |\ | | | | | | |__| | -/__/ \__\ | _| | _| \______| \______/ |__| \__| |__| |__| \______| + / ^ \ | |_) | | |_) | | ,----'| | | | | \| | | |__ | | | | __ + / /_\ \ | ___/ | ___/ | | | | | | | . ` | | __| | | | | |_ | + / _____ \ | | | | | `----.| `--' | | |\ | | | | | | |__| | +/__/ \__\ | _| | _| \______| \______/ |__| \__| |__| |__| \______| */ // AVM Module Customized due for east of settings. module app_config './app-configuration/main.bicep' = { @@ -397,12 +403,12 @@ values.yaml: | } /* - ______ ______ .__ __. _______ __ _______ .___ ___. ___ .______ - / | / __ \ | \ | | | ____|| | / _____|| \/ | / \ | _ \ -| ,----'| | | | | \| | | |__ | | | | __ | \ / | / ^ \ | |_) | -| | | | | | | . ` | | __| | | | | |_ | | |\/| | / /_\ \ | ___/ -| `----.| `--' | | |\ | | | | | | |__| | | | | | / _____ \ | | - \______| \______/ |__| \__| |__| |__| \______| |__| |__| /__/ \__\ | _| + ______ ______ .__ __. _______ __ _______ .___ ___. ___ .______ + / | / __ \ | \ | | | ____|| | / _____|| \/ | / \ | _ \ +| ,----'| | | | | \| | | |__ | | | | __ | \ / | / ^ \ | |_) | +| | | | | | | . ` | | __| | | | | |_ | | |\/| | / /_\ \ | ___/ +| `----.| `--' | | |\ | | | | | | |__| | | | | | / _____ \ | | + \______| \______/ |__| \__| |__| |__| \______| |__| |__| /__/ \__\ | _| */ module appConfigMap './aks-config-map/main.bicep' = { name: '${bladeConfig.sectionName}-cluster-appconfig-configmap' @@ -411,7 +417,7 @@ module appConfigMap './aks-config-map/main.bicep' = { location: location name: 'config-map-values' namespace: 'default' - + newOrExistingManagedIdentity: 'existing' managedIdentityName: managedIdentityName existingManagedIdentitySubId: subscription().subscriptionId @@ -419,8 +425,8 @@ module appConfigMap './aks-config-map/main.bicep' = { // Order of items matters here. fileData: [ - format(configMaps.appConfigTemplate, - subscription().tenantId, + format(configMaps.appConfigTemplate, + subscription().tenantId, appIdentity.properties.clientId, app_config.outputs.endpoint, kvUri, @@ -457,9 +463,9 @@ var serviceLayerConfig = { /* _______ __ .___________. ______ .______ _______. / _____|| | | | / __ \ | _ \ / | | | __ | | `---| |----`| | | | | |_) | | (----` -| | |_ | | | | | | | | | | ___/ \ \ -| |__| | | | | | | `--' | | | .----) | - \______| |__| |__| \______/ | _| |_______/ +| | |_ | | | | | | | | | | ___/ \ \ +| |__| | | | | | | `--' | | | .----) | + \______| |__| |__| \______/ | _| |_______/ */ //--------------Flux Config--------------- module fluxConfiguration 'br/public:avm/res/kubernetes-configuration/flux-configuration:0.3.3' = if(enableSoftwareLoad) { diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index db6d2815..6350a959 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -27,6 +27,9 @@ spec: - kind: ConfigMap name: configmap-software valuesKey: value.yaml + - kind: ConfigMap + name: configmap-common-values + valuesKey: value.yaml - kind: ConfigMap name: configmap-repo-override optional: true @@ -102,16 +105,18 @@ spec: secret: name: azure-resources key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" - - name: SERVER_SERVLET_CONTEXTPATH - value: "/api/entitlements/v2/" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" + # - name: AZURE_ISTIOAUTH_ENABLED + # value: "true" + # - name: AZURE_PAAS_PODIDENTITY_ISENABLED + # value: "true" + # - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + # value: "true" + # - name: SERVER_SERVLET_CONTEXTPATH + # value: "/api/entitlements/v2/" + # - name: SERVER_PORT + # value: "80" + # - name: ACCEPT_HTTP + # value: "true" - name: SPRING_APPLICATION_NAME value: "entitlements" - name: SPRING_CONFIG_NAME From 564b49be7dbd9856e3667db0a15346992aa09afa Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sat, 30 Nov 2024 17:27:50 -0600 Subject: [PATCH 045/122] Added configuration for workload identity. --- .../applications/osdu-core/entitlements.yaml | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index 6350a959..09616b54 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -30,10 +30,10 @@ spec: - kind: ConfigMap name: configmap-common-values valuesKey: value.yaml - - kind: ConfigMap - name: configmap-repo-override - optional: true - valuesKey: repository.yaml + # - kind: ConfigMap + # name: configmap-repo-override + # optional: true + # valuesKey: repository.yaml values: nameOverride: entitlements installationType: osduCore @@ -105,18 +105,18 @@ spec: secret: name: azure-resources key: insights-connection - # - name: AZURE_ISTIOAUTH_ENABLED - # value: "true" - # - name: AZURE_PAAS_PODIDENTITY_ISENABLED - # value: "true" - # - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - # value: "true" - # - name: SERVER_SERVLET_CONTEXTPATH - # value: "/api/entitlements/v2/" - # - name: SERVER_PORT - # value: "80" - # - name: ACCEPT_HTTP - # value: "true" + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_PODIDENTITY_ISENABLED + value: "false" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_SERVLET_CONTEXTPATH + value: "/api/entitlements/v2/" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" - name: SPRING_APPLICATION_NAME value: "entitlements" - name: SPRING_CONFIG_NAME From 26faccf0262cd746393eda592ca318ce88f3cc86 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 1 Dec 2024 17:22:14 -0600 Subject: [PATCH 046/122] Updated Envoy with debugging and modified request-authentication for multiple jwt audiences. --- .../templates/envoy-filter.yaml | 134 ++++-------------- .../templates/request-authentication.yaml | 3 + 2 files changed, 28 insertions(+), 109 deletions(-) diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index efe8289d..15ab6739 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -17,7 +17,7 @@ spec: filterChain: filter: name: envoy.filters.network.http_connection_manager - subFilter: + subFilter: name: envoy.filters.http.router patch: operation: INSERT_BEFORE @@ -30,28 +30,28 @@ spec: 1. AAD v1 User Token (sts.windows.net): - With OID: x-user-id = oid - Without OID: x-user-id = upn/unique_name - + 2. AAD v1 Service-to-Service Delegation: - Service identity (appid) matches managedIdentityClientId - Delegation flow: x-user-id = x-on-behalf-of (delegated user) - Fallback: x-user-id = appid (direct service call) - + 3. AAD v1 Application (non-delegated): - x-user-id = appid - + 4. AAD v2 User Token (login.microsoftonline.com): - Has 'scp' claim: x-user-id = oid - + 5. AAD v2 Service-to-Service Delegation: - Service identity (azp) matches managedIdentityClientId - Delegation flow: x-user-id = x-on-behalf-of (delegated user) - Fallback: x-user-id = azp (direct service call) - + 6. AAD v2 Application (non-delegated): - x-user-id = azp or oid Note: All scenarios set x-app-id from 'aud' claim when present - + OAuth Delegation (On-Behalf-Of) Flow: - When a service calls another service on behalf of a user - The original user's identity is passed via x-on-behalf-of header @@ -60,17 +60,28 @@ spec: ]] function envoy_on_request(request_handle) + -- Add initial debug log + request_handle:logDebug("Starting envoy_on_request processing") + -- Remove headers first request_handle:headers():remove("x-user-id") request_handle:headers():remove("x-app-id") request_handle:logInfo("x-user-id and x-app-id headers removed") - -- Get JWT metadata + -- Get JWT metadata with debug logging local meta = request_handle:streamInfo():dynamicMetadata():get("envoy.filters.http.jwt_authn") + if meta then + request_handle:logDebug("JWT metadata found") + else + request_handle:logDebug("No JWT metadata found") + return + end - if meta and meta["payload"] then + if meta["payload"] then + request_handle:logDebug("JWT payload found") local payload = meta["payload"] - request_handle:logDebug("JWT Payload: " .. tableToString(payload)) + -- Log the raw payload for debugging + request_handle:logDebug("Raw JWT payload: " .. tableToString(payload)) -- Always set x-app-id from aud claim if present if payload["aud"] then @@ -78,7 +89,7 @@ spec: request_handle:logWarn("x-app-id set from 'aud' claim: " .. payload["aud"]) end - -- Check issuer + -- Check issuer if string.find(payload["iss"], "sts.windows.net") then -- AAD v1 token handling if payload["upn"] then @@ -152,7 +163,8 @@ spec: request_handle:logError("Issuer does not match known issuers") end else - request_handle:logError("No JWT metadata found or payload is malformed") + request_handle:logDebug("No JWT payload in metadata") + return end end @@ -171,100 +183,4 @@ spec: end end return table.concat(lines, "\n") - end - -# # This command increases logging --> istioctl proxy-config log --level lua:debug -# apiVersion: networking.istio.io/v1alpha3 -# kind: EnvoyFilter -# metadata: -# name: microsoft-identity-filter -# spec: -# configPatches: -# - applyTo: HTTP_FILTER -# match: -# context: SIDECAR_INBOUND -# listener: -# filterChain: -# filter: -# name: envoy.filters.network.http_connection_manager -# subFilter: -# name: envoy.filters.http.router -# patch: -# operation: INSERT_BEFORE -# value: -# name: envoy.lua.remove-user-appid-header -# typed_config: -# "@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua" -# inlineCode: | -# function envoy_on_request(request_handle) -# -- Remove the Well Known Headers -# request_handle:headers():remove("x-user-id") -# request_handle:headers():remove("x-app-id") -# request_handle:logInfo("x-user-id and x-app-id headers removed") - -# -- Retrieve the JWT Payload -# local meta = request_handle:streamInfo():dynamicMetadata():get("envoy.filters.http.jwt_authn") - -# if meta and meta["payload"] then -# local payload = meta["payload"] -# request_handle:logDebug("JWT Payload: " .. tableToString(payload)) - -# -- Set the x-app-id Well Known Header if 'aud' claim is present -# if payload["aud"] then -# request_handle:headers():add("x-app-id", payload["aud"]) -# request_handle:logWarn("x-app-id set from 'aud' claim: " .. payload["aud"]) -# else -# request_handle:logError("JWT Payload does not contain 'aud' claim; cannot set x-app-id") -# end - -# -- Check issuer -# if string.find(payload["iss"], "sts.windows.net") then -# -- Set Well Known Header with an order of preference: upn, unique_name, appid -# if payload["upn"] then -# request_handle:headers():add("x-user-id", payload["upn"]) -# request_handle:logWarn("x-user-id set from 'upn' claim: " .. payload["upn"]) -# elseif payload["unique_name"] then -# request_handle:headers():add("x-user-id", payload["unique_name"]) -# request_handle:logWarn("x-user-id set from 'unique_name' claim: " .. payload["unique_name"]) -# elseif payload["appid"] then -# request_handle:headers():add("x-user-id", payload["appid"]) -# request_handle:logWarn("x-user-id set from 'appid' claim: " .. payload["appid"]) -# else -# request_handle:logError("No valid user ID claim (upn, unique_name, appid) found for sts.windows.net") -# end - -# elseif string.find(payload["iss"], "login.microsoftonline.com") then -# -- Set Well Known Header with an order of preference: azp, oid -# if payload["azp"] then -# request_handle:headers():add("x-user-id", payload["azp"]) -# request_handle:logWarn("x-user-id set from 'azp' claim: " .. payload["azp"]) -# elseif payload["oid"] then -# request_handle:headers():add("x-user-id", payload["oid"]) -# request_handle:logWarn("x-user-id set from 'oid' claim: " .. payload["oid"]) -# else -# request_handle:logError("No valid user ID claim (azp, oid) found for login.microsoftonline.com") -# end -# else -# request_handle:logError("Issuer does not match known issuers") -# end -# else -# request_handle:logError("No JWT metadata found or payload is malformed") -# end -# end - -# -- Helper function to convert a table to a string for logging -# function tableToString(tbl, indent) -# if not indent then indent = 0 end -# if type(tbl) ~= 'table' then return tostring(tbl) end -# local lines = {} -# for k, v in pairs(tbl) do -# local formatting = string.rep(" ", indent) .. k .. ": " -# if type(v) == "table" then -# table.insert(lines, formatting) -# table.insert(lines, tableToString(v, indent+1)) -# else -# table.insert(lines, formatting .. tostring(v)) -# end -# end -# return table.concat(lines, "\n") -# end \ No newline at end of file + end \ No newline at end of file diff --git a/charts/osdu-developer-base/templates/request-authentication.yaml b/charts/osdu-developer-base/templates/request-authentication.yaml index f9c05634..89eb0c78 100644 --- a/charts/osdu-developer-base/templates/request-authentication.yaml +++ b/charts/osdu-developer-base/templates/request-authentication.yaml @@ -1,6 +1,7 @@ {{- if .Values.enableRequestAuthentication }} {{- $tenantId := .Values.azure.tenantId -}} {{- $appId := .Values.azure.appId -}} +{{- $clientId := .Values.azure.clientId -}} apiVersion: security.istio.io/v1beta1 kind: RequestAuthentication metadata: @@ -12,6 +13,7 @@ spec: jwksUri: "https://login.microsoftonline.com:443/common/discovery/v2.0/keys" audiences: - "{{ $appId }}" + - "{{ $clientId }}" outputPayloadToHeader: "x-payload" forwardOriginalToken: true fromHeaders: @@ -21,6 +23,7 @@ spec: jwksUri: "https://login.microsoftonline.com/common/discovery/v2.0/keys" audiences: - "{{ $appId }}" + - "{{ $clientId }}" outputPayloadToHeader: "x-payload" forwardOriginalToken: true fromHeaders: From 2eeab0c34e4e240c4c649bf6a4b74e3f74eff55a Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 2 Dec 2024 13:34:36 -0600 Subject: [PATCH 047/122] Added an additional audience. --- charts/osdu-developer-base/templates/envoy-filter.yaml | 2 ++ .../osdu-developer-base/templates/request-authentication.yaml | 1 + 2 files changed, 3 insertions(+) diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index 15ab6739..79e4713b 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -1,4 +1,6 @@ # istioctl proxy-config log --level lua:debug +# istioctl proxy-config log --level jwt:debug +# istioctl proxy-config log --level rbac:debug {{- $namespace := .Release.Namespace }} {{- $managedIdentityClientId := .Values.azure.clientId }} diff --git a/charts/osdu-developer-base/templates/request-authentication.yaml b/charts/osdu-developer-base/templates/request-authentication.yaml index 89eb0c78..af1b2dc1 100644 --- a/charts/osdu-developer-base/templates/request-authentication.yaml +++ b/charts/osdu-developer-base/templates/request-authentication.yaml @@ -14,6 +14,7 @@ spec: audiences: - "{{ $appId }}" - "{{ $clientId }}" + - "https://management.azure.com" outputPayloadToHeader: "x-payload" forwardOriginalToken: true fromHeaders: From 8c292973bc6e31992fb7fe36ce63cd90cebac429 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Tue, 3 Dec 2024 12:15:28 -0600 Subject: [PATCH 048/122] Moved back to SP --- .../applications/osdu-core/entitlements.yaml | 38 +++++++++---------- .../applications/osdu-core/partition.yaml | 6 ++- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index 09616b54..c1d80e4a 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -73,26 +73,26 @@ spec: - "/api/entitlements/v2/api-docs*" - "/api/entitlements/v2/webjars/*" env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword + - name: AZURE_TENANT_ID + secret: + name: active-directory + key: tenant-id + - name: AZURE_SUBSCRIPTION_ID + secret: + name: active-directory + key: subscription-id + - name: AZURE_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: AZURE_CLIENT_SECRET + secret: + name: active-directory + key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources - key: keyvault-uri + key: keyvault-uri - name: AAD_CLIENT_ID secret: name: active-directory @@ -110,7 +110,7 @@ spec: - name: AZURE_PAAS_PODIDENTITY_ISENABLED value: "false" - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - value: "true" + value: "false" - name: SERVER_SERVLET_CONTEXTPATH value: "/api/entitlements/v2/" - name: SERVER_PORT @@ -142,7 +142,7 @@ metadata: namespace: osdu-core annotations: clusterconfig.azure.com/use-managed-source: "true" - fluxcd.io/retrigger: "initial" + fluxcd.io/retrigger: "initial" spec: dependsOn: - name: osdu-entitlements diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 2d251212..6a1812ea 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -88,7 +88,9 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "true" + value: "false" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "false" - name: SERVER_SERVLET_CONTEXTPATH value: "/api/partition/v1/" - name: SERVER_PORT @@ -139,7 +141,7 @@ spec: valuesFrom: - kind: ConfigMap name: configmap-software - valuesKey: value.yaml + valuesKey: value.yaml - kind: Secret name: active-directory targetPath: clientId From 6abb7443ca377ac6a07e39d1507ae79ee3183ee5 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Tue, 3 Dec 2024 14:25:30 -0600 Subject: [PATCH 049/122] Moved to Workload Identity. --- .../applications/osdu-core/entitlements.yaml | 34 +++++++++---------- .../applications/osdu-core/partition.yaml | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index c1d80e4a..10279907 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -73,22 +73,22 @@ spec: - "/api/entitlements/v2/api-docs*" - "/api/entitlements/v2/webjars/*" env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_SUBSCRIPTION_ID - secret: - name: active-directory - key: subscription-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword + # - name: AZURE_TENANT_ID + # secret: + # name: active-directory + # key: tenant-id + # - name: AZURE_SUBSCRIPTION_ID + # secret: + # name: active-directory + # key: subscription-id + # - name: AZURE_CLIENT_ID + # secret: + # name: active-directory + # key: principal-clientid + # - name: AZURE_CLIENT_SECRET + # secret: + # name: active-directory + # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -110,7 +110,7 @@ spec: - name: AZURE_PAAS_PODIDENTITY_ISENABLED value: "false" - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_SERVLET_CONTEXTPATH value: "/api/entitlements/v2/" - name: SERVER_PORT diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 6a1812ea..5fb90fa7 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -90,7 +90,7 @@ spec: - name: AZURE_PAAS_PODIDENTITY_ISENABLED value: "false" - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_SERVLET_CONTEXTPATH value: "/api/partition/v1/" - name: SERVER_PORT From ba977863ccddb16d1d7eb2728c9eb19c92f77d70 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Tue, 3 Dec 2024 14:29:03 -0600 Subject: [PATCH 050/122] Changed auth method in partition init. --- charts/osdu-developer-init/templates/partition-init.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index 1419d0cd..95c81a3d 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -38,7 +38,7 @@ spec: - name: PARTITION value: {{ .Values.partition | quote }} - name: SERVICE_BUS_NAME - value: {{ .Values.serviceBus | quote }} + value: {{ .Values.serviceBus | quote }} containers: - name: sleep image: istio/base @@ -215,8 +215,8 @@ data: -u ${AZURE_CLIENT_ID} \ -t ${AZURE_TENANT_ID} - # Get token (no resource needed) - TOKEN=$(az account get-access-token --resource ${AZURE_AD_APPLICATION_ID} --query accessToken -o tsv) + # Get token with the correct application ID as resource + TOKEN=$(az account get-access-token --resource "api://${AZURE_AD_APPLICATION_ID}" --query accessToken -o tsv) OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://partition.{{ $namespace }}/api/partition/v1/partitions/${PARTITION} \ @@ -229,7 +229,7 @@ data: HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$') BODY=${OUTPUT%???} - + if [ "$HTTP_STATUS_CODE" == "201" ]; then echo "Success: $(echo "$BODY" | jq .)" elif [ "$HTTP_STATUS_CODE" == "409" ]; then From 30ecdb2e249941cd908d613f759f342ce75d8144 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 4 Dec 2024 08:44:24 -0600 Subject: [PATCH 051/122] Updated secrets and bitnami chart location. --- bicep/main.bicep | 247 +++++++++--------- bicep/modules/blade_configuration.bicep | 13 +- bicep/modules/keyvault_secrets.bicep | 16 +- .../templates/partition-init.yaml | 2 + software/components/osdu-system/cache.yaml | 5 +- 5 files changed, 147 insertions(+), 136 deletions(-) diff --git a/bicep/main.bicep b/bicep/main.bicep index d5a80db0..a8f96d2c 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -10,9 +10,9 @@ param emailAddress string @description('Specify the Application Client Id. (This is the unique application ID of this application.)') param applicationClientId string -@description('Specify the Application Client Secret. (A valid secret for the application client ID.)') -@secure() -param applicationClientSecret string +// @description('Specify the Application Client Secret. (A valid secret for the application client ID.)') +// @secure() +// param applicationClientSecret string @description('Specify the Enterprise Application Object Id. (This is the unique ID of the service principal object associated with the application.)') param applicationClientPrincipalOid string @@ -80,7 +80,7 @@ param vnetConfiguration object = { } ///////////////////////////////// -// Configuration +// Configuration ///////////////////////////////// // Internal Feature Flags Start -> @@ -108,10 +108,10 @@ var configuration = { tenantId: 'tenant-id' subscriptionId: 'subscription-id' registryName: 'container-registry' - applicationId: 'aad-client-id' - clientId: 'app-dev-sp-username' - clientSecret: 'app-dev-sp-password' - applicationPrincipalId: 'app-dev-sp-id' + // applicationId: 'aad-client-id' + // clientId: 'app-dev-sp-username' + // clientSecret: 'app-dev-sp-password' + // applicationPrincipalId: 'app-dev-sp-id' stampIdentity: 'osdu-identity-id' storageAccountName: 'common-storage' storageAccountKey: 'common-storage-key' @@ -157,12 +157,12 @@ var rg_unique_id = '${replace(configuration.name, '-', '')}${uniqueString(resour /* - __ _______ _______ .__ __. .___________. __ .___________.____ ____ -| | | \ | ____|| \ | | | || | | |\ \ / / -| | | .--. || |__ | \| | `---| |----`| | `---| |----` \ \/ / -| | | | | || __| | . ` | | | | | | | \_ _/ -| | | '--' || |____ | |\ | | | | | | | | | -|__| |_______/ |_______||__| \__| |__| |__| |__| |__| + __ _______ _______ .__ __. .___________. __ .___________.____ ____ +| | | \ | ____|| \ | | | || | | |\ \ / / +| | | .--. || |__ | \| | `---| |----`| | `---| |----` \ \/ / +| | | | | || __| | . ` | | | | | | | \_ _/ +| | | '--' || |____ | |\ | | | | | | | | | +|__| |_______/ |_______||__| \__| |__| |__| |__| |__| */ module stampIdentity 'br/public:avm/res/managed-identity/user-assigned-identity:0.4.0' = { name: '${configuration.name}-user-managed-identity' @@ -185,9 +185,9 @@ module stampIdentity 'br/public:avm/res/managed-identity/user-assigned-identity: ___ .__ __. ___ __ ____ ____ .___________. __ ______ _______. / \ | \ | | / \ | | \ \ / / | || | / | / | / ^ \ | \| | / ^ \ | | \ \/ / `---| |----`| | | ,----' | (----` - / /_\ \ | . ` | / /_\ \ | | \_ _/ | | | | | | \ \ - / _____ \ | |\ | / _____ \ | `----. | | | | | | | `----.----) | -/__/ \__\ |__| \__| /__/ \__\ |_______| |__| |__| |__| \______|_______/ + / /_\ \ | . ` | / /_\ \ | | \_ _/ | | | | | | \ \ + / _____ \ | |\ | / _____ \ | `----. | | | | | | | `----.----) | +/__/ \__\ |__| \__| /__/ \__\ |_______| |__| |__| |__| \______|_______/ */ module logAnalytics 'br/public:avm/res/operational-insights/workspace:0.7.1' = { name: '${configuration.name}-log-analytics' @@ -211,9 +211,9 @@ module logAnalytics 'br/public:avm/res/operational-insights/workspace:0.7.1' = { __ .__ __. _______. __ _______ __ __ .___________. _______. | | | \ | | / || | / _____|| | | | | | / | | | | \| | | (----`| | | | __ | |__| | `---| |----` | (----` -| | | . ` | \ \ | | | | |_ | | __ | | | \ \ -| | | |\ | .----) | | | | |__| | | | | | | | .----) | -|__| |__| \__| |_______/ |__| \______| |__| |__| |__| |_______/ +| | | . ` | \ \ | | | | |_ | | __ | | | \ \ +| | | |\ | .----) | | | | |__| | | | | | | | .----) | +|__| |__| \__| |_______/ |__| \______| |__| |__| |__| |_______/ */ module insights 'br/public:avm/res/insights/component:0.3.0' = { @@ -232,7 +232,7 @@ module insights 'br/public:avm/res/insights/component:0.3.0' = { kind: configuration.insights.sku workspaceResourceId: logAnalytics.outputs.resourceId - + diagnosticSettings: [ { metricCategories: [ @@ -249,12 +249,12 @@ module insights 'br/public:avm/res/insights/component:0.3.0' = { /* - ______ ___ ______ __ __ _______ + ______ ___ ______ __ __ _______ / | / \ / || | | | | ____| -| ,----' / ^ \ | ,----'| |__| | | |__ -| | / /_\ \ | | | __ | | __| -| `----./ _____ \ | `----.| | | | | |____ - \______/__/ \__\ \______||__| |__| |_______| +| ,----' / ^ \ | ,----'| |__| | | |__ +| | / /_\ \ | | | __ | | __| +| `----./ _____ \ | `----.| | | | | |____ + \______/__/ \__\ \______||__| |__| |_______| */ // This takes a long time to deploy so we are starting as soon as possible. module redis 'br/public:avm/res/cache/redis:0.3.2' = { @@ -270,7 +270,7 @@ module redis 'br/public:avm/res/cache/redis:0.3.2' = { id: rg_unique_id } - skuName: 'Basic' + skuName: 'Basic' capacity: 1 replicasPerMaster: 1 replicasPerPrimary: 1 @@ -281,17 +281,17 @@ module redis 'br/public:avm/res/cache/redis:0.3.2' = { /* -.__ __. _______ .___________.____ __ ____ ______ .______ __ ___ -| \ | | | ____|| |\ \ / \ / / / __ \ | _ \ | |/ / -| \| | | |__ `---| |----` \ \/ \/ / | | | | | |_) | | ' / -| . ` | | __| | | \ / | | | | | / | < -| |\ | | |____ | | \ /\ / | `--' | | |\ \----.| . \ -|__| \__| |_______| |__| \__/ \__/ \______/ | _| `._____||__|\__\ -.______ __ ___ _______ _______ +.__ __. _______ .___________.____ __ ____ ______ .______ __ ___ +| \ | | | ____|| |\ \ / \ / / / __ \ | _ \ | |/ / +| \| | | |__ `---| |----` \ \/ \/ / | | | | | |_) | | ' / +| . ` | | __| | | \ / | | | | | / | < +| |\ | | |____ | | \ /\ / | `--' | | |\ \----.| . \ +|__| \__| |_______| |__| \__/ \__/ \______/ | _| `._____||__|\__\ +.______ __ ___ _______ _______ | _ \ | | / \ | \ | ____| -| |_) | | | / ^ \ | .--. || |__ -| _ < | | / /_\ \ | | | || __| -| |_) | | `----./ _____ \ | '--' || |____ +| |_) | | | / ^ \ | .--. || |__ +| _ < | | / /_\ \ | | | || __| +| |_) | | `----./ _____ \ | '--' || |____ |______/ |_______/__/ \__\ |_______/ |_______| */ module networkBlade 'modules/blade_network.bicep' = if (enableVnetInjection) { @@ -314,7 +314,7 @@ module networkBlade 'modules/blade_network.bicep' = if (enableVnetInjection) { enablePodSubnet: vnetConfiguration.podSubnet.name != '' && vnetConfiguration.podSubnet.prefix != '' ? true: false enableVnetInjection: enableVnetInjection - + vnetConfiguration: { group: vnetConfiguration.group name: vnetConfiguration.name @@ -338,17 +338,17 @@ module networkBlade 'modules/blade_network.bicep' = if (enableVnetInjection) { /* - ______ __ __ __ _______.___________. _______ .______ - / || | | | | | / | || ____|| _ \ -| ,----'| | | | | | | (----`---| |----`| |__ | |_) | -| | | | | | | | \ \ | | | __| | / + ______ __ __ __ _______.___________. _______ .______ + / || | | | | | / | || ____|| _ \ +| ,----'| | | | | | | (----`---| |----`| |__ | |_) | +| | | | | | | | \ \ | | | __| | / | `----.| `----.| `--' | .----) | | | | |____ | |\ \----. \______||_______| \______/ |_______/ |__| |_______|| _| `._____| -.______ __ ___ _______ _______ +.______ __ ___ _______ _______ | _ \ | | / \ | \ | ____| -| |_) | | | / ^ \ | .--. || |__ -| _ < | | / /_\ \ | | | || __| -| |_) | | `----./ _____ \ | '--' || |____ +| |_) | | | / ^ \ | .--. || |__ +| _ < | | / /_\ \ | | | || __| +| |_) | | `----./ _____ \ | '--' || |____ |______/ |_______/__/ \__\ |_______/ |_______| */ module clusterBlade 'modules/blade_cluster.bicep' = { @@ -373,7 +373,7 @@ module clusterBlade 'modules/blade_cluster.bicep' = { workspaceResourceId: logAnalytics.outputs.resourceId identityId: enableVnetInjection ? networkBlade.outputs.networkConfiguration.identityId : stampIdentity.outputs.resourceId managedIdentityName: stampIdentity.outputs.name - + aksSubnetId: enableVnetInjection ? networkBlade.outputs.aksSubnetId : '' podSubnetId: enableVnetInjection ? networkBlade.outputs.podSubnetId : '' vmSize: customVMSize @@ -386,12 +386,12 @@ module clusterBlade 'modules/blade_cluster.bicep' = { /* - __________ ___ .___________. _______ .__ __. _______. __ ______ .__ __. -| ____\ \ / / | || ____|| \ | | / || | / __ \ | \ | | -| |__ \ V / `---| |----`| |__ | \| | | (----`| | | | | | | \| | -| __| > < | | | __| | . ` | \ \ | | | | | | | . ` | -| |____ / . \ | | | |____ | |\ | .----) | | | | `--' | | |\ | -|_______/__/ \__\ |__| |_______||__| \__| |_______/ |__| \______/ |__| \__| + __________ ___ .___________. _______ .__ __. _______. __ ______ .__ __. +| ____\ \ / / | || ____|| \ | | / || | / __ \ | \ | | +| |__ \ V / `---| |----`| |__ | \| | | (----`| | | | | | | \| | +| __| > < | | | __| | . ` | \ \ | | | | | | | . ` | +| |____ / . \ | | | |____ | |\ | .----) | | | | `--' | | |\ | +|_______/__/ \__\ |__| |_______||__| \__| |_______/ |__| \______/ |__| \__| */ // AVM doesn't support output of the principalId from the extension module so we have to use a deployment script to get it. // This takes a long time to deploy so we are starting as soon as possible. @@ -401,7 +401,7 @@ module fluxExtension 'modules/flux-extension/main.bicep' = { clusterName: clusterBlade.outputs.clusterName location: location extensionType: 'microsoft.flux' - name: 'flux' + name: 'flux' releaseNamespace: 'flux-system' releaseTrain: 'Stable' @@ -425,13 +425,13 @@ module fluxExtension 'modules/flux-extension/main.bicep' = { _______. ______ .______ __ .______ .___________. / | / || _ \ | | | _ \ | | | (----`| ,----'| |_) | | | | |_) | `---| |----` - \ \ | | | / | | | ___/ | | -.----) | | `----.| |\ \----.| | | | | | -|_______/ \______|| _| `._____||__| | _| |__| + \ \ | | | / | | | ___/ | | +.----) | | `----.| |\ \----.| | | | | | +|_______/ \______|| _| `._____||__| | _| |__| */ module extensionClientId 'br/public:avm/res/resources/deployment-script:0.4.0' = { name: '${configuration.name}-script-clientId' - + params: { kind: 'AzureCLI' name: 'script-${configuration.name}-aks-extension' @@ -453,7 +453,7 @@ module extensionClientId 'br/public:avm/res/resources/deployment-script:0.4.0' = value: fluxExtension.outputs.principalId } ] - + timeout: 'PT30M' retentionInterval: 'PT1H' @@ -462,7 +462,7 @@ module extensionClientId 'br/public:avm/res/resources/deployment-script:0.4.0' = echo "Looking up client ID for $principalId in ResourceGroup $rgName" clientId=$(az identity list --resource-group $rgName --query "[?principalId=='$principalId'] | [0].clientId" -otsv) - + echo "Found ClientId: $clientId" echo "{\"clientId\":\"$clientId\"}" | jq -c '.' > $AZ_SCRIPTS_OUTPUT_PATH ''' @@ -474,12 +474,12 @@ module extensionClientId 'br/public:avm/res/resources/deployment-script:0.4.0' = /* -.______ _______ _______ __ _______.___________..______ ____ ____ -| _ \ | ____| / _____|| | / | || _ \ \ \ / / -| |_) | | |__ | | __ | | | (----`---| |----`| |_) | \ \/ / -| / | __| | | |_ | | | \ \ | | | / \_ _/ -| |\ \----.| |____ | |__| | | | .----) | | | | |\ \----. | | -| _| `._____||_______| \______| |__| |_______/ |__| | _| `._____| |__| +.______ _______ _______ __ _______.___________..______ ____ ____ +| _ \ | ____| / _____|| | / | || _ \ \ \ / / +| |_) | | |__ | | __ | | | (----`---| |----`| |_) | \ \/ / +| / | __| | | |_ | | | \ \ | | | / \_ _/ +| |\ \----.| |____ | |__| | | | .----) | | | | |\ \----. | | +| _| `._____||_______| \______| |__| |_______/ |__| | _| `._____| |__| */ module registry 'br/public:avm/res/container-registry/registry:0.1.1' = { name: '${configuration.name}-container-registry' @@ -532,15 +532,15 @@ module registry 'br/public:avm/res/container-registry/registry:0.1.1' = { __ ___ ___________ ____ ____ ____ ___ __ __ __ .___________. | |/ / | ____\ \ / / \ \ / / / \ | | | | | | | | | ' / | |__ \ \/ / \ \/ / / ^ \ | | | | | | `---| |----` -| < | __| \_ _/ \ / / /_\ \ | | | | | | | | -| . \ | |____ | | \ / / _____ \ | `--' | | `----. | | -|__|\__\ |_______| |__| \__/ /__/ \__\ \______/ |_______| |__| +| < | __| \_ _/ \ / / /_\ \ | | | | | | | | +| . \ | |____ | | \ / / _____ \ | `--' | | `----. | | +|__|\__\ |_______| |__| \__/ /__/ \__\ \______/ |_______| |__| */ var name = '${replace(configuration.name, '-', '')}${uniqueString(resourceGroup().id, configuration.name)}' @description('The list of secrets to persist to the Key Vault') -var vaultSecrets = [ +var vaultSecrets = [ { secretName: 'tenant-id' secretValue: subscription().tenantId @@ -554,14 +554,15 @@ var vaultSecrets = [ secretValue: subscription().subscriptionId } // Azure AD Secrets - { - secretName: 'app-dev-sp-password' - secretValue: applicationClientSecret == '' ? 'dummy' : applicationClientSecret - } // { - // secretName: 'app-dev-sp-id' - // secretValue: applicationClientSecret == '' ? stampIdentity.outputs.clientId : applicationClientId + // secretName: 'app-dev-sp-password' + // secretValue: 'dummy' + // // secretValue: applicationClientSecret == '' ? 'dummy' : applicationClientSecret // } + { + secretName: 'app-dev-sp-id' + secretValue: applicationClientId + } { secretName: 'cpng-user-name' secretValue: 'dbuser' @@ -606,7 +607,7 @@ module keyvault 'br/public:avm/res/key-vault/vault:0.5.1' = { name: length(name) > 24 ? substring(name, 0, 24) : name location: location enableTelemetry: enableTelemetry - + // Assign Tags tags: { layer: configuration.displayName @@ -620,7 +621,7 @@ module keyvault 'br/public:avm/res/key-vault/vault:0.5.1' = { ] enablePurgeProtection: false - + // Configure RBAC enableRbacAuthorization: true roleAssignments: union( @@ -667,9 +668,9 @@ module keyvault 'br/public:avm/res/key-vault/vault:0.5.1' = { _______. _______ ______ .______ _______ .___________. _______. / || ____| / || _ \ | ____|| | / | | (----`| |__ | ,----'| |_) | | |__ `---| |----` | (----` - \ \ | __| | | | / | __| | | \ \ -.----) | | |____ | `----.| |\ \----.| |____ | | .----) | -|_______/ |_______| \______|| _| `._____||_______| |__| |_______/ + \ \ | __| | | | / | __| | | \ \ +.----) | | |____ | `----.| |\ \----.| |____ | | .----) | +|_______/ |_______| \______|| _| `._____||_______| |__| |_______/ */ // This custom module is used to persist insights, cache and workspace secrets to the Key Vault. module keyvaultSecrets 'modules/keyvault_secrets.bicep' = { @@ -707,12 +708,12 @@ var commonLayerConfig = { } -/* _______.___________. ______ .______ ___ _______ _______ +/* _______.___________. ______ .______ ___ _______ _______ / | | / __ \ | _ \ / \ / _____|| ____| - | (----`---| |----`| | | | | |_) | / ^ \ | | __ | |__ - \ \ | | | | | | | / / /_\ \ | | |_ | | __| -.----) | | | | `--' | | |\ \----./ _____ \ | |__| | | |____ -|_______/ |__| \______/ | _| `._____/__/ \__\ \______| |_______| + | (----`---| |----`| | | | | |_) | / ^ \ | | __ | |__ + \ \ | | | | | | | / / /_\ \ | | |_ | | __| +.----) | | | | `--' | | |\ \----./ _____ \ | |__| | | |____ +|_______/ |__| \______/ | _| `._____/__/ \__\ \______| |_______| */ // AVM Module Customized due to required Secrets. module storage 'modules/storage-account/main.bicep' = { @@ -727,7 +728,7 @@ module storage 'modules/storage-account/main.bicep' = { layer: configuration.displayName id: rg_unique_id } - + // Hook up Diagnostics diagnosticSettings: [ { @@ -780,7 +781,7 @@ module storage 'modules/storage-account/main.bicep' = { publicNetworkAccess: 'Enabled' // TODO: This is required for Partition Service to access the storage account. Issue: https://github.com/Azure/osdu-developer/issues/230 - allowSharedKeyAccess: true + allowSharedKeyAccess: true // https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template?tabs=CLI#debug-deployment-scripts networkAcls: { @@ -806,7 +807,7 @@ module storage 'modules/storage-account/main.bicep' = { ] connectionString1: [ 'system-storage-connection' - ] + ] blobEndpoint: [ 'system-storage-blob-endpoint' ] @@ -823,12 +824,12 @@ module storage 'modules/storage-account/main.bicep' = { /* - _______ .______ ___ .______ __ __ - / _____|| _ \ / \ | _ \ | | | | -| | __ | |_) | / ^ \ | |_) | | |__| | -| | |_ | | / / /_\ \ | ___/ | __ | -| |__| | | |\ \----./ _____ \ | | | | | | - \______| | _| `._____/__/ \__\ | _| |__| |__| + _______ .______ ___ .______ __ __ + / _____|| _ \ / \ | _ \ | | | | +| | __ | |_) | / ^ \ | |_) | | |__| | +| | |_ | | / / /_\ \ | ___/ | __ | +| |__| | | |\ \----./ _____ \ | | | | | | + \______| | _| `._____/__/ \__\ | _| |__| |__| */ // AVM Module Customized due to required Secrets. module database 'modules/cosmos-db/main.bicep' = { @@ -883,7 +884,7 @@ module database 'modules/cosmos-db/main.bicep' = { databaseEndpointSecretName: 'graph-db-endpoint' databasePrimaryKeySecretName: 'graph-db-primary-key' databaseConnectionStringSecretName: 'graph-db-connection' - + roleAssignments: [ { @@ -904,9 +905,9 @@ module database 'modules/cosmos-db/main.bicep' = { _______. ______ .______ __ .______ .___________. _______. / | / || _ \ | | | _ \ | | / | | (----`| ,----'| |_) | | | | |_) | `---| |----` | (----` - \ \ | | | / | | | ___/ | | \ \ -.----) | | `----.| |\ \----.| | | | | | .----) | -|_______/ \______|| _| `._____||__| | _| |__| |_______/ + \ \ | | | / | | | ___/ | | \ \ +.----) | | `----.| |\ \----.| | | | | | .----) | +|_______/ \______|| _| `._____||__| | _| |__| |_______/ */ @@ -933,16 +934,16 @@ module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for retentionInterval: 'PT1H' timeout: 'PT30M' runOnce: true - + managedIdentities: { userAssignedResourcesIds: [ stampIdentity.outputs.resourceId ] - } + } kind: 'AzureCLI' azCliVersion: '2.63.0' - + environmentVariables: [ { name: 'AZURE_STORAGE_ACCOUNT', value: storage.outputs.name } { name: 'FILE', value: 'main.zip' } @@ -970,17 +971,17 @@ module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for /* -.______ ___ .______ .___________. __ .___________. __ ______ .__ __. -| _ \ / \ | _ \ | || | | || | / __ \ | \ | | -| |_) | / ^ \ | |_) | `---| |----`| | `---| |----`| | | | | | | \| | -| ___/ / /_\ \ | / | | | | | | | | | | | | | . ` | -| | / _____ \ | |\ \----. | | | | | | | | | `--' | | |\ | -| _| /__/ \__\ | _| `._____| |__| |__| |__| |__| \______/ |__| \__| -.______ __ ___ _______ _______ +.______ ___ .______ .___________. __ .___________. __ ______ .__ __. +| _ \ / \ | _ \ | || | | || | / __ \ | \ | | +| |_) | / ^ \ | |_) | `---| |----`| | `---| |----`| | | | | | | \| | +| ___/ / /_\ \ | / | | | | | | | | | | | | | . ` | +| | / _____ \ | |\ \----. | | | | | | | | | `--' | | |\ | +| _| /__/ \__\ | _| `._____| |__| |__| |__| |__| \______/ |__| \__| +.______ __ ___ _______ _______ | _ \ | | / \ | \ | ____| -| |_) | | | / ^ \ | .--. || |__ -| _ < | | / /_\ \ | | | || __| -| |_) | | `----./ _____ \ | '--' || |____ +| |_) | | | / ^ \ | .--. || |__ +| _ < | | / /_\ \ | | | || __| +| |_) | | `----./ _____ \ | '--' || |____ |______/ |_______/__/ \__\ |_______/ |_______| */ module partitionBlade 'modules/blade_partition.bicep' = { @@ -999,7 +1000,7 @@ module partitionBlade 'modules/blade_partition.bicep' = { workspaceResourceId: logAnalytics.outputs.resourceId kvName: keyvault.outputs.name natClusterIP: clusterBlade.outputs.natClusterIP - + enableBlobPublicAccess: false partitions: configuration.partitions @@ -1019,17 +1020,17 @@ module partitionBlade 'modules/blade_partition.bicep' = { /* - ______ ______ .__ __. _______ __ _______ + ______ ______ .__ __. _______ __ _______ / | / __ \ | \ | | | ____|| | / _____| -| ,----'| | | | | \| | | |__ | | | | __ -| | | | | | | . ` | | __| | | | | |_ | -| `----.| `--' | | |\ | | | | | | |__| | - \______| \______/ |__| \__| |__| |__| \______| -.______ __ ___ _______ _______ +| ,----'| | | | | \| | | |__ | | | | __ +| | | | | | | . ` | | __| | | | | |_ | +| `----.| `--' | | |\ | | | | | | |__| | + \______| \______/ |__| \__| |__| |__| \______| +.______ __ ___ _______ _______ | _ \ | | / \ | \ | ____| -| |_) | | | / ^ \ | .--. || |__ -| _ < | | / /_\ \ | | | || __| -| |_) | | `----./ _____ \ | '--' || |____ +| |_) | | | / ^ \ | .--. || |__ +| _ < | | / /_\ \ | | | || __| +| |_) | | `----./ _____ \ | '--' || |____ |______/ |_______/__/ \__\ |_______/ |_______| */ module configBlade 'modules/blade_configuration.bicep' = { @@ -1065,7 +1066,7 @@ module configBlade 'modules/blade_configuration.bicep' = { appInsightsKey: insights.outputs.instrumentationKey partitionStorageNames: partitionBlade.outputs.partitionStorageNames partitionServiceBusNames: partitionBlade.outputs.partitionServiceBusNames - + clusterName: clusterBlade.outputs.clusterName oidcIssuerUrl: clusterBlade.outputs.oidcIssuerUrl clusterIngress: ingressType == '' ? 'External' : ingressType diff --git a/bicep/modules/blade_configuration.bicep b/bicep/modules/blade_configuration.bicep index 0f2cdc60..a13a5a09 100644 --- a/bicep/modules/blade_configuration.bicep +++ b/bicep/modules/blade_configuration.bicep @@ -113,12 +113,21 @@ resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = { name: kvName } -resource keySecret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { +resource keySecretSpUsername 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { name: 'app-dev-sp-username' parent: keyVault properties: { - value: applicationClientId + value: appIdentity.properties.clientId + } +} + +resource keySecretSpPassword 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { + name: 'app-dev-sp-password' + parent: keyVault + + properties: { + value: 'dummy' } } diff --git a/bicep/modules/keyvault_secrets.bicep b/bicep/modules/keyvault_secrets.bicep index 3c246add..d1b6fdb3 100644 --- a/bicep/modules/keyvault_secrets.bicep +++ b/bicep/modules/keyvault_secrets.bicep @@ -16,7 +16,7 @@ param cacheName string @description('The name of the identity.') @minLength(0) -param identityName string +param identityName string resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = { @@ -102,13 +102,13 @@ resource insightsConnection 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { } } -resource identityClientIdSecret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { - name: 'app-dev-sp-id' - parent: keyVault +// resource identityClientIdSecret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { +// name: 'app-dev-sp-id' +// parent: keyVault - properties: { - value: identity.properties.clientId - } -} +// properties: { +// value: identity.properties.clientId +// } +// } output keyVaultName string = keyVault.name diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index 95c81a3d..f4f385b0 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -209,6 +209,8 @@ data: echo " Logging in using Workload Identity" echo "==================================================================" + sleep 100000 + # Login using the federated token from the environment variable az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ --service-principal \ diff --git a/software/components/osdu-system/cache.yaml b/software/components/osdu-system/cache.yaml index f88b81ff..da3542e8 100644 --- a/software/components/osdu-system/cache.yaml +++ b/software/components/osdu-system/cache.yaml @@ -13,7 +13,6 @@ spec: name: root-ca-cluster-issuer kind: ClusterIssuer --- ---- apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: @@ -40,7 +39,7 @@ spec: valuesKey: values.yaml values: secrets: - - secretName: keyvault-secrets + - secretName: keyvault-secrets data: - key: redis-password vaultSecret: redis-password @@ -52,7 +51,7 @@ metadata: namespace: flux-system spec: interval: 10m - url: https://charts.bitnami.com/bitnami + url: oci://registry-1.docker.io/bitnamicharts --- apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease From 4c7f22783ddd47feb3442876356aa8fa2af65e45 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 4 Dec 2024 09:08:34 -0600 Subject: [PATCH 052/122] Moved to v1 api from source.toolkit.fluxcd.io/v1beta2. --- software/components/airflow/source.yaml | 4 ++-- software/components/certs/source.yaml | 2 +- software/components/osdu-system/cache.yaml | 3 ++- software/components/osdu-system/database.yaml | 4 ++-- software/components/osdu-system/mesh.yaml | 8 ++++---- software/components/osdu-system/reloader.yaml | 2 +- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/software/components/airflow/source.yaml b/software/components/airflow/source.yaml index 63507f10..79ff0fc6 100644 --- a/software/components/airflow/source.yaml +++ b/software/components/airflow/source.yaml @@ -1,5 +1,5 @@ --- -apiVersion: source.toolkit.fluxcd.io/v1beta2 +apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: name: airflow-official @@ -8,7 +8,7 @@ spec: interval: 5m url: https://airflow.apache.org --- -apiVersion: source.toolkit.fluxcd.io/v1beta2 +apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: name: airflow-community diff --git a/software/components/certs/source.yaml b/software/components/certs/source.yaml index ab088d12..f67eb4a9 100644 --- a/software/components/certs/source.yaml +++ b/software/components/certs/source.yaml @@ -1,5 +1,5 @@ --- -apiVersion: source.toolkit.fluxcd.io/v1beta2 +apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: name: cert-manager diff --git a/software/components/osdu-system/cache.yaml b/software/components/osdu-system/cache.yaml index da3542e8..d7cebf80 100644 --- a/software/components/osdu-system/cache.yaml +++ b/software/components/osdu-system/cache.yaml @@ -44,12 +44,13 @@ spec: - key: redis-password vaultSecret: redis-password --- -apiVersion: source.toolkit.fluxcd.io/v1beta2 +apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: name: redis namespace: flux-system spec: + type: oci interval: 10m url: oci://registry-1.docker.io/bitnamicharts --- diff --git a/software/components/osdu-system/database.yaml b/software/components/osdu-system/database.yaml index aadf54be..3debde4b 100644 --- a/software/components/osdu-system/database.yaml +++ b/software/components/osdu-system/database.yaml @@ -1,5 +1,5 @@ --- -apiVersion: source.toolkit.fluxcd.io/v1beta2 +apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: name: cnpg @@ -18,7 +18,7 @@ spec: releaseName: database-operator chart: spec: - chart: cloudnative-pg + chart: cloudnative-pg sourceRef: kind: HelmRepository name: cnpg diff --git a/software/components/osdu-system/mesh.yaml b/software/components/osdu-system/mesh.yaml index 9cf8e107..1e7d2caf 100644 --- a/software/components/osdu-system/mesh.yaml +++ b/software/components/osdu-system/mesh.yaml @@ -6,7 +6,7 @@ metadata: labels: toolkit.fluxcd.io/tenant: component --- -apiVersion: source.toolkit.fluxcd.io/v1beta2 +apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: name: istio @@ -15,7 +15,7 @@ spec: interval: 10m url: https://istio-release.storage.googleapis.com/charts --- -apiVersion: source.toolkit.fluxcd.io/v1beta2 +apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: name: jetstack @@ -181,7 +181,7 @@ spec: port: 443 protocol: TCP targetPort: 443 - annotations: + annotations: service.beta.kubernetes.io/azure-load-balancer-internal: 'true' --- apiVersion: helm.toolkit.fluxcd.io/v2beta1 @@ -212,7 +212,7 @@ spec: values: service: type: LoadBalancer - annotations: + annotations: service.beta.kubernetes.io/azure-load-balancer-internal: 'false' # service.beta.kubernetes.io/azure-dns-label-name: 'osdu-developer' ports: diff --git a/software/components/osdu-system/reloader.yaml b/software/components/osdu-system/reloader.yaml index 8ef01dfb..b25ee034 100644 --- a/software/components/osdu-system/reloader.yaml +++ b/software/components/osdu-system/reloader.yaml @@ -1,5 +1,5 @@ --- -apiVersion: source.toolkit.fluxcd.io/v1beta2 +apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: name: stakater From 1dd3f1a5a399c1a8c1c35727a67f11c2e19019df Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 4 Dec 2024 10:34:57 -0600 Subject: [PATCH 053/122] Changing partition job to different scope for managed identity. --- charts/osdu-developer-init/templates/partition-init.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index f4f385b0..3fa2dd3c 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -218,7 +218,7 @@ data: -t ${AZURE_TENANT_ID} # Get token with the correct application ID as resource - TOKEN=$(az account get-access-token --resource "api://${AZURE_AD_APPLICATION_ID}" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://management.azure.com/" --query accessToken -o tsv) OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://partition.{{ $namespace }}/api/partition/v1/partitions/${PARTITION} \ From bffc3978fb93b8aafdaf3998557012b43173eeda Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 4 Dec 2024 10:46:46 -0600 Subject: [PATCH 054/122] Updated job --- charts/osdu-developer-init/templates/partition-init.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index 3fa2dd3c..9a9acd85 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -209,8 +209,6 @@ data: echo " Logging in using Workload Identity" echo "==================================================================" - sleep 100000 - # Login using the federated token from the environment variable az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ --service-principal \ From 11d4c8c8db2aa724a6328b388cac9232f678bbd4 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 4 Dec 2024 11:23:35 -0600 Subject: [PATCH 055/122] Updated jobs --- software/applications/osdu-core/entitlements.yaml | 2 +- software/applications/osdu-core/partition.yaml | 4 ++-- software/applications/osdu-core/user-init.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index 10279907..8f0f26e8 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -138,7 +138,7 @@ spec: apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: - name: osdu-init-entitlements + name: osdu-entitlements-init namespace: osdu-core annotations: clusterconfig.azure.com/use-managed-source: "true" diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index 5fb90fa7..a731174b 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -104,11 +104,11 @@ spec: - name: PARTITION_SPRING_LOGGING_LEVEL value: "DEBUG" --- -# Retrigger: kubectl annotate helmrelease osdu-init-partition fluxcd.io/retrigger=$(date +%s) -n osdu-core +# Retrigger: kubectl annotate helmrelease osdu-partition-init fluxcd.io/retrigger=$(date +%s) -n osdu-core apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: - name: osdu-init-partition + name: osdu-partition-init namespace: osdu-core annotations: clusterconfig.azure.com/use-managed-source: "true" diff --git a/software/applications/osdu-core/user-init.yaml b/software/applications/osdu-core/user-init.yaml index 4b9adfb6..8536fa68 100644 --- a/software/applications/osdu-core/user-init.yaml +++ b/software/applications/osdu-core/user-init.yaml @@ -9,7 +9,7 @@ # clusterconfig.azure.com/use-managed-source: "true" # spec: # dependsOn: -# - name: osdu-init-entitlements +# - name: osdu-entitlements-init # namespace: osdu-core # targetNamespace: osdu-core # chart: @@ -28,7 +28,7 @@ # jobs: # partitionInit: false # entitlementInit: false -# userInit: true +# userInit: true # elasticInit: false # schemaInit: false # clientSecret: From 0046ca3450f69e350ab4bed8de11a6e5a46d86d1 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 4 Dec 2024 11:42:00 -0600 Subject: [PATCH 056/122] Updated job --- charts/osdu-developer-init/templates/entitlement-init.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/osdu-developer-init/templates/entitlement-init.yaml b/charts/osdu-developer-init/templates/entitlement-init.yaml index e25bcb73..a7774b88 100644 --- a/charts/osdu-developer-init/templates/entitlement-init.yaml +++ b/charts/osdu-developer-init/templates/entitlement-init.yaml @@ -72,6 +72,8 @@ data: # Get token (no resource needed) TOKEN=$(az account get-access-token --resource ${AZURE_AD_APPLICATION_ID} --query accessToken -o tsv) + sleep 100000 + OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://entitlements.{{ $namespace }}/api/entitlements/v2/tenant-provisioning \ --header "Host: entitlements.{{ $namespace }}" \ @@ -82,7 +84,7 @@ data: HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$') BODY=${OUTPUT%???} - + if [ "$HTTP_STATUS_CODE" == "200" ]; then echo "Success: $(echo "$BODY" | jq .)" else From 9b94e1d4059e55b3cee2734a20870072c8071fb5 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Wed, 4 Dec 2024 12:34:09 -0600 Subject: [PATCH 057/122] change to workload identity. --- .../templates/entitlement-init.yaml | 4 +- .../applications/osdu-core/entitlements.yaml | 30 +-- software/applications/osdu-core/legal.yaml | 240 +++++++++--------- .../applications/osdu-core/partition.yaml | 2 - 4 files changed, 129 insertions(+), 147 deletions(-) diff --git a/charts/osdu-developer-init/templates/entitlement-init.yaml b/charts/osdu-developer-init/templates/entitlement-init.yaml index a7774b88..17aab362 100644 --- a/charts/osdu-developer-init/templates/entitlement-init.yaml +++ b/charts/osdu-developer-init/templates/entitlement-init.yaml @@ -70,9 +70,7 @@ data: -t ${AZURE_TENANT_ID} # Get token (no resource needed) - TOKEN=$(az account get-access-token --resource ${AZURE_AD_APPLICATION_ID} --query accessToken -o tsv) - - sleep 100000 + TOKEN=$(az account get-access-token --resource https://management.azure.com/ --query accessToken -o tsv) OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://entitlements.{{ $namespace }}/api/entitlements/v2/tenant-provisioning \ diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index 8f0f26e8..b9d77dc9 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -27,13 +27,13 @@ spec: - kind: ConfigMap name: configmap-software valuesKey: value.yaml - - kind: ConfigMap - name: configmap-common-values - valuesKey: value.yaml # - kind: ConfigMap - # name: configmap-repo-override - # optional: true - # valuesKey: repository.yaml + # name: configmap-common-values + # valuesKey: value.yaml + - kind: ConfigMap + name: configmap-repo-override + optional: true + valuesKey: repository.yaml values: nameOverride: entitlements installationType: osduCore @@ -73,22 +73,6 @@ spec: - "/api/entitlements/v2/api-docs*" - "/api/entitlements/v2/webjars/*" env: - # - name: AZURE_TENANT_ID - # secret: - # name: active-directory - # key: tenant-id - # - name: AZURE_SUBSCRIPTION_ID - # secret: - # name: active-directory - # key: subscription-id - # - name: AZURE_CLIENT_ID - # secret: - # name: active-directory - # key: principal-clientid - # - name: AZURE_CLIENT_SECRET - # secret: - # name: active-directory - # key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources @@ -107,8 +91,6 @@ spec: key: insights-connection - name: AZURE_ISTIOAUTH_ENABLED value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED value: "true" - name: SERVER_SERVLET_CONTEXTPATH diff --git a/software/applications/osdu-core/legal.yaml b/software/applications/osdu-core/legal.yaml index 10bb8f8e..ef554df5 100644 --- a/software/applications/osdu-core/legal.yaml +++ b/software/applications/osdu-core/legal.yaml @@ -1,118 +1,122 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-legal -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-partition -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: legal -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: legal -# path: /api/legal/v1/ -# hosts: -# - "*" -# cors: -# - "http://localhost:8080" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/legal/legal- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/legal/v1/info" -# - "/api/legal/v1/swagger*" -# - "/api/legal/v1/api-docs*" -# - "/api/legal/v1/webjars/*" -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_PODIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_SERVLET_CONTEXTPATH -# value: "/api/legal/v1/" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: "legal" -# - name: SPRING_CONFIG_NAME -# value: "common,application" -# - name: LOG_PREFIX -# value: "legal" -# - name: AZURE_STORAGE_ENABLE_HTTPS -# value: "true" -# - name: COSMOSDB_DATABASE -# value: "osdu-db" -# - name: AZURE_STORAGE_CONTAINER_NAME -# value: "legal-service-azure-configuration" -# - name: LEGAL_SERVICE_REGION -# value: "us" -# - name: SERVICEBUS_TOPIC_NAME -# value: "legaltags" -# - name: REDIS_DATABASE -# value: "2" -# - name: PARTITION_SERVICE_ENDPOINT -# value: "http://partition/api/partition/v1" -# - name: ENTITLEMENTS_SERVICE_ENDPOINT -# value: "http://entitlements/api/entitlements/v2" -# - name: ENTITLEMENTS_SERVICE_API_KEY -# value: "OBSOLETE" \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-legal + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-partition + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-repo-override + optional: true + valuesKey: repository.yaml + values: + nameOverride: legal + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: legal + path: /api/legal/v1/ + hosts: + - "*" + cors: + - "http://localhost:8080" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/legal/legal- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/legal/v1/info" + - "/api/legal/v1/swagger*" + - "/api/legal/v1/api-docs*" + - "/api/legal/v1/webjars/*" + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_SERVLET_CONTEXTPATH + value: "/api/legal/v1/" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: "legal" + - name: SPRING_CONFIG_NAME + value: "common,application" + - name: LOG_PREFIX + value: "legal" + - name: AZURE_STORAGE_ENABLE_HTTPS + value: "true" + - name: COSMOSDB_DATABASE + value: "osdu-db" + - name: AZURE_STORAGE_CONTAINER_NAME + value: "legal-service-azure-configuration" + - name: LEGAL_SERVICE_REGION + value: "us" + - name: SERVICEBUS_TOPIC_NAME + value: "legaltags" + - name: REDIS_DATABASE + value: "2" + - name: PARTITION_SERVICE_ENDPOINT + value: "http://partition/api/partition/v1" + - name: ENTITLEMENTS_SERVICE_ENDPOINT + value: "http://entitlements/api/entitlements/v2" + - name: ENTITLEMENTS_SERVICE_API_KEY + value: "OBSOLETE" \ No newline at end of file diff --git a/software/applications/osdu-core/partition.yaml b/software/applications/osdu-core/partition.yaml index a731174b..2e9af5a0 100644 --- a/software/applications/osdu-core/partition.yaml +++ b/software/applications/osdu-core/partition.yaml @@ -87,8 +87,6 @@ spec: key: insights-connection - name: AZURE_ISTIOAUTH_ENABLED value: "true" - - name: AZURE_PAAS_PODIDENTITY_ISENABLED - value: "false" - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED value: "true" - name: SERVER_SERVLET_CONTEXTPATH From 9c75af06387787cc6497913d0aaf795db5c92737 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 5 Dec 2024 08:13:45 -0600 Subject: [PATCH 058/122] Added back user init. --- .../templates/user-init.yaml | 26 ++--- .../applications/osdu-core/user-init.yaml | 104 +++++++++--------- 2 files changed, 60 insertions(+), 70 deletions(-) diff --git a/charts/osdu-developer-init/templates/user-init.yaml b/charts/osdu-developer-init/templates/user-init.yaml index 23ff0474..cb76af4d 100644 --- a/charts/osdu-developer-init/templates/user-init.yaml +++ b/charts/osdu-developer-init/templates/user-init.yaml @@ -67,27 +67,17 @@ data: set -o nounset echo "==================================================================" - echo " Creating Bearer Token for Application: ${AZURE_CLIENT_ID} " + echo " Logging in using Workload Identity" echo "==================================================================" - OUTPUT=$(curl -s -k -w "%{http_code}" --request POST \ - --url https://login.microsoftonline.com/${AZURE_TENANT_ID}/oauth2/token \ - --header "content-type: application/x-www-form-urlencoded" \ - --data "grant_type=client_credentials" \ - --data "client_id=${AZURE_CLIENT_ID}" \ - --data "client_secret=${AZURE_CLIENT_SECRET}" \ - --data "resource=${AZURE_CLIENT_ID}") + # Login using the federated token from the environment variable + az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ + --service-principal \ + -u ${AZURE_CLIENT_ID} \ + -t ${AZURE_TENANT_ID} - HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$') - BODY=${OUTPUT%???} - - if [[ "$HTTP_STATUS_CODE" != "200" ]]; then - echo "Error: Unexpected HTTP status code $HTTP_STATUS_CODE" - echo "Response body: $BODY" - exit 1 - fi - - TOKEN=$(echo "$BODY" | jq .access_token | tr -d '"') + # Get token (no resource needed) + TOKEN=$(az account get-access-token --resource https://management.azure.com/ --query accessToken -o tsv) echo "==================================================================" echo " Adding the first user... " diff --git a/software/applications/osdu-core/user-init.yaml b/software/applications/osdu-core/user-init.yaml index 8536fa68..fde07ce7 100644 --- a/software/applications/osdu-core/user-init.yaml +++ b/software/applications/osdu-core/user-init.yaml @@ -1,52 +1,52 @@ -# --- -# # kubectl create job --namespace osdu-core --from=cronjob/user-init user-init-run-$(date +%s) -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-users -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-entitlements-init -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# installationType: osduCore -# jobs: -# partitionInit: false -# entitlementInit: false -# userInit: true -# elasticInit: false -# schemaInit: false -# clientSecret: -# name: active-directory -# key: principal-clientpassword -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: clientId -# valuesKey: client_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: tenantId -# valuesKey: tenant_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: emailAddress -# valuesKey: first_user_id \ No newline at end of file +--- +# kubectl create job --namespace osdu-core --from=cronjob/user-init user-init-run-$(date +%s) +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-users + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-entitlements-init + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + installationType: osduCore + jobs: + partitionInit: false + entitlementInit: false + userInit: true + elasticInit: false + schemaInit: false + clientSecret: + name: active-directory + key: principal-clientpassword + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: clientId + valuesKey: client_id + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id + - kind: ConfigMap + name: configmap-services + targetPath: emailAddress + valuesKey: first_user_id \ No newline at end of file From 3be5bca4f6e443c3863894ea6054ad01b1a0573e Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 5 Dec 2024 08:29:54 -0600 Subject: [PATCH 059/122] Added default scope to scripts. --- charts/osdu-developer-init/templates/entitlement-init.yaml | 2 +- charts/osdu-developer-init/templates/partition-init.yaml | 2 +- charts/osdu-developer-init/templates/user-init.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/osdu-developer-init/templates/entitlement-init.yaml b/charts/osdu-developer-init/templates/entitlement-init.yaml index 17aab362..67194de2 100644 --- a/charts/osdu-developer-init/templates/entitlement-init.yaml +++ b/charts/osdu-developer-init/templates/entitlement-init.yaml @@ -70,7 +70,7 @@ data: -t ${AZURE_TENANT_ID} # Get token (no resource needed) - TOKEN=$(az account get-access-token --resource https://management.azure.com/ --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://management.azure.com/.default" --query accessToken -o tsv) OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://entitlements.{{ $namespace }}/api/entitlements/v2/tenant-provisioning \ diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index 9a9acd85..2edd6450 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -216,7 +216,7 @@ data: -t ${AZURE_TENANT_ID} # Get token with the correct application ID as resource - TOKEN=$(az account get-access-token --resource "https://management.azure.com/" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://management.azure.com/.default" --query accessToken -o tsv) OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://partition.{{ $namespace }}/api/partition/v1/partitions/${PARTITION} \ diff --git a/charts/osdu-developer-init/templates/user-init.yaml b/charts/osdu-developer-init/templates/user-init.yaml index cb76af4d..8e7be507 100644 --- a/charts/osdu-developer-init/templates/user-init.yaml +++ b/charts/osdu-developer-init/templates/user-init.yaml @@ -77,7 +77,7 @@ data: -t ${AZURE_TENANT_ID} # Get token (no resource needed) - TOKEN=$(az account get-access-token --resource https://management.azure.com/ --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://management.azure.com/.default" --query accessToken -o tsv) echo "==================================================================" echo " Adding the first user... " From 1d92b9f5b940b3d04e166b446cf96efb88bfb4d8 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 5 Dec 2024 10:03:09 -0600 Subject: [PATCH 060/122] Removed ./default to scripts. --- charts/osdu-developer-init/templates/entitlement-init.yaml | 2 +- charts/osdu-developer-init/templates/partition-init.yaml | 2 +- charts/osdu-developer-init/templates/user-init.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/osdu-developer-init/templates/entitlement-init.yaml b/charts/osdu-developer-init/templates/entitlement-init.yaml index 67194de2..e0f83f38 100644 --- a/charts/osdu-developer-init/templates/entitlement-init.yaml +++ b/charts/osdu-developer-init/templates/entitlement-init.yaml @@ -70,7 +70,7 @@ data: -t ${AZURE_TENANT_ID} # Get token (no resource needed) - TOKEN=$(az account get-access-token --resource "https://management.azure.com/.default" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://management.azure.com/" --query accessToken -o tsv) OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://entitlements.{{ $namespace }}/api/entitlements/v2/tenant-provisioning \ diff --git a/charts/osdu-developer-init/templates/partition-init.yaml b/charts/osdu-developer-init/templates/partition-init.yaml index 2edd6450..9a9acd85 100644 --- a/charts/osdu-developer-init/templates/partition-init.yaml +++ b/charts/osdu-developer-init/templates/partition-init.yaml @@ -216,7 +216,7 @@ data: -t ${AZURE_TENANT_ID} # Get token with the correct application ID as resource - TOKEN=$(az account get-access-token --resource "https://management.azure.com/.default" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://management.azure.com/" --query accessToken -o tsv) OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://partition.{{ $namespace }}/api/partition/v1/partitions/${PARTITION} \ diff --git a/charts/osdu-developer-init/templates/user-init.yaml b/charts/osdu-developer-init/templates/user-init.yaml index 8e7be507..ae60139e 100644 --- a/charts/osdu-developer-init/templates/user-init.yaml +++ b/charts/osdu-developer-init/templates/user-init.yaml @@ -77,7 +77,7 @@ data: -t ${AZURE_TENANT_ID} # Get token (no resource needed) - TOKEN=$(az account get-access-token --resource "https://management.azure.com/.default" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://management.azure.com/" --query accessToken -o tsv) echo "==================================================================" echo " Adding the first user... " From fb2ad16248ef1721fe2f9afa2b91d84197388f7c Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 5 Dec 2024 10:16:12 -0600 Subject: [PATCH 061/122] Reworked user-init script. --- .../osdu-developer-init/templates/user-init.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/charts/osdu-developer-init/templates/user-init.yaml b/charts/osdu-developer-init/templates/user-init.yaml index ae60139e..08cbf676 100644 --- a/charts/osdu-developer-init/templates/user-init.yaml +++ b/charts/osdu-developer-init/templates/user-init.yaml @@ -9,8 +9,12 @@ metadata: namespace: osdu-core # Ensure the correct namespace spec: ttlSecondsAfterFinished: 120 + metadata: + labels: + azure.workload.identity/use: "true" template: spec: + serviceAccountName: workload-identity-sa volumes: - name: script configMap: @@ -18,13 +22,9 @@ spec: defaultMode: 0500 initContainers: - name: data-seed - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - command: ["/bin/sh"] - args: - - -c - - | - tdnf install -y curl jq && \ - /script/init.sh + image: mcr.microsoft.com/azure-cli:cbl-mariner2.0 + command: + - script/init.sh volumeMounts: - name: script mountPath: "/script" @@ -66,6 +66,8 @@ data: set -euo pipefail set -o nounset + tdnf install -y curl jq + echo "==================================================================" echo " Logging in using Workload Identity" echo "==================================================================" From cff3fde6a5a5d517c0d0348aec00ac351c71f553 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 5 Dec 2024 10:35:45 -0600 Subject: [PATCH 062/122] Reworked user-init script. --- charts/osdu-developer-init/templates/user-init.yaml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/charts/osdu-developer-init/templates/user-init.yaml b/charts/osdu-developer-init/templates/user-init.yaml index 08cbf676..896964a0 100644 --- a/charts/osdu-developer-init/templates/user-init.yaml +++ b/charts/osdu-developer-init/templates/user-init.yaml @@ -33,17 +33,12 @@ spec: value: "" # Placeholder value - name: AZURE_TENANT_ID value: {{ .Values.tenantId | quote }} - - name: AZURE_CLIENT_ID + - name: AZURE_AD_APPLICATION_ID value: {{ .Values.clientId | quote }} - - name: EMAIL_ADDRESS - value: {{ .Values.emailAddress | quote }} - - name: AZURE_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.clientSecret.name | quote }} - key: {{ .Values.clientSecret.key | quote }} - name: PARTITION value: {{ .Values.partition | quote }} + - name: EMAIL_ADDRESS + value: {{ .Values.emailAddress | quote }} - name: AUTH_INGRESS value: {{ .Values.authIngress | quote }} containers: @@ -81,6 +76,7 @@ data: # Get token (no resource needed) TOKEN=$(az account get-access-token --resource "https://management.azure.com/" --query accessToken -o tsv) + sleep 100000 echo "==================================================================" echo " Adding the first user... " echo "==================================================================" From f4d0736e58d78124b5148ef07fd221cef72a0adb Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 5 Dec 2024 10:49:16 -0600 Subject: [PATCH 063/122] Reworked user-init script. --- charts/osdu-developer-init/templates/user-init.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/charts/osdu-developer-init/templates/user-init.yaml b/charts/osdu-developer-init/templates/user-init.yaml index 896964a0..6d8e668e 100644 --- a/charts/osdu-developer-init/templates/user-init.yaml +++ b/charts/osdu-developer-init/templates/user-init.yaml @@ -6,13 +6,13 @@ apiVersion: batch/v1 kind: Job metadata: name: user-init - namespace: osdu-core # Ensure the correct namespace + namespace: osdu-core spec: ttlSecondsAfterFinished: 120 - metadata: + template: + metadata: labels: azure.workload.identity/use: "true" - template: spec: serviceAccountName: workload-identity-sa volumes: @@ -45,7 +45,7 @@ spec: - name: sleep image: istio/base command: ["/bin/sleep", "30"] - volumeMounts: # Ensure this container also mounts the volume if needed + volumeMounts: - name: script mountPath: "/script" restartPolicy: Never @@ -54,7 +54,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: user-init-script - namespace: osdu-core # Ensure the correct namespace + namespace: osdu-core data: init.sh: | #!/usr/bin/env sh @@ -67,6 +67,8 @@ data: echo " Logging in using Workload Identity" echo "==================================================================" + sleep 100000 + # Login using the federated token from the environment variable az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ --service-principal \ @@ -76,7 +78,6 @@ data: # Get token (no resource needed) TOKEN=$(az account get-access-token --resource "https://management.azure.com/" --query accessToken -o tsv) - sleep 100000 echo "==================================================================" echo " Adding the first user... " echo "==================================================================" From a30270f1bf066ccd0034219288d878a91f2b5e85 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 5 Dec 2024 11:01:10 -0600 Subject: [PATCH 064/122] Reworked user-init script. --- charts/osdu-developer-init/templates/user-init.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/charts/osdu-developer-init/templates/user-init.yaml b/charts/osdu-developer-init/templates/user-init.yaml index 6d8e668e..7509179a 100644 --- a/charts/osdu-developer-init/templates/user-init.yaml +++ b/charts/osdu-developer-init/templates/user-init.yaml @@ -29,8 +29,6 @@ spec: - name: script mountPath: "/script" env: - - name: AUTH_CODE - value: "" # Placeholder value - name: AZURE_TENANT_ID value: {{ .Values.tenantId | quote }} - name: AZURE_AD_APPLICATION_ID @@ -67,8 +65,6 @@ data: echo " Logging in using Workload Identity" echo "==================================================================" - sleep 100000 - # Login using the federated token from the environment variable az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ --service-principal \ @@ -82,8 +78,7 @@ data: echo " Adding the first user... " echo "==================================================================" - AUTH_USER="${EMAIL_ADDRESS}" - json_payload=$(jq -n --arg email "$AUTH_USER" '{"email": $email, "role": "MEMBER"}') + json_payload=$(jq -n --arg email "$EMAIL_ADDRESS" '{"email": $email, "role": "MEMBER"}') OUTPUT=$(curl -s -k -w "%{http_code}" -X POST "http://entitlements.{{ $namespace }}/api/entitlements/v2/groups/users@opendes.dataservices.energy/members" \ --insecure \ From d620b9caa815044784c77b5e278060d7c77a44a6 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 5 Dec 2024 11:55:46 -0600 Subject: [PATCH 065/122] Added sp user capability. --- .../applications/osdu-core/user-init.yaml | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/software/applications/osdu-core/user-init.yaml b/software/applications/osdu-core/user-init.yaml index fde07ce7..79931c76 100644 --- a/software/applications/osdu-core/user-init.yaml +++ b/software/applications/osdu-core/user-init.yaml @@ -3,7 +3,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: - name: osdu-init-users + name: osdu-init-user namespace: osdu-core annotations: clusterconfig.azure.com/use-managed-source: "true" @@ -49,4 +49,47 @@ spec: - kind: ConfigMap name: configmap-services targetPath: emailAddress - valuesKey: first_user_id \ No newline at end of file + valuesKey: first_user_id +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-user-sp + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-entitlements-init + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + installationType: osduCore + jobs: + partitionInit: false + entitlementInit: false + userInit: true + elasticInit: false + schemaInit: false + clientSecret: + name: active-directory + key: principal-clientpassword + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: emailAddress + valuesKey: client_id \ No newline at end of file From 53f1ea601091fb05ebd24dd8037882fd34add0f6 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 5 Dec 2024 12:09:41 -0600 Subject: [PATCH 066/122] Added sp user capability. --- software/applications/osdu-core/user-init.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/software/applications/osdu-core/user-init.yaml b/software/applications/osdu-core/user-init.yaml index 79931c76..fbd71bd9 100644 --- a/software/applications/osdu-core/user-init.yaml +++ b/software/applications/osdu-core/user-init.yaml @@ -89,6 +89,14 @@ spec: - kind: ConfigMap name: configmap-software valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: clientId + valuesKey: client_id + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id - kind: ConfigMap name: configmap-services targetPath: emailAddress From ca303e2644cf641ae97b6c2ee7ef147efc83ae44 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 5 Dec 2024 12:16:37 -0600 Subject: [PATCH 067/122] Added sp user capability. --- charts/osdu-developer-init/templates/user-init.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/osdu-developer-init/templates/user-init.yaml b/charts/osdu-developer-init/templates/user-init.yaml index 7509179a..4db834ee 100644 --- a/charts/osdu-developer-init/templates/user-init.yaml +++ b/charts/osdu-developer-init/templates/user-init.yaml @@ -5,8 +5,8 @@ apiVersion: batch/v1 kind: Job metadata: - name: user-init - namespace: osdu-core + name: {{ .Release.Name }} + namespace: {{ $namespace }} spec: ttlSecondsAfterFinished: 120 template: From c184c2f1390c5618ed60ed301dd995e59668a294 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Thu, 5 Dec 2024 12:41:19 -0600 Subject: [PATCH 068/122] Added sp user capability. --- .../templates/user-init.yaml | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/charts/osdu-developer-init/templates/user-init.yaml b/charts/osdu-developer-init/templates/user-init.yaml index 4db834ee..67296088 100644 --- a/charts/osdu-developer-init/templates/user-init.yaml +++ b/charts/osdu-developer-init/templates/user-init.yaml @@ -1,11 +1,12 @@ {{- $enabled := eq (include "osdu-developer-init.isEnabled" .) "1" -}} {{- $namespace := .Release.Namespace -}} -{{- if and $enabled .Values.jobs.userInit }} +{{- if $enabled }} +{{- range $job := .Values.jobs.list }} --- apiVersion: batch/v1 kind: Job metadata: - name: {{ .Release.Name }} + name: {{ .Release.Name }}-{{ $job.name }} namespace: {{ $namespace }} spec: ttlSecondsAfterFinished: 120 @@ -14,31 +15,25 @@ spec: labels: azure.workload.identity/use: "true" spec: - serviceAccountName: workload-identity-sa + serviceAccountName: {{ $job.serviceAccountName }} volumes: - name: script configMap: - name: user-init-script + name: user-init-script-{{ $job.name }} defaultMode: 0500 initContainers: - name: data-seed - image: mcr.microsoft.com/azure-cli:cbl-mariner2.0 + image: {{ $job.initContainerImage }} command: - - script/init.sh + - /script/init.sh volumeMounts: - name: script mountPath: "/script" env: - - name: AZURE_TENANT_ID - value: {{ .Values.tenantId | quote }} - - name: AZURE_AD_APPLICATION_ID - value: {{ .Values.clientId | quote }} - - name: PARTITION - value: {{ .Values.partition | quote }} - - name: EMAIL_ADDRESS - value: {{ .Values.emailAddress | quote }} - - name: AUTH_INGRESS - value: {{ .Values.authIngress | quote }} + {{- range $env := $job.env }} + - name: {{ $env.name }} + value: {{ $env.value | quote }} + {{- end }} containers: - name: sleep image: istio/base @@ -51,8 +46,8 @@ spec: apiVersion: v1 kind: ConfigMap metadata: - name: user-init-script - namespace: osdu-core + name: user-init-script-{{ $job.name }} + namespace: {{ $namespace }} data: init.sh: | #!/usr/bin/env sh @@ -132,3 +127,4 @@ data: exit 0 {{- end }} +{{- end }} \ No newline at end of file From 1005ff71d7d4c857fce19c4ba18012bf8fe8b087 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sat, 21 Dec 2024 11:50:16 -0600 Subject: [PATCH 069/122] Added indexer-queue. --- scripts/settings.ps1 | 6 +++--- scripts/template.yaml | 39 +++++++++++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/scripts/settings.ps1 b/scripts/settings.ps1 index 42923f10..422494d1 100644 --- a/scripts/settings.ps1 +++ b/scripts/settings.ps1 @@ -275,7 +275,7 @@ function New-YamlFile { $currentLevel = 0 $nodePath = @($nodeName) $osduGroupNode = $nodeName - + # Create output directory for the new OSDU group node $outputDirectory = "./src/$osduGroupNode".ToLower() New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null @@ -399,7 +399,7 @@ function New-ServiceEnvFile { $currentLevel = 0 $nodePath = @($nodeName) $osduGroupNode = $nodeName - + $outputDirectory = "./src/$osduGroupNode".ToLower() New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null @@ -465,7 +465,7 @@ function Get-AppInsights { Write-Host "Downloading Application Insights Agent" Write-Host "==================================================================" - $url = "https://github.com/microsoft/ApplicationInsights-Java/releases/download/3.5.4/applicationinsights-agent-3.5.4.jar" + $url = "https://github.com/microsoft/ApplicationInsights-Java/releases/download/3.6.2/applicationinsights-agent-3.6.2.jar" $outputPath = "./src/applicationinsights-agent.jar" try { diff --git a/scripts/template.yaml b/scripts/template.yaml index 8ec9f2c5..34414cd5 100644 --- a/scripts/template.yaml +++ b/scripts/template.yaml @@ -76,7 +76,7 @@ CORE: AZURE_LEGAL_STORAGE_ACCOUNT: "%DATA_NAME%" AZURE_LEGAL_STORAGE_KEY: "%DATA_STORAGE_KEY%" AZURE_LEGAL_SERVICEBUS: "%DATA_SERVICEBUS_CONNECTION%" - SCHEMA: + SCHEMA-SERVICE: RUN: APPINSIGHTS_KEY: "%INSTRUMENTATION_KEY%" KEYVAULT_URI: "https://%COMMON_NAME%.vault.azure.net" @@ -124,7 +124,7 @@ CORE: AZURE_PAAS_PODIDENTITY: "false" AZURE_ISTIOAUTH_ENABLED: "true" AZURE_PAAS_PODIDENTITY_ISENABLED: "false" - INDEXER: + INDEXER-SERVICE: RUN: APPINSIGHTS_KEY: "%INSTRUMENTATION_KEY%" KEYVAULT_URI: "https://%COMMON_NAME%.vault.azure.net" @@ -150,7 +150,7 @@ CORE: AZURE_PAAS_PODIDENTITY: "false" AZURE_ISTIOAUTH_ENABLED: "true" AZURE_PAAS_PODIDENTITY_ISENABLED: "false" - SEARCH: + SEARCH-SERVICE: RUN: APPINSIGHTS_KEY: "%INSTRUMENTATION_KEY%" APPLICATIONINSIGHTS_CONNECTION_STRING: "%APPLICATIONINSIGHTS_CONNECTION_STRING%" @@ -221,4 +221,35 @@ CORE: OSDU_AIRFLOW_VERSION2_ENABLED: true DP_AIRFLOW_FOR_SYSTEM_DAG: false IGNORE_DAGCONTENT: true - IGNORE_CUSTOMOPERATORCONTENT: true \ No newline at end of file + IGNORE_CUSTOMOPERATORCONTENT: true + INDEXER-QUEUE: + RUN: + SPRING_APPLICATION_NAME: "indexer-queue" + SERVER_PORT: "8080" + AZURE_SERVICEBUS_TOPIC_NAME: "recordstopic" + AZURE_SERVICEBUS_TOPIC_SUBSCRIPTION: "recordstopicsubscription" + AZURE_REINDEX_TOPIC_NAME: "reindextopic" + AZURE_REINDEX_TOPIC_SUBSCRIPTION: "reindextopicsubscription" + AZURE_SCHEMACHANGED_TOPIC_NAME: "schemachangedtopiceg" + AZURE_SCHEMACHANGED_TOPIC_SUBSCRIPTION: "schemachangedtopicsubscription" + INDEXER_WORKER_URL: "http://indexer/api/indexer/v2/_dps/task-handlers/index-worker" + SCHEMA_WORKER_URL: "http://indexer/api/indexer/v2/_dps/task-handlers/schema-worker" + PARTITION_API: "http://%AUTH_INGRESS%/api/partition/v1" + MAX_CONCURRENT_CALLS: "32" + EXECUTOR_N_THREADS: "32" + MAX_LOCK_RENEW_DURATION_SECONDS: "600" + MAX_DELIVERY_COUNT: "5" + AZURE_PAAS_PODIDENTITY_ISENABLED: "false" + KEYVAULT_URI: "https://%COMMON_NAME%.vault.azure.net" + AAD_CLIENT_ID: "%AZURE_CLIENT_ID%" + AZURE_APP_RESOURCE_ID: "%AZURE_CLIENT_ID%" + AZURE_CLIENT_ID: "%AZURE_CLIENT_ID%" + AZURE_TENANT_ID: "%AZURE_TENANT_ID%" + APPLICATIONINSIGHTS_ROLE_NAME: "indexer-queue" + TEST: + AZURE_AD_TENANT_ID: "%AZURE_TENANT_ID%" + INTEGRATION_TESTER: "%AZURE_CLIENT_ID%" + AZURE_TESTER_SERVICEPRINCIPAL_SECRET: "%AZURE_CLIENT_SECRET%" + AZURE_AD_APP_RESOURCE_ID: "%AZURE_CLIENT_ID%" + INDEXER_QUEUE_BASE_URL: "http://%AUTH_INGRESS%/api/indexer-queue/v1/" + ENVIRONMENT: "CLOUD" \ No newline at end of file From 439e474f727da84dd4645c8e2a47039353132d3f Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 09:25:11 -0600 Subject: [PATCH 070/122] Added legal coo upload back. --- software/applications/osdu-core/base.yaml | 72 +++++++++++------------ 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 0741db96..8466b94a 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -33,39 +33,39 @@ spec: defaultCpuLimits: "2" defaultMemoryLimits: "4Gi" --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: blob-upload -# namespace: default -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-developer-base-core -# namespace: default -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/blob-upload -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: config-map-values -# valuesKey: values.yaml -# values: -# global: -# configmapNamespace: osdu-core -# blobUpload: -# enabled: true -# items: -# - name: legal -# file: "Legal_COO.json" -# url: "https://raw.githubusercontent.com/Azure/osdu-developer/refs/heads/main/bicep/modules/script-blob-upload/Legal_COO.json" +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: blob-upload + namespace: default + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-developer-base-core + namespace: default + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/blob-upload + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: config-map-values + valuesKey: values.yaml + values: + global: + configmapNamespace: osdu-core + blobUpload: + enabled: true + items: + - name: legal + file: "Legal_COO.json" + url: "https://raw.githubusercontent.com/Azure/osdu-developer/refs/heads/main/bicep/modules/script-blob-upload/Legal_COO.json" From f80b941258e6cdedf056fea6b808d87a60bdc6e1 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 09:57:39 -0600 Subject: [PATCH 071/122] Updated blob-upload chart to use workload identity. --- .../templates/storage-container-job.yaml | 21 ++++-- software/applications/osdu-core/base.yaml | 74 +++++++++---------- 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/charts/blob-upload/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml index c2f0e658..d3632fab 100644 --- a/charts/blob-upload/templates/storage-container-job.yaml +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -23,14 +23,21 @@ spec: - | # Install curl tdnf install -y curl - + # Download the file echo "Downloading file from {{ .url }}" curl -kso {{ .file }} "{{ .url }}" - - # Login using workload identity - az login --identity - + + echo "==================================================================" + echo " Logging in using Workload Identity" + echo "==================================================================" + + # Login using the federated token from the environment variable + az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ + --service-principal \ + -u ${AZURE_CLIENT_ID} \ + -t ${AZURE_TENANT_ID} + # Upload directly to blob storage using Azure CLI az storage blob upload \ -f {{ .file }} \ @@ -38,7 +45,7 @@ spec: -n {{ .file }} \ --overwrite \ --auth-mode login - + echo "File uploaded to container {{ $.Values.blobUpload.container }} in storage account {{ $value }}" sleep 300000 restartPolicy: Never @@ -46,4 +53,4 @@ spec: {{- $i = add $i 1 }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} \ No newline at end of file diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 8466b94a..495ee98d 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -32,40 +32,40 @@ spec: defaultMemoryRequests: "1Gi" defaultCpuLimits: "2" defaultMemoryLimits: "4Gi" ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: blob-upload - namespace: default - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-developer-base-core - namespace: default - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/blob-upload - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: config-map-values - valuesKey: values.yaml - values: - global: - configmapNamespace: osdu-core - blobUpload: - enabled: true - items: - - name: legal - file: "Legal_COO.json" - url: "https://raw.githubusercontent.com/Azure/osdu-developer/refs/heads/main/bicep/modules/script-blob-upload/Legal_COO.json" +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: blob-upload +# namespace: default +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-developer-base-core +# namespace: default +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/blob-upload +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: config-map-values +# valuesKey: values.yaml +# values: +# global: +# configmapNamespace: osdu-core +# blobUpload: +# enabled: true +# items: +# - name: legal +# file: "Legal_COO.json" +# url: "https://raw.githubusercontent.com/Azure/osdu-developer/refs/heads/main/bicep/modules/script-blob-upload/Legal_COO.json" From 7fcd4e74f5a7e78673309e968d4f7cfdbfb68c16 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 10:33:11 -0600 Subject: [PATCH 072/122] Updated blob-upload chart to use workload identity. --- software/applications/osdu-core/base.yaml | 74 +++++++++++------------ 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 495ee98d..8466b94a 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -32,40 +32,40 @@ spec: defaultMemoryRequests: "1Gi" defaultCpuLimits: "2" defaultMemoryLimits: "4Gi" -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: blob-upload -# namespace: default -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-developer-base-core -# namespace: default -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/blob-upload -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: config-map-values -# valuesKey: values.yaml -# values: -# global: -# configmapNamespace: osdu-core -# blobUpload: -# enabled: true -# items: -# - name: legal -# file: "Legal_COO.json" -# url: "https://raw.githubusercontent.com/Azure/osdu-developer/refs/heads/main/bicep/modules/script-blob-upload/Legal_COO.json" +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: blob-upload + namespace: default + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-developer-base-core + namespace: default + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/blob-upload + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: config-map-values + valuesKey: values.yaml + values: + global: + configmapNamespace: osdu-core + blobUpload: + enabled: true + items: + - name: legal + file: "Legal_COO.json" + url: "https://raw.githubusercontent.com/Azure/osdu-developer/refs/heads/main/bicep/modules/script-blob-upload/Legal_COO.json" From 1f853d45589fbb4997e596c1d928fa28199bec32 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 12:00:04 -0600 Subject: [PATCH 073/122] Updated blob-upload chart to use workload identity. --- software/applications/osdu-core/base.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 8466b94a..6305b883 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -60,6 +60,14 @@ spec: - kind: ConfigMap name: config-map-values valuesKey: values.yaml + - kind: Secret + name: active-directory + targetPath: clientId + valuesKey: msi-clientid + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id values: global: configmapNamespace: osdu-core From 1a3678038cebcd0296ce9018010cef801086c961 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 12:10:15 -0600 Subject: [PATCH 074/122] Updated blob-upload chart to use workload identity. --- charts/blob-upload/Chart.yaml | 2 +- charts/blob-upload/templates/storage-container-job.yaml | 4 ++-- software/applications/osdu-core/base.yaml | 8 -------- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/charts/blob-upload/Chart.yaml b/charts/blob-upload/Chart.yaml index 76c69e0c..9df11430 100644 --- a/charts/blob-upload/Chart.yaml +++ b/charts/blob-upload/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: blob-upload type: application description: Uploads files to Azure Blob Storage -version: 0.0.2 +version: 0.0.3 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/blob-upload/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml index d3632fab..3ca8d844 100644 --- a/charts/blob-upload/templates/storage-container-job.yaml +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -35,8 +35,8 @@ spec: # Login using the federated token from the environment variable az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ --service-principal \ - -u ${AZURE_CLIENT_ID} \ - -t ${AZURE_TENANT_ID} + -u {{ $.Values.azure.clientId }} \ + -t {{ $.Values.azure.tenantId }} # Upload directly to blob storage using Azure CLI az storage blob upload \ diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 6305b883..8466b94a 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -60,14 +60,6 @@ spec: - kind: ConfigMap name: config-map-values valuesKey: values.yaml - - kind: Secret - name: active-directory - targetPath: clientId - valuesKey: msi-clientid - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id values: global: configmapNamespace: osdu-core From 8d32cb82293f23b128bf69de721aade0c3ff457b Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 12:41:24 -0600 Subject: [PATCH 075/122] Updated blob-upload chart to use workload identity. --- charts/blob-upload/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/blob-upload/Chart.yaml b/charts/blob-upload/Chart.yaml index 9df11430..28fbe399 100644 --- a/charts/blob-upload/Chart.yaml +++ b/charts/blob-upload/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: blob-upload type: application description: Uploads files to Azure Blob Storage -version: 0.0.3 +version: 0.0.1 appVersion: 0.0.1 maintainers: - name: danielscholl From e9b361bcd71fb96625bc90f23de3f6b3c5bcf0ea Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 12:51:28 -0600 Subject: [PATCH 076/122] Updated blob-upload chart to use workload identity. --- charts/blob-upload/templates/storage-container-job.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/blob-upload/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml index 3ca8d844..bc6a0f22 100644 --- a/charts/blob-upload/templates/storage-container-job.yaml +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -12,6 +12,9 @@ metadata: spec: ttlSecondsAfterFinished: 300 template: + metadata: + labels: + azure.workload.identity/use: "true" spec: serviceAccountName: workload-identity-sa containers: From 74057e04cb7d2a34897aa571bfdf95ae87b86c7f Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 13:13:31 -0600 Subject: [PATCH 077/122] Updated blob-upload chart to use workload identity. --- charts/blob-upload/templates/storage-container-job.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/charts/blob-upload/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml index bc6a0f22..4dbe976c 100644 --- a/charts/blob-upload/templates/storage-container-job.yaml +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -37,15 +37,16 @@ spec: # Login using the federated token from the environment variable az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ - --service-principal \ - -u {{ $.Values.azure.clientId }} \ - -t {{ $.Values.azure.tenantId }} + --service-principal \ + -u ${AZURE_CLIENT_ID} \ + -t ${AZURE_TENANT_ID} # Upload directly to blob storage using Azure CLI az storage blob upload \ -f {{ .file }} \ -c {{ $.Values.blobUpload.container }} \ -n {{ .file }} \ + --account-name {{ $value }} \ --overwrite \ --auth-mode login From a78d0219f852cf61c0e7cc9b4dc49dbf27ed2d7e Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 13:27:58 -0600 Subject: [PATCH 078/122] Updated blob-upload chart to use workload identity. --- charts/blob-upload/templates/storage-container-job.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/blob-upload/templates/storage-container-job.yaml b/charts/blob-upload/templates/storage-container-job.yaml index 4dbe976c..78340d04 100644 --- a/charts/blob-upload/templates/storage-container-job.yaml +++ b/charts/blob-upload/templates/storage-container-job.yaml @@ -39,7 +39,7 @@ spec: az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ --service-principal \ -u ${AZURE_CLIENT_ID} \ - -t ${AZURE_TENANT_ID} + -t ${AZURE_TENANT_ID} || exit 1 # Upload directly to blob storage using Azure CLI az storage blob upload \ @@ -48,10 +48,10 @@ spec: -n {{ .file }} \ --account-name {{ $value }} \ --overwrite \ - --auth-mode login + --auth-mode login || exit 1 echo "File uploaded to container {{ $.Values.blobUpload.container }} in storage account {{ $value }}" - sleep 300000 + exit 0 restartPolicy: Never {{- end }} {{- $i = add $i 1 }} From 3d218e29a4ee0e4b5c9b215cd83fc42438857501 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 13:37:14 -0600 Subject: [PATCH 079/122] Added indexer and schema. --- software/applications/osdu-core/indexer.yaml | 476 +++++++++---------- software/applications/osdu-core/schema.yaml | 334 ++++++------- 2 files changed, 405 insertions(+), 405 deletions(-) diff --git a/software/applications/osdu-core/indexer.yaml b/software/applications/osdu-core/indexer.yaml index 50841b2e..978cfd20 100644 --- a/software/applications/osdu-core/indexer.yaml +++ b/software/applications/osdu-core/indexer.yaml @@ -1,238 +1,238 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-indexer-service -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-legal -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: indexer -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: indexer -# path: /api/indexer/v2/ -# hosts: -# - "*" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/system/indexer-service/indexer-service- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/indexer/v2/info" -# - /api/indexer/v2/swagger* -# - /api/indexer/v2/api-docs* -# - "/api/indexer/v2/webjars/*" -# - '*/index-worker' -# - '*/_dps/task-handlers' -# - '*/reindex' -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_PODIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SECURITY_HTTPS_CERTIFICATE_TRUST -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: indexer -# - name: SERVER_SERVLET_CONTEXTPATH -# value: /api/indexer/v2/ -# - name: COSMOSDB_DATABASE -# value: osdu-db -# - name: REDIS_DATABASE -# value: "4" -# - name: REDIS_TTL_SECONDS -# value: "3600" -# - name: SERVICEBUS_TOPIC_NAME -# value: indexing-progress -# - name: REINDEX_TOPIC_NAME -# value: recordstopic -# - name: PARTITION_SERVICE_ENDPOINT -# value: http://partition/api/partition/v1 -# - name: ENTITLEMENTS_SERVICE_ENDPOINT -# value: http://entitlements/api/entitlements/v2 -# - name: ENTITLEMENTS_SERVICE_API_KEY -# value: "OBSOLETE" -# - name: SCHEMA_SERVICE_URL -# value: http://schema/api/schema-service/v1 -# - name: STORAGE_SERVICE_URL -# value: http://storage/api/storage/v2 -# - name: STORAGE_SCHEMA_HOST -# value: http://storage/api/storage/v2/schemas -# - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST -# value: http://storage/api/storage/v2/query/records:batch -# - name: STORAGE_QUERY_RECORD_HOST -# value: http://storage/api/storage/v2/query/records -# - name: SEARCH_SERVICE_URL -# value: http://search/api/search/v2 -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-indexer-queue -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-legal -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: indexer-queue -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: indexer-queue -# repository: community.opengroup.org:5555/osdu/platform/system/indexer-queue/indexer-queue- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# env: -# - name: AZURE_TENANT_ID -# secret: -# name: active-directory -# key: tenant-id -# - name: AZURE_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: AZURE_CLIENT_SECRET -# secret: -# name: active-directory -# key: principal-clientpassword -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: AZURE_APP_RESOURCE_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: AZURE_APPLICATION_INSIGHTS_INSTRUMENTATION_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_PODIDENTITY_ISENABLED -# value: "false" -# - name: SERVER_PORT -# value: "80" -# - name: SPRING_APPLICATION_NAME -# value: indexer-queue -# - name: AZURE_SERVICEBUS_TOPIC_NAME -# value: recordstopic -# - name: AZURE_REINDEX_TOPIC_NAME -# value: reindextopic -# - name: AZURE_SERVICEBUS_TOPIC_SUBSCRIPTION -# value: recordstopicsubscription -# - name: AZURE_REINDEX_TOPIC_SUBSCRIPTION -# value: reindextopicsubscription -# - name: AZURE_SCHEMACHANGED_TOPIC_NAME -# value: schemachangedtopic -# - name: AZURE_SCHEMACHANGED_TOPIC_SUBSCRIPTION -# value: schemachangedtopiceg -# - name: MAX_CONCURRENT_CALLS -# value: "32" -# - name: MAX_DELIVERY_COUNT -# value: "5" -# - name: EXECUTOR_N_THREADS -# value: "32" -# - name: MAX_LOCK_RENEW_DURATION_SECONDS -# value: "600" -# - name: PARTITION_API -# value: http://partition/api/partition/v1 -# - name: INDEXER_WORKER_URL -# value: http://indexer/api/indexer/v2/_dps/task-handlers/index-worker -# - name: schema_worker_url -# value: http://indexer-service/api/indexer/v2/_dps/task-handlers/schema-worker \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-indexer-service + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-legal + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: indexer + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: indexer + path: /api/indexer/v2/ + hosts: + - "*" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/system/indexer-service/indexer-service- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/indexer/v2/info" + - /api/indexer/v2/swagger* + - /api/indexer/v2/api-docs* + - "/api/indexer/v2/webjars/*" + - '*/index-worker' + - '*/_dps/task-handlers' + - '*/reindex' + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SECURITY_HTTPS_CERTIFICATE_TRUST + value: "true" + - name: SPRING_APPLICATION_NAME + value: indexer + - name: SERVER_SERVLET_CONTEXTPATH + value: /api/indexer/v2/ + - name: COSMOSDB_DATABASE + value: osdu-db + - name: REDIS_DATABASE + value: "4" + - name: REDIS_TTL_SECONDS + value: "3600" + - name: SERVICEBUS_TOPIC_NAME + value: indexing-progress + - name: REINDEX_TOPIC_NAME + value: recordstopic + - name: PARTITION_SERVICE_ENDPOINT + value: http://partition/api/partition/v1 + - name: ENTITLEMENTS_SERVICE_ENDPOINT + value: http://entitlements/api/entitlements/v2 + - name: ENTITLEMENTS_SERVICE_API_KEY + value: "OBSOLETE" + - name: SCHEMA_SERVICE_URL + value: http://schema/api/schema-service/v1 + - name: STORAGE_SERVICE_URL + value: http://storage/api/storage/v2 + - name: STORAGE_SCHEMA_HOST + value: http://storage/api/storage/v2/schemas + - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST + value: http://storage/api/storage/v2/query/records:batch + - name: STORAGE_QUERY_RECORD_HOST + value: http://storage/api/storage/v2/query/records + - name: SEARCH_SERVICE_URL + value: http://search/api/search/v2 +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-indexer-queue + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-legal + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: indexer-queue + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: indexer-queue + repository: community.opengroup.org:5555/osdu/platform/system/indexer-queue/indexer-queue- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + env: + - name: AZURE_TENANT_ID + secret: + name: active-directory + key: tenant-id + - name: AZURE_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: AZURE_CLIENT_SECRET + secret: + name: active-directory + key: principal-clientpassword + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: AZURE_APP_RESOURCE_ID + secret: + name: active-directory + key: principal-clientid + - name: AZURE_APPLICATION_INSIGHTS_INSTRUMENTATION_KEY + secret: + name: azure-resources + key: insights-key + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "false" + - name: SERVER_PORT + value: "80" + - name: SPRING_APPLICATION_NAME + value: indexer-queue + - name: AZURE_SERVICEBUS_TOPIC_NAME + value: recordstopic + - name: AZURE_REINDEX_TOPIC_NAME + value: reindextopic + - name: AZURE_SERVICEBUS_TOPIC_SUBSCRIPTION + value: recordstopicsubscription + - name: AZURE_REINDEX_TOPIC_SUBSCRIPTION + value: reindextopicsubscription + - name: AZURE_SCHEMACHANGED_TOPIC_NAME + value: schemachangedtopic + - name: AZURE_SCHEMACHANGED_TOPIC_SUBSCRIPTION + value: schemachangedtopiceg + - name: MAX_CONCURRENT_CALLS + value: "32" + - name: MAX_DELIVERY_COUNT + value: "5" + - name: EXECUTOR_N_THREADS + value: "32" + - name: MAX_LOCK_RENEW_DURATION_SECONDS + value: "600" + - name: PARTITION_API + value: http://partition/api/partition/v1 + - name: INDEXER_WORKER_URL + value: http://indexer/api/indexer/v2/_dps/task-handlers/index-worker + - name: schema_worker_url + value: http://indexer-service/api/indexer/v2/_dps/task-handlers/schema-worker \ No newline at end of file diff --git a/software/applications/osdu-core/schema.yaml b/software/applications/osdu-core/schema.yaml index 6a162bbb..98578b0f 100644 --- a/software/applications/osdu-core/schema.yaml +++ b/software/applications/osdu-core/schema.yaml @@ -1,167 +1,167 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-schema -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-indexer-service -# namespace: osdu-core -# - name: osdu-indexer-queue -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: schema -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: schema -# path: /api/schema-service/v1/ -# hosts: -# - "*" -# cors: -# - "http://localhost:8080" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/system/schema-service/schema-service- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# keyvault: true -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/schema-service/v1/info" -# - "/api/schema-service/v1/swagger*" -# - "/api/schema-service/v1/api-docs*" -# - "/api/schema-service/v2/webjars/*" -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_PODIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_SERVLET_CONTEXTPATH -# value: "/api/schema-service/v1/" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: "schema" -# - name: COSMOSDB_DATABASE -# value: "osdu-db" -# - name: LOG_PREFIX -# value: "schema" -# - name: AZURE_SYSTEM_STORAGECONTAINERNAME -# value: "system" -# - name: SERVICEBUS_TOPIC_NAME -# value: "schemachangedtopic" -# - name: EVENT_GRID_ENABLED -# value: 'false' -# - name: EVENT_GRID_TOPIC -# value: "schemachangedtopic" -# - name: SERVICE_BUS_ENABLED -# value: 'true' -# - name: PARTITION_SERVICE_ENDPOINT -# value: "http://partition/api/partition/v1" -# - name: ENTITLEMENTS_SERVICE_ENDPOINT -# value: "http://entitlements/api/entitlements/v2" -# - name: ENTITLEMENTS_SERVICE_API_KEY -# value: "OBSOLETE" -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-schema -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-schema -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# installationType: osduCore -# jobs: -# partitionInit: false -# entitlementInit: false -# userInit: false -# schemaInit: true -# elasticInit: false -# partition: opendes -# clientSecret: -# name: active-directory -# key: principal-clientpassword -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: clientId -# valuesKey: client_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: tenantId -# valuesKey: tenant_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: serviceBus -# valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-schema + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-indexer-service + namespace: osdu-core + - name: osdu-indexer-queue + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: schema + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: schema + path: /api/schema-service/v1/ + hosts: + - "*" + cors: + - "http://localhost:8080" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/system/schema-service/schema-service- + tag: latest + probe: + path: /actuator/health + port: 8081 + keyvault: true + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/schema-service/v1/info" + - "/api/schema-service/v1/swagger*" + - "/api/schema-service/v1/api-docs*" + - "/api/schema-service/v2/webjars/*" + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_SERVLET_CONTEXTPATH + value: "/api/schema-service/v1/" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: "schema" + - name: COSMOSDB_DATABASE + value: "osdu-db" + - name: LOG_PREFIX + value: "schema" + - name: AZURE_SYSTEM_STORAGECONTAINERNAME + value: "system" + - name: SERVICEBUS_TOPIC_NAME + value: "schemachangedtopic" + - name: EVENT_GRID_ENABLED + value: 'false' + - name: EVENT_GRID_TOPIC + value: "schemachangedtopic" + - name: SERVICE_BUS_ENABLED + value: 'true' + - name: PARTITION_SERVICE_ENDPOINT + value: "http://partition/api/partition/v1" + - name: ENTITLEMENTS_SERVICE_ENDPOINT + value: "http://entitlements/api/entitlements/v2" + - name: ENTITLEMENTS_SERVICE_API_KEY + value: "OBSOLETE" +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-schema + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-schema + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + installationType: osduCore + jobs: + partitionInit: false + entitlementInit: false + userInit: false + schemaInit: true + elasticInit: false + partition: opendes + clientSecret: + name: active-directory + key: principal-clientpassword + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: clientId + valuesKey: client_id + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id + - kind: ConfigMap + name: configmap-services + targetPath: serviceBus + valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file From 47e649cfad51393b88dd3961770b02ae738f3a7b Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 14:06:01 -0600 Subject: [PATCH 080/122] Renabled Indexer-Queue --- software/applications/osdu-core/indexer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/applications/osdu-core/indexer.yaml b/software/applications/osdu-core/indexer.yaml index 978cfd20..38f0bbe1 100644 --- a/software/applications/osdu-core/indexer.yaml +++ b/software/applications/osdu-core/indexer.yaml @@ -205,7 +205,7 @@ spec: - name: AZURE_ISTIOAUTH_ENABLED value: "true" - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - value: "false" + value: "true" - name: SERVER_PORT value: "80" - name: SPRING_APPLICATION_NAME From 7c1ca44a3bb67a9f1266c32d43e484958a142693 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 14:24:25 -0600 Subject: [PATCH 081/122] Renabled Indexer-Queue --- software/applications/osdu-core/indexer.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/software/applications/osdu-core/indexer.yaml b/software/applications/osdu-core/indexer.yaml index 38f0bbe1..f3623733 100644 --- a/software/applications/osdu-core/indexer.yaml +++ b/software/applications/osdu-core/indexer.yaml @@ -174,18 +174,6 @@ spec: seconds: 10 keyvault: true env: - - name: AZURE_TENANT_ID - secret: - name: active-directory - key: tenant-id - - name: AZURE_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_CLIENT_SECRET - secret: - name: active-directory - key: principal-clientpassword - name: KEYVAULT_URI secret: name: azure-resources From a7e47877a845b0fb494bdd6e710b2dff112e1e46 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 14:49:12 -0600 Subject: [PATCH 082/122] Fixing schema init. --- .../templates/schema-init.yaml | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/charts/osdu-developer-init/templates/schema-init.yaml b/charts/osdu-developer-init/templates/schema-init.yaml index 9a66b61e..dcf9d5a9 100644 --- a/charts/osdu-developer-init/templates/schema-init.yaml +++ b/charts/osdu-developer-init/templates/schema-init.yaml @@ -10,7 +10,11 @@ metadata: spec: ttlSecondsAfterFinished: 120 template: + metadata: + labels: + azure.workload.identity/use: "true" spec: + serviceAccountName: workload-identity-sa volumes: - name: script configMap: @@ -30,13 +34,6 @@ spec: value: {{ .Values.clientId | quote }} - name: AZURE_TENANT_ID value: {{ .Values.tenantId | quote }} - - name: AZURE_CLIENT_ID - value: {{ .Values.clientId | quote }} - - name: AZURE_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.clientSecret.name | quote }} - key: {{ .Values.clientSecret.key | quote }} containers: - name: sleep image: istio/base @@ -79,7 +76,17 @@ data: currentMessage="Schema loading failed. Please check error logs for more details." fi if [ ! -z "$CONFIG_MAP_NAME" -a "$CONFIG_MAP_NAME" != " " ]; then - az login --identity --username $OSDU_IDENTITY_ID + + echo "==================================================================" + echo " Logging in using Workload Identity" + echo "==================================================================" + + # Login using the federated token from the environment variable + az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ + --service-principal \ + -u ${AZURE_CLIENT_ID} \ + -t ${AZURE_TENANT_ID} + ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv) az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS kubectl config set-context $RESOURCE_GROUP_NAME --cluster $ENV_AKS From f56ae59d8167652e57211322969dfb3885fe2ee4 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 15:20:20 -0600 Subject: [PATCH 083/122] Fixing schema init. --- .../templates/schema-init.yaml | 61 ++++++++++++++++++- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/charts/osdu-developer-init/templates/schema-init.yaml b/charts/osdu-developer-init/templates/schema-init.yaml index dcf9d5a9..2d74f7b1 100644 --- a/charts/osdu-developer-init/templates/schema-init.yaml +++ b/charts/osdu-developer-init/templates/schema-init.yaml @@ -27,6 +27,9 @@ spec: - name: script mountPath: "/home/osdu/deployments/scripts/azure/bootstrap.sh" subPath: init.sh + - name: token + mountPath: "/home/osdu/deployments/scripts/azure/Token.py" + subPath: token.py env: - name: DATA_PARTITION value: {{ .Values.partition | quote }} @@ -83,9 +86,9 @@ data: # Login using the federated token from the environment variable az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ - --service-principal \ - -u ${AZURE_CLIENT_ID} \ - -t ${AZURE_TENANT_ID} + --service-principal \ + -u ${AZURE_CLIENT_ID} \ + -t ${AZURE_TENANT_ID} ENV_AKS=$(az aks list --resource-group $RESOURCE_GROUP_NAME --query [].name -otsv) az aks get-credentials --resource-group $RESOURCE_GROUP_NAME --name $ENV_AKS @@ -108,4 +111,56 @@ data: else exit 1 fi + token.py: | + import os + import msal + + class AzureToken(object): + def get_azure_id_token(self): + tenant_id = os.getenv('AZURE_TENANT_ID') + client_id = os.getenv('AZURE_CLIENT_ID') + + # Read the federated token provided by workload identity + token_path = os.getenv('AZURE_FEDERATED_TOKEN_FILE', '/var/run/secrets/azure/tokens/azure-identity-token') + + if not all([tenant_id, client_id]): + print('Missing required environment variables: AZURE_TENANT_ID and AZURE_CLIENT_ID are required') + exit(1) + + try: + # Read the federated token + with open(token_path, 'r') as f: + federated_token = f.read().strip() + + authority_host_uri = 'https://login.microsoftonline.com' + authority_uri = authority_host_uri + '/' + tenant_id + + # Configure MSAL for federated token exchange + app = msal.ConfidentialClientApplication( + client_id=client_id, + authority=authority_uri, + client_credential={ + "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + "client_assertion": federated_token + } + ) + + # Use the same scope as az cli would use + scopes = ["https://management.azure.com/.default"] + result = app.acquire_token_for_client(scopes=scopes) + + if 'access_token' in result: + token = 'Bearer ' + result['access_token'] + print(token) + return token + else: + print(f"Error getting token: {result.get('error_description', 'Unknown error')}") + exit(1) + + except Exception as e: + print(f"Error: {str(e)}") + exit(1) + + if __name__ == '__main__': + AzureToken().get_azure_id_token() {{- end }} From 8126dc382c46c32802deb96b79d3e3a17b96eb82 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 15:32:33 -0600 Subject: [PATCH 084/122] Fixing schema init. --- charts/osdu-developer-init/Chart.yaml | 2 +- charts/osdu-developer-init/templates/schema-init.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/charts/osdu-developer-init/Chart.yaml b/charts/osdu-developer-init/Chart.yaml index 2f97a4f5..19721525 100644 --- a/charts/osdu-developer-init/Chart.yaml +++ b/charts/osdu-developer-init/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-init type: application description: Installs the OSDU developer Initialization resources -version: 0.0.2 +version: 0.0.3 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/osdu-developer-init/templates/schema-init.yaml b/charts/osdu-developer-init/templates/schema-init.yaml index 2d74f7b1..f32ea282 100644 --- a/charts/osdu-developer-init/templates/schema-init.yaml +++ b/charts/osdu-developer-init/templates/schema-init.yaml @@ -20,6 +20,10 @@ spec: configMap: name: schema-init-script defaultMode: 0777 + - name: token + configMap: + name: schema-init-script + defaultMode: 0777 initContainers: - name: data-seed image: community.opengroup.org:5555/osdu/platform/system/schema-service/schema-service-schema-load-release-0-27:beb6f65c1d9c303e86a6047adc93b2192d0c62ba From 1e5cc1fc0302f33047907a33ce4c873640d9e9b9 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 15:43:54 -0600 Subject: [PATCH 085/122] Adding in additional services. --- .../applications/osdu-core/entitlements.yaml | 3 - software/applications/osdu-core/file.yaml | 276 ++++++------- software/applications/osdu-core/search.yaml | 242 +++++------ software/applications/osdu-core/storage.yaml | 293 +++++++------ software/applications/osdu-core/workflow.yaml | 390 +++++++++--------- 5 files changed, 592 insertions(+), 612 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index b9d77dc9..b14e7ca3 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -27,9 +27,6 @@ spec: - kind: ConfigMap name: configmap-software valuesKey: value.yaml - # - kind: ConfigMap - # name: configmap-common-values - # valuesKey: value.yaml - kind: ConfigMap name: configmap-repo-override optional: true diff --git a/software/applications/osdu-core/file.yaml b/software/applications/osdu-core/file.yaml index 054edcfd..34a020b1 100644 --- a/software/applications/osdu-core/file.yaml +++ b/software/applications/osdu-core/file.yaml @@ -1,138 +1,138 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-file -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-indexer-queue -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: file -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: file -# path: /api/file/ -# hosts: -# - "*" -# cors: -# - "http://localhost:8080" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/system/file/file- -# branch: release-0-26 -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# request: -# cpu: 1000m -# memory: 1Gi -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/file/v2/info" -# - "/api/file/v2/swagger*" -# - "/api/file/v2/api-docs*" -# - "/api/file/v2/webjars/*" -# env: -# - name: KEYVAULT_URL -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AZURE_AD_APP_RESOURCE_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_PODIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_SERVLET_CONTEXTPATH -# value: "/api/file/" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: "file" -# - name: SPRING_CONFIG_NAME -# value: "common,application" -# - name: LOG_PREFIX -# value: file -# - name: LOGGING_LEVEL -# value: INFO -# - name: APPLICATION_PORT -# value: 80 -# - name: COSMOSDB_DATABASE -# value: osdu-db -# - name: OSDU_ENTITLEMENTS_APP_KEY -# value: OBSOLETE -# - name: PARTITION_SERVICE_ENDPOINT -# value: "http://partition/api/partition/v1" -# - name: OSDU_ENTITLEMENTS_URL -# value: http://entitlements/api/entitlements/v2 -# - name: authorizeAPI -# value: http://entitlements/api/entitlements/v2 -# - name: OSDU_STORAGE_URL -# value: http://storage/api/storage/v2 -# - name: SEARCH_HOST -# value: http://search/api/search/v2 -# - name: AZURE_PUBSUB_PUBLISH -# value: "true" -# - name: SERVICE_BUS_ENABLED_STATUS -# value: "true" -# - name: SERVICE_BUS_TOPIC_STATUS -# value: "statuschangedtopic" -# - name: BATCH_SIZE -# value: "100" -# - name: SEARCH_QUERY_LIMIT -# value: "1000" -# - name: FILE_CHECKSUM_CALCULATION_LIMIT -# value: "5368709120L" \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-file + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-indexer-queue + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: file + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: file + path: /api/file/ + hosts: + - "*" + cors: + - "http://localhost:8080" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/system/file/file- + branch: release-0-26 + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + request: + cpu: 1000m + memory: 1Gi + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/file/v2/info" + - "/api/file/v2/swagger*" + - "/api/file/v2/api-docs*" + - "/api/file/v2/webjars/*" + env: + - name: KEYVAULT_URL + secret: + name: azure-resources + key: keyvault-uri + - name: AZURE_AD_APP_RESOURCE_ID + secret: + name: active-directory + key: principal-clientid + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_SERVLET_CONTEXTPATH + value: "/api/file/" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: "file" + - name: SPRING_CONFIG_NAME + value: "common,application" + - name: LOG_PREFIX + value: file + - name: LOGGING_LEVEL + value: INFO + - name: APPLICATION_PORT + value: 80 + - name: COSMOSDB_DATABASE + value: osdu-db + - name: OSDU_ENTITLEMENTS_APP_KEY + value: OBSOLETE + - name: PARTITION_SERVICE_ENDPOINT + value: "http://partition/api/partition/v1" + - name: OSDU_ENTITLEMENTS_URL + value: http://entitlements/api/entitlements/v2 + - name: authorizeAPI + value: http://entitlements/api/entitlements/v2 + - name: OSDU_STORAGE_URL + value: http://storage/api/storage/v2 + - name: SEARCH_HOST + value: http://search/api/search/v2 + - name: AZURE_PUBSUB_PUBLISH + value: "true" + - name: SERVICE_BUS_ENABLED_STATUS + value: "true" + - name: SERVICE_BUS_TOPIC_STATUS + value: "statuschangedtopic" + - name: BATCH_SIZE + value: "100" + - name: SEARCH_QUERY_LIMIT + value: "1000" + - name: FILE_CHECKSUM_CALCULATION_LIMIT + value: "5368709120L" \ No newline at end of file diff --git a/software/applications/osdu-core/search.yaml b/software/applications/osdu-core/search.yaml index c2958375..864ac275 100644 --- a/software/applications/osdu-core/search.yaml +++ b/software/applications/osdu-core/search.yaml @@ -1,121 +1,121 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-search -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-indexer-queue -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: search -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: search -# path: /api/search/v2/ -# hosts: -# - "*" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/system/search-service/search-service- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# request: -# cpu: 1000m -# memory: 1Gi -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/search/v2/info" -# - "/api/search/v2/swagger*" -# - "/api/search/v2/api-docs*" -# - "/api/search/v2/webjars/*" -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_PODIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: search -# - name: SERVER_SERVLET_CONTEXTPATH -# value: /api/search/v2/ -# - name: LOG_PREFIX -# value: "search" -# - name: SEARCH_SERVICE_SPRING_LOGGING_LEVEL -# value: "DEBUG" -# - name: COSMOSDB_DATABASE -# value: "osdu-db" -# - name: REDIS_DATABASE -# value: "5" -# - name: ENVIRONMENT -# value: "evt" -# - name: ELASTIC_CACHE_EXPIRATION -# value: 1 -# - name: MAX_CACHE_VALUE_SIZE -# value: 60 -# - name: POLICY_SERVICE_ENABLED -# value: "false" -# - name: ENTITLEMENTS_SERVICE_API_KEY -# value: "OBSOLETE" -# - name: PARTITION_SERVICE_ENDPOINT -# value: "http://partition/api/partition/v1" -# - name: ENTITLEMENTS_SERVICE_ENDPOINT -# value: "http://entitlements/api/entitlements/v2" -# - name: POLICY_SERVICE_ENDPOINT -# value: http://policy/api/policy/v1 \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-search + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-indexer-queue + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: search + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: search + path: /api/search/v2/ + hosts: + - "*" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/system/search-service/search-service- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + request: + cpu: 1000m + memory: 1Gi + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/search/v2/info" + - "/api/search/v2/swagger*" + - "/api/search/v2/api-docs*" + - "/api/search/v2/webjars/*" + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: search + - name: SERVER_SERVLET_CONTEXTPATH + value: /api/search/v2/ + - name: LOG_PREFIX + value: "search" + - name: SEARCH_SERVICE_SPRING_LOGGING_LEVEL + value: "DEBUG" + - name: COSMOSDB_DATABASE + value: "osdu-db" + - name: REDIS_DATABASE + value: "5" + - name: ENVIRONMENT + value: "evt" + - name: ELASTIC_CACHE_EXPIRATION + value: 1 + - name: MAX_CACHE_VALUE_SIZE + value: 60 + - name: POLICY_SERVICE_ENABLED + value: "false" + - name: ENTITLEMENTS_SERVICE_API_KEY + value: "OBSOLETE" + - name: PARTITION_SERVICE_ENDPOINT + value: "http://partition/api/partition/v1" + - name: ENTITLEMENTS_SERVICE_ENDPOINT + value: "http://entitlements/api/entitlements/v2" + - name: POLICY_SERVICE_ENDPOINT + value: http://policy/api/policy/v1 \ No newline at end of file diff --git a/software/applications/osdu-core/storage.yaml b/software/applications/osdu-core/storage.yaml index e3eae9ab..f503481f 100644 --- a/software/applications/osdu-core/storage.yaml +++ b/software/applications/osdu-core/storage.yaml @@ -1,155 +1,138 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-storage -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-indexer-queue -# namespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# targetNamespace: osdu-core -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: storage -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: storage -# path: /api/storage/v2/ -# hosts: -# - "*" -# cors: -# - "http://localhost:8080" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/system/storage/storage- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# request: -# cpu: 800m -# memory: 1Gi -# # limit: -# # cpu: 1000m -# # memory: 4Gi -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/storage/v2/info" -# - "/api/storage/v2/swagger*" -# - "/api/storage/v2/api-docs*" -# - "/api/storage/v2/webjars/*" -# env: -# - name: AZURE_TENANT_ID -# secret: -# name: active-directory -# key: tenant-id -# - name: AZURE_SUBSCRIPTION_ID -# secret: -# name: active-directory -# key: subscription-id -# - name: AZURE_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: AZURE_CLIENT_SECRET -# secret: -# name: active-directory -# key: principal-clientpassword -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_PODIDENTITY_ISENABLED -# value: "false" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: storage -# - name: SERVER_SERVLET_CONTEXTPATH -# value: /api/storage/v2/ -# - name: COSMOSDB_DATABASE -# value: osdu-db -# - name: AZURE_EVENTGRID_ENABLED -# value: "false" -# - name: AZURE_SERVICEBUS_ENABLED -# value: "true" -# - name: SERVICEBUS_TOPIC_NAME -# value: recordstopic -# - name: SERVICEBUS_V2_TOPIC_NAME -# value: recordstopic-v2 -# - name: REDIS_DATABASE -# value: "4" -# - name: PARTITION_SERVICE_ENDPOINT -# value: http://partition/api/partition/v1 -# - name: ENTITLEMENTS_SERVICE_ENDPOINT -# value: http://entitlements/api/entitlements/v2 -# - name: ENTITLEMENTS_SERVICE_API_KEY -# value: "OBSOLETE" -# - name: LEGAL_SERVICE_ENDPOINT -# value: http://legal/api/legal/v1 -# - name: LEGAL_SERVICE_REGION -# value: southcentralus -# - name: LEGAL_SERVICEBUS_TOPIC_NAME -# value: legaltagschangedtopiceg -# - name: LEGAL_SERVICEBUS_TOPIC_SUBSCRIPTION -# value: eg_sb_legaltagchangedsubscription -# - name: CRS_CONVERSION_SERVICE_ENDPOINT -# value: http://crs-conversion/api/crs/converter/v2 -# - name: POLICY_SERVICE_ENDPOINT -# value: http://policy/api/policy/v1 -# - name: OPA_ENABLED -# value: "false" -# - name: REDIS_HOST_KEY -# value: redis-hostname -# - name: REDIS_PASSWORD_KEY -# value: redis-password - \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-storage + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-indexer-queue + namespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + targetNamespace: osdu-core + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: storage + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: storage + path: /api/storage/v2/ + hosts: + - "*" + cors: + - "http://localhost:8080" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/system/storage/storage- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + request: + cpu: 800m + memory: 1Gi + # limit: + # cpu: 1000m + # memory: 4Gi + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/storage/v2/info" + - "/api/storage/v2/swagger*" + - "/api/storage/v2/api-docs*" + - "/api/storage/v2/webjars/*" + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: storage + - name: SERVER_SERVLET_CONTEXTPATH + value: /api/storage/v2/ + - name: COSMOSDB_DATABASE + value: osdu-db + - name: AZURE_EVENTGRID_ENABLED + value: "false" + - name: AZURE_SERVICEBUS_ENABLED + value: "true" + - name: SERVICEBUS_TOPIC_NAME + value: recordstopic + - name: SERVICEBUS_V2_TOPIC_NAME + value: recordstopic-v2 + - name: REDIS_DATABASE + value: "4" + - name: PARTITION_SERVICE_ENDPOINT + value: http://partition/api/partition/v1 + - name: ENTITLEMENTS_SERVICE_ENDPOINT + value: http://entitlements/api/entitlements/v2 + - name: ENTITLEMENTS_SERVICE_API_KEY + value: "OBSOLETE" + - name: LEGAL_SERVICE_ENDPOINT + value: http://legal/api/legal/v1 + - name: LEGAL_SERVICE_REGION + value: southcentralus + - name: LEGAL_SERVICEBUS_TOPIC_NAME + value: legaltagschangedtopiceg + - name: LEGAL_SERVICEBUS_TOPIC_SUBSCRIPTION + value: eg_sb_legaltagchangedsubscription + - name: CRS_CONVERSION_SERVICE_ENDPOINT + value: http://crs-conversion/api/crs/converter/v2 + - name: POLICY_SERVICE_ENDPOINT + value: http://policy/api/policy/v1 + - name: OPA_ENABLED + value: "false" + - name: REDIS_HOST_KEY + value: redis-hostname + - name: REDIS_PASSWORD_KEY + value: redis-password diff --git a/software/applications/osdu-core/workflow.yaml b/software/applications/osdu-core/workflow.yaml index 8e67b2a2..752c5bd8 100644 --- a/software/applications/osdu-core/workflow.yaml +++ b/software/applications/osdu-core/workflow.yaml @@ -1,195 +1,195 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-workflow -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-partition -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: workflow -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: workflow -# path: /api/workflow/ -# hosts: -# - "*" -# cors: -# - "http://localhost:8080" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/data-flow/ingestion/ingestion-workflow/ingestion-workflow- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/workflow/v3/info" -# - "/api/workflow/v3/swagger*" -# - "/api/workflow/v3/api-docs*" -# - "/api/workflow/v3/webjars/*" -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_PODIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_SERVLET_CONTEXTPATH -# value: "/api/workflow/" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: "workflow" -# - name: SPRING_CONFIG_NAME -# value: "common,application" -# - name: LOG_PREFIX -# value: "workflow" -# - name: AZURE_STORAGE_ENABLE_HTTPS -# value: "true" -# - name: COSMOSDB_DATABASE -# value: "osdu-db" -# - name: COSMOSDB_SYSTEM_DATABASE -# value: osdu-system-db -# - name: AIRFLOW_STORAGE_ACCOUNT_NAME -# secret: -# name: azure-resources -# key: azurestorageaccountname -# - name: AIRFLOW_STORAGE_ACCOUNT_KEY -# secret: -# name: azure-resources -# key: azurestorageaccountkey -# - name: OSDU_AIRFLOW_USERNAME -# secret: -# name: azure-resources -# key: airflow-username -# - name: OSDU_AIRFLOW_PASSWORD -# secret: -# name: azure-resources -# key: airflow-password -# - name: AUTHORIZEAPI -# value: http://entitlements/api/entitlements/v2 -# - name: AUTHORIZEAPIKEY -# value: "OBSOLETE" -# - name: PARTITION_SERVICE_ENDPOINT -# value: "http://partition/api/partition/v1" -# - name: OSDU_ENTITLEMENTS_URL -# value: "http://entitlements/api/entitlements/v2" -# - name: OSDU_AIRFLOW_URL -# value: "http://airflow-web.airflow.svc.cluster.local:8080/airflow" -# - name: OSDU_ENTITLEMENTS_APPKEY -# value: "OBSOLETE" -# - name: OSDU_AIRFLOW_VERSION2_ENABLED -# value: true -# - name: DP_AIRFLOW_FOR_SYSTEM_DAG -# value: "false" -# - name: IGNORE_DAGCONTENT -# value: "true" -# - name: IGNORE_CUSTOMOPERATORCONTENT -# value: "true" -# --- -# # Retrigger: kubectl annotate helmrelease osdu-init-workflow fluxcd.io/retrigger=$(date +%s) -n osdu-core -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-workflow -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# fluxcd.io/retrigger: "initial" -# spec: -# dependsOn: -# - name: osdu-workflow -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# installationType: osduCore -# jobs: -# workflowInit: true -# workflows: -# - name: "Osdu_ingest" -# description: "Manifest Ingest workflow for OSDU" -# - name: "Osdu_ingest_by_reference" -# description: "Manifest Ingest by reference workflow for OSDU" -# - name: 'csv-parser' -# description: 'CSV Parser workflow for OSDU' -# partition: opendes -# clientSecret: -# name: active-directory -# key: principal-clientpassword -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: clientId -# valuesKey: client_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: tenantId -# valuesKey: tenant_id +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-workflow + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-partition + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: workflow + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: workflow + path: /api/workflow/ + hosts: + - "*" + cors: + - "http://localhost:8080" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/data-flow/ingestion/ingestion-workflow/ingestion-workflow- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/workflow/v3/info" + - "/api/workflow/v3/swagger*" + - "/api/workflow/v3/api-docs*" + - "/api/workflow/v3/webjars/*" + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_SERVLET_CONTEXTPATH + value: "/api/workflow/" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: "workflow" + - name: SPRING_CONFIG_NAME + value: "common,application" + - name: LOG_PREFIX + value: "workflow" + - name: AZURE_STORAGE_ENABLE_HTTPS + value: "true" + - name: COSMOSDB_DATABASE + value: "osdu-db" + - name: COSMOSDB_SYSTEM_DATABASE + value: osdu-system-db + - name: AIRFLOW_STORAGE_ACCOUNT_NAME + secret: + name: azure-resources + key: azurestorageaccountname + - name: AIRFLOW_STORAGE_ACCOUNT_KEY + secret: + name: azure-resources + key: azurestorageaccountkey + - name: OSDU_AIRFLOW_USERNAME + secret: + name: azure-resources + key: airflow-username + - name: OSDU_AIRFLOW_PASSWORD + secret: + name: azure-resources + key: airflow-password + - name: AUTHORIZEAPI + value: http://entitlements/api/entitlements/v2 + - name: AUTHORIZEAPIKEY + value: "OBSOLETE" + - name: PARTITION_SERVICE_ENDPOINT + value: "http://partition/api/partition/v1" + - name: OSDU_ENTITLEMENTS_URL + value: "http://entitlements/api/entitlements/v2" + - name: OSDU_AIRFLOW_URL + value: "http://airflow-web.airflow.svc.cluster.local:8080/airflow" + - name: OSDU_ENTITLEMENTS_APPKEY + value: "OBSOLETE" + - name: OSDU_AIRFLOW_VERSION2_ENABLED + value: true + - name: DP_AIRFLOW_FOR_SYSTEM_DAG + value: "false" + - name: IGNORE_DAGCONTENT + value: "true" + - name: IGNORE_CUSTOMOPERATORCONTENT + value: "true" +--- +# Retrigger: kubectl annotate helmrelease osdu-init-workflow fluxcd.io/retrigger=$(date +%s) -n osdu-core +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-workflow + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" + fluxcd.io/retrigger: "initial" +spec: + dependsOn: + - name: osdu-workflow + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + installationType: osduCore + jobs: + workflowInit: true + workflows: + - name: "Osdu_ingest" + description: "Manifest Ingest workflow for OSDU" + - name: "Osdu_ingest_by_reference" + description: "Manifest Ingest by reference workflow for OSDU" + - name: 'csv-parser' + description: 'CSV Parser workflow for OSDU' + partition: opendes + clientSecret: + name: active-directory + key: principal-clientpassword + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: clientId + valuesKey: client_id + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id From aec2d942d4c02dcf0fb3a168975634309fefbbbd Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 15:50:08 -0600 Subject: [PATCH 086/122] Added workflow init. --- .../templates/workflow-init.yaml | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/charts/osdu-developer-init/templates/workflow-init.yaml b/charts/osdu-developer-init/templates/workflow-init.yaml index 74456450..021a2028 100644 --- a/charts/osdu-developer-init/templates/workflow-init.yaml +++ b/charts/osdu-developer-init/templates/workflow-init.yaml @@ -10,7 +10,11 @@ metadata: spec: ttlSecondsAfterFinished: 120 template: + metadata: + labels: + azure.workload.identity/use: "true" spec: + serviceAccountName: workload-identity-sa volumes: - name: script configMap: @@ -18,7 +22,7 @@ spec: defaultMode: 0500 initContainers: - name: data-seed - image: alpine + image: mcr.microsoft.com/azure-cli:cbl-mariner2.0 command: - script/init.sh volumeMounts: @@ -60,30 +64,20 @@ data: set -euo pipefail set -o nounset - apk add --no-cache curl jq + tdnf install -y curl jq echo "==================================================================" - echo " Creating Bearer Token for Application: ${AZURE_CLIENT_ID} " + echo " Logging in using Workload Identity" echo "==================================================================" - echo " Identity Client Id: ${AZURE_CLIENT_ID}" - OUTPUT=$(curl -s -w "%{http_code}" --request POST \ - --url https://login.microsoftonline.com/${AZURE_TENANT_ID}/oauth2/token \ - --header "content-type: application/x-www-form-urlencoded" \ - --data "grant_type=client_credentials" \ - --data "client_id=${AZURE_CLIENT_ID}" \ - --data "client_secret=${AZURE_CLIENT_SECRET}" \ - --data "resource=${AZURE_CLIENT_ID}") + # Login using the federated token from the environment variable + az login --federated-token "$(cat ${AZURE_FEDERATED_TOKEN_FILE})" \ + --service-principal \ + -u ${AZURE_CLIENT_ID} \ + -t ${AZURE_TENANT_ID} - HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$') - BODY=${OUTPUT%???} - - if [[ "$HTTP_STATUS_CODE" != "200" ]]; then - echo "Error: Unexpected HTTP status code $HTTP_STATUS_CODE" - exit 1 - fi - - TOKEN=$(echo "$BODY" | jq .access_token | tr -d '"') + # Get token with the correct application ID as resource + TOKEN=$(az account get-access-token --resource "https://management.azure.com/" --query accessToken -o tsv) # Log the WORKFLOWS variable to check its format echo "WORKFLOWS: $WORKFLOWS" From b9f98fe38e9d71b1edef109bdffb46fb68890b58 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 16:03:49 -0600 Subject: [PATCH 087/122] Added workflow init. --- charts/osdu-developer-init/Chart.yaml | 2 +- charts/osdu-developer-init/templates/workflow-init.yaml | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/charts/osdu-developer-init/Chart.yaml b/charts/osdu-developer-init/Chart.yaml index 19721525..f40a0cf7 100644 --- a/charts/osdu-developer-init/Chart.yaml +++ b/charts/osdu-developer-init/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-init type: application description: Installs the OSDU developer Initialization resources -version: 0.0.3 +version: 0.0.4 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/osdu-developer-init/templates/workflow-init.yaml b/charts/osdu-developer-init/templates/workflow-init.yaml index 021a2028..93827121 100644 --- a/charts/osdu-developer-init/templates/workflow-init.yaml +++ b/charts/osdu-developer-init/templates/workflow-init.yaml @@ -33,15 +33,6 @@ spec: value: {{ $namespace }} - name: PARTITION value: {{ .Values.partition | quote }} - - name: AZURE_TENANT_ID - value: {{ .Values.tenantId | quote }} - - name: AZURE_CLIENT_ID - value: {{ .Values.clientId | quote }} - - name: AZURE_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.clientSecret.name | quote }} - key: {{ .Values.clientSecret.key | quote }} - name: WORKFLOWS value: {{ .Values.workflows | toJson | quote }} containers: From f9a7f45fdaf5f2cbfef0fb804827dbe197e62d17 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 17:56:26 -0600 Subject: [PATCH 088/122] Removed init users. --- .../applications/osdu-core/user-init.yaml | 202 +++++++++--------- 1 file changed, 101 insertions(+), 101 deletions(-) diff --git a/software/applications/osdu-core/user-init.yaml b/software/applications/osdu-core/user-init.yaml index fbd71bd9..850c090b 100644 --- a/software/applications/osdu-core/user-init.yaml +++ b/software/applications/osdu-core/user-init.yaml @@ -1,103 +1,103 @@ --- # kubectl create job --namespace osdu-core --from=cronjob/user-init user-init-run-$(date +%s) -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-user - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-entitlements-init - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - installationType: osduCore - jobs: - partitionInit: false - entitlementInit: false - userInit: true - elasticInit: false - schemaInit: false - clientSecret: - name: active-directory - key: principal-clientpassword - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id - - kind: ConfigMap - name: configmap-services - targetPath: emailAddress - valuesKey: first_user_id ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-user-sp - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-entitlements-init - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - installationType: osduCore - jobs: - partitionInit: false - entitlementInit: false - userInit: true - elasticInit: false - schemaInit: false - clientSecret: - name: active-directory - key: principal-clientpassword - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id - - kind: ConfigMap - name: configmap-services - targetPath: emailAddress - valuesKey: client_id \ No newline at end of file +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-user +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-entitlements-init +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# installationType: osduCore +# jobs: +# partitionInit: false +# entitlementInit: false +# userInit: true +# elasticInit: false +# schemaInit: false +# clientSecret: +# name: active-directory +# key: principal-clientpassword +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: clientId +# valuesKey: client_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: tenantId +# valuesKey: tenant_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: emailAddress +# valuesKey: first_user_id +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-user-sp +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-entitlements-init +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# installationType: osduCore +# jobs: +# partitionInit: false +# entitlementInit: false +# userInit: true +# elasticInit: false +# schemaInit: false +# clientSecret: +# name: active-directory +# key: principal-clientpassword +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: clientId +# valuesKey: client_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: tenantId +# valuesKey: tenant_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: emailAddress +# valuesKey: client_id \ No newline at end of file From ee999d57eadc03a37596b230dca7653bd302c6e0 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Fri, 3 Jan 2025 18:13:18 -0600 Subject: [PATCH 089/122] Added back user --- .../applications/osdu-core/user-init.yaml | 100 +++++++++--------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/software/applications/osdu-core/user-init.yaml b/software/applications/osdu-core/user-init.yaml index 850c090b..17619247 100644 --- a/software/applications/osdu-core/user-init.yaml +++ b/software/applications/osdu-core/user-init.yaml @@ -1,55 +1,55 @@ --- # kubectl create job --namespace osdu-core --from=cronjob/user-init user-init-run-$(date +%s) -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-user -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-entitlements-init -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# installationType: osduCore -# jobs: -# partitionInit: false -# entitlementInit: false -# userInit: true -# elasticInit: false -# schemaInit: false -# clientSecret: -# name: active-directory -# key: principal-clientpassword -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: clientId -# valuesKey: client_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: tenantId -# valuesKey: tenant_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: emailAddress -# valuesKey: first_user_id +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-user + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-entitlements-init + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + installationType: osduCore + jobs: + partitionInit: false + entitlementInit: false + userInit: true + elasticInit: false + schemaInit: false + clientSecret: + name: active-directory + key: principal-clientpassword + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: clientId + valuesKey: client_id + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id + - kind: ConfigMap + name: configmap-services + targetPath: emailAddress + valuesKey: first_user_id # --- # apiVersion: helm.toolkit.fluxcd.io/v2beta1 # kind: HelmRelease From 97be15c498b210c02f6f59dc22b90b28570c38ab Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sat, 4 Jan 2025 13:55:04 -0600 Subject: [PATCH 090/122] Reworked the init scripts. --- charts/osdu-developer-init/Chart.yaml | 2 +- charts/osdu-developer-init/README.md | 12 ++++++------ .../templates/user-init.yaml | 19 ++++++++----------- charts/osdu-developer-init/values.yaml | 11 +++++------ .../applications/osdu-core/user-init.yaml | 16 ---------------- 5 files changed, 20 insertions(+), 40 deletions(-) diff --git a/charts/osdu-developer-init/Chart.yaml b/charts/osdu-developer-init/Chart.yaml index f40a0cf7..af8bcbbf 100644 --- a/charts/osdu-developer-init/Chart.yaml +++ b/charts/osdu-developer-init/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-init type: application description: Installs the OSDU developer Initialization resources -version: 0.0.4 +version: 0.0.1 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/osdu-developer-init/README.md b/charts/osdu-developer-init/README.md index b27e6c1f..f6e58e3e 100644 --- a/charts/osdu-developer-init/README.md +++ b/charts/osdu-developer-init/README.md @@ -9,11 +9,11 @@ cat > custom_values.yaml << EOF nameOverride: "" fullnameOverride: "osdu-init" -tenantId: -clientId: -clientSecret: -serviceBus: -partition: +tenantId: +clientId: +clientSecret: +serviceBus: +partition: EOF @@ -24,7 +24,7 @@ Install the helm chart. ```bash # Create Namespace NAMESPACE=osdu-core -helm template osdu-core-osdu-init-partition -f custom_values.yaml . +helm template osdu-core-osdu-init-user -f custom_values.yaml . helm upgrade --install osdu-core-osdu-init-partition . -n $NAMESPACE -f custom_values.yaml ``` diff --git a/charts/osdu-developer-init/templates/user-init.yaml b/charts/osdu-developer-init/templates/user-init.yaml index 67296088..604f899c 100644 --- a/charts/osdu-developer-init/templates/user-init.yaml +++ b/charts/osdu-developer-init/templates/user-init.yaml @@ -1,12 +1,12 @@ {{- $enabled := eq (include "osdu-developer-init.isEnabled" .) "1" -}} {{- $namespace := .Release.Namespace -}} +{{- $releaseName := .Release.Name -}} # Capture the release name {{- if $enabled }} -{{- range $job := .Values.jobs.list }} --- apiVersion: batch/v1 kind: Job metadata: - name: {{ .Release.Name }}-{{ $job.name }} + name: {{ $releaseName }} namespace: {{ $namespace }} spec: ttlSecondsAfterFinished: 120 @@ -15,25 +15,23 @@ spec: labels: azure.workload.identity/use: "true" spec: - serviceAccountName: {{ $job.serviceAccountName }} + serviceAccountName: workload-identity-sa volumes: - name: script configMap: - name: user-init-script-{{ $job.name }} + name: configmap-{{ $releaseName }}-script defaultMode: 0500 initContainers: - name: data-seed - image: {{ $job.initContainerImage }} + image: mcr.microsoft.com/azure-cli:cbl-mariner2.0 command: - /script/init.sh volumeMounts: - name: script mountPath: "/script" env: - {{- range $env := $job.env }} - - name: {{ $env.name }} - value: {{ $env.value | quote }} - {{- end }} + - name: EMAIL_ADDRESS + value: {{ .Values.emailAddress | quote }} containers: - name: sleep image: istio/base @@ -46,7 +44,7 @@ spec: apiVersion: v1 kind: ConfigMap metadata: - name: user-init-script-{{ $job.name }} + name: configmap-{{ $releaseName }}-script namespace: {{ $namespace }} data: init.sh: | @@ -126,5 +124,4 @@ data: fi exit 0 -{{- end }} {{- end }} \ No newline at end of file diff --git a/charts/osdu-developer-init/values.yaml b/charts/osdu-developer-init/values.yaml index 8853f655..f7260e43 100644 --- a/charts/osdu-developer-init/values.yaml +++ b/charts/osdu-developer-init/values.yaml @@ -1,6 +1,5 @@ -tenantId: -clientId: -clientSecret: -serviceBus: -partition: - \ No newline at end of file +tenantId: +clientId: +clientSecret: +serviceBus: +partition: diff --git a/software/applications/osdu-core/user-init.yaml b/software/applications/osdu-core/user-init.yaml index 17619247..e5e78b02 100644 --- a/software/applications/osdu-core/user-init.yaml +++ b/software/applications/osdu-core/user-init.yaml @@ -24,28 +24,12 @@ spec: remediation: retries: 3 values: - installationType: osduCore jobs: - partitionInit: false - entitlementInit: false userInit: true - elasticInit: false - schemaInit: false - clientSecret: - name: active-directory - key: principal-clientpassword valuesFrom: - kind: ConfigMap name: configmap-software valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id - kind: ConfigMap name: configmap-services targetPath: emailAddress From 6379d10da5347c4b0c09d3a652631644bd10436a Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sat, 4 Jan 2025 14:27:01 -0600 Subject: [PATCH 091/122] Updated userInit values. --- .../templates/_helpers.tpl | 26 ++++++++++++------- .../templates/user-init.yaml | 4 +-- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/charts/osdu-developer-init/templates/_helpers.tpl b/charts/osdu-developer-init/templates/_helpers.tpl index 3ec562c2..df218aae 100644 --- a/charts/osdu-developer-init/templates/_helpers.tpl +++ b/charts/osdu-developer-init/templates/_helpers.tpl @@ -55,20 +55,28 @@ app.kubernetes.io/instance: {{ .Release.Name }} Determine if the installation type is enabled */}} {{- define "osdu-developer-init.isEnabled" -}} - {{- $installationType := .Values.installationType | default "osduCore" -}} - {{- if eq $installationType "osduReference" -}} - {{- if hasKey .Values "osduReferenceEnabled" -}} - {{- if eq .Values.osduReferenceEnabled "true" }}1{{else}}0{{end -}} + {{- if hasKey .Values "installationType" -}} + {{- $installationType := .Values.installationType | default "osduCore" -}} + {{- if eq $installationType "osduReference" -}} + {{- if hasKey .Values "osduReferenceEnabled" -}} + {{- if eq .Values.osduReferenceEnabled "true" }}1{{else}}0{{end -}} + {{- else -}} + {{- 0 -}} + {{- end -}} + {{- else if eq $installationType "osduCore" -}} + {{- if hasKey .Values "osduCoreEnabled" -}} + {{- if eq .Values.osduCoreEnabled "true" }}1{{else}}0{{end -}} + {{- else -}} + {{- 0 -}} + {{- end -}} {{- else -}} {{- 0 -}} {{- end -}} - {{- else if eq $installationType "osduCore" -}} - {{- if hasKey .Values "osduCoreEnabled" -}} - {{- if eq .Values.osduCoreEnabled "true" }}1{{else}}0{{end -}} + {{- else -}} + {{- if and (hasKey .Values "jobs") (hasKey .Values.jobs "userInit") -}} + {{- if eq .Values.jobs.userInit true }}1{{else}}0{{end -}} {{- else -}} {{- 0 -}} {{- end -}} - {{- else -}} - {{- 0 -}} {{- end -}} {{- end }} \ No newline at end of file diff --git a/charts/osdu-developer-init/templates/user-init.yaml b/charts/osdu-developer-init/templates/user-init.yaml index 604f899c..bd1ebc7b 100644 --- a/charts/osdu-developer-init/templates/user-init.yaml +++ b/charts/osdu-developer-init/templates/user-init.yaml @@ -1,7 +1,7 @@ {{- $enabled := eq (include "osdu-developer-init.isEnabled" .) "1" -}} {{- $namespace := .Release.Namespace -}} -{{- $releaseName := .Release.Name -}} # Capture the release name -{{- if $enabled }} +{{- $releaseName := .Release.Name -}} +{{- if and $enabled .Values.jobs.userInit }} --- apiVersion: batch/v1 kind: Job From 9bea6ebd76c5aeb06101c21c7efe3fc240635745 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sat, 4 Jan 2025 14:39:44 -0600 Subject: [PATCH 092/122] Added sp user init. --- .../applications/osdu-core/user-init.yaml | 86 ++++++++----------- 1 file changed, 35 insertions(+), 51 deletions(-) diff --git a/software/applications/osdu-core/user-init.yaml b/software/applications/osdu-core/user-init.yaml index e5e78b02..33ac5b02 100644 --- a/software/applications/osdu-core/user-init.yaml +++ b/software/applications/osdu-core/user-init.yaml @@ -34,54 +34,38 @@ spec: name: configmap-services targetPath: emailAddress valuesKey: first_user_id -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-user-sp -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-entitlements-init -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# installationType: osduCore -# jobs: -# partitionInit: false -# entitlementInit: false -# userInit: true -# elasticInit: false -# schemaInit: false -# clientSecret: -# name: active-directory -# key: principal-clientpassword -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: clientId -# valuesKey: client_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: tenantId -# valuesKey: tenant_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: emailAddress -# valuesKey: client_id \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-user-sp + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-entitlements-init + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + jobs: + userInit: true + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: emailAddress + valuesKey: client_id \ No newline at end of file From b37c2b9135c92df7cfa754966b42a77b04fc7006 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 5 Jan 2025 18:23:30 -0600 Subject: [PATCH 093/122] Rework Envoy Filter. --- charts/osdu-developer-base/Chart.yaml | 2 +- .../templates/envoy-filter.yaml | 244 +++++++----------- 2 files changed, 97 insertions(+), 149 deletions(-) diff --git a/charts/osdu-developer-base/Chart.yaml b/charts/osdu-developer-base/Chart.yaml index 9150f73b..faf81a66 100644 --- a/charts/osdu-developer-base/Chart.yaml +++ b/charts/osdu-developer-base/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-base type: application description: Installs the OSDU developer Base Components -version: 0.0.3 +version: 0.0.4 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index 79e4713b..190e61f1 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -28,161 +28,109 @@ spec: typed_config: "@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua" inlineCode: | - --[[ Token Scenarios Handled: - 1. AAD v1 User Token (sts.windows.net): - - With OID: x-user-id = oid - - Without OID: x-user-id = upn/unique_name + -- Constants + local AAD_V1_ISSUER = "sts.windows.net" + local AAD_V2_ISSUER = "login.microsoftonline.com" - 2. AAD v1 Service-to-Service Delegation: - - Service identity (appid) matches managedIdentityClientId - - Delegation flow: x-user-id = x-on-behalf-of (delegated user) - - Fallback: x-user-id = appid (direct service call) - - 3. AAD v1 Application (non-delegated): - - x-user-id = appid - - 4. AAD v2 User Token (login.microsoftonline.com): - - Has 'scp' claim: x-user-id = oid - - 5. AAD v2 Service-to-Service Delegation: - - Service identity (azp) matches managedIdentityClientId - - Delegation flow: x-user-id = x-on-behalf-of (delegated user) - - Fallback: x-user-id = azp (direct service call) - - 6. AAD v2 Application (non-delegated): - - x-user-id = azp or oid - - Note: All scenarios set x-app-id from 'aud' claim when present - - OAuth Delegation (On-Behalf-Of) Flow: - - When a service calls another service on behalf of a user - - The original user's identity is passed via x-on-behalf-of header - - Maintains user context through the service chain - - Enables proper authorization based on user identity - ]] - - function envoy_on_request(request_handle) - -- Add initial debug log - request_handle:logDebug("Starting envoy_on_request processing") - - -- Remove headers first - request_handle:headers():remove("x-user-id") - request_handle:headers():remove("x-app-id") - request_handle:logInfo("x-user-id and x-app-id headers removed") - - -- Get JWT metadata with debug logging - local meta = request_handle:streamInfo():dynamicMetadata():get("envoy.filters.http.jwt_authn") - if meta then - request_handle:logDebug("JWT metadata found") + -- Helper function to log a table + function tableToString(tbl, indent) + if not indent then indent = 0 end + if type(tbl) ~= 'table' then return tostring(tbl) end + local lines = {} + for k, v in pairs(tbl) do + local formatting = string.rep(" ", indent) .. k .. ": " + if type(v) == "table" then + table.insert(lines, formatting) + table.insert(lines, tableToString(v, indent+1)) else - request_handle:logDebug("No JWT metadata found") - return + table.insert(lines, formatting .. tostring(v)) end + end + return table.concat(lines, "\n") + end - if meta["payload"] then - request_handle:logDebug("JWT payload found") - local payload = meta["payload"] - -- Log the raw payload for debugging - request_handle:logDebug("Raw JWT payload: " .. tableToString(payload)) - - -- Always set x-app-id from aud claim if present - if payload["aud"] then - request_handle:headers():add("x-app-id", payload["aud"]) - request_handle:logWarn("x-app-id set from 'aud' claim: " .. payload["aud"]) - end - - -- Check issuer - if string.find(payload["iss"], "sts.windows.net") then - -- AAD v1 token handling - if payload["upn"] then - -- Scenario 1: AAD v1 User Token - if payload["oid"] then - request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logWarn("x-user-id set from 'oid' claim with upn present") - else - request_handle:headers():add("x-user-id", payload["upn"]) - request_handle:logWarn("x-user-id set from 'upn' claim") - end - elseif payload["unique_name"] then - -- Scenario 1: AAD v1 User Token (alternate claim) - if payload["oid"] then - request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logWarn("x-user-id set from 'oid' claim with unique_name present") - else - request_handle:headers():add("x-user-id", payload["unique_name"]) - request_handle:logWarn("x-user-id set from 'unique_name' claim") - end - elseif payload["appid"] then - -- Check for service identity - if payload["appid"] == "{{ $managedIdentityClientId }}" then - -- Scenario 2: AAD v1 Service-to-Service Delegation - local on_behalf_of = request_handle:headers():get("x-on-behalf-of") - if on_behalf_of and on_behalf_of ~= "" then - request_handle:headers():add("x-user-id", on_behalf_of) - request_handle:logWarn("x-user-id set from on-behalf-of header (delegation flow)") - else - request_handle:headers():add("x-user-id", payload["appid"]) - request_handle:logWarn("x-user-id set from appid (direct service call)") - end - else - -- Scenario 3: AAD v1 Application Token - request_handle:headers():add("x-user-id", payload["appid"]) - request_handle:logWarn("x-user-id set from 'appid' claim (non-delegated)") - end - end - - elseif string.find(payload["iss"], "login.microsoftonline.com") then - -- AAD v2 token handling - if payload["scp"] then - -- Scenario 4: AAD v2 User Token - if payload["oid"] then - request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logWarn("x-user-id set from 'oid' claim with scp present") - end - else - if payload["azp"] then - if payload["azp"] == "{{ $managedIdentityClientId }}" then - -- Scenario 5: AAD v2 Service-to-Service Delegation - local on_behalf_of = request_handle:headers():get("x-on-behalf-of") - if on_behalf_of and on_behalf_of ~= "" then - request_handle:headers():add("x-user-id", on_behalf_of) - request_handle:logWarn("x-user-id set from on-behalf-of header (v2 delegation flow)") - else - request_handle:headers():add("x-user-id", payload["azp"]) - request_handle:logWarn("x-user-id set from azp (direct service call)") - end - else - -- Scenario 6: AAD v2 Application Token - request_handle:headers():add("x-user-id", payload["azp"]) - request_handle:logWarn("x-user-id set from 'azp' claim (non-delegated)") - end - elseif payload["oid"] then - request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logWarn("x-user-id set from 'oid' claim") - end - end - else - request_handle:logError("Issuer does not match known issuers") - end + -- Process AAD v1 tokens + function processAADV1Token(payload, request_handle, managedIdentityClientId) + if payload["upn"] or payload["unique_name"] then + if payload["oid"] then + request_handle:headers():add("x-user-id", payload["oid"]) + request_handle:logWarn("x-user-id set from 'oid' claim") + else + local fallback = payload["upn"] or payload["unique_name"] + request_handle:headers():add("x-user-id", fallback) + request_handle:logWarn("x-user-id set from fallback claim: " .. fallback) + end + elseif payload["appid"] then + if payload["appid"] == managedIdentityClientId then + local on_behalf_of = request_handle:headers():get("x-on-behalf-of") + if on_behalf_of and on_behalf_of ~= "" then + request_handle:headers():add("x-user-id", on_behalf_of) + request_handle:logWarn("x-user-id set from on-behalf-of header") + else + request_handle:headers():add("x-user-id", payload["appid"]) + request_handle:logWarn("x-user-id set from appid claim") + end else - request_handle:logDebug("No JWT payload in metadata") - return + request_handle:headers():add("x-user-id", payload["appid"]) + request_handle:logWarn("x-user-id set from non-delegated appid claim") end + end end - -- Helper function to convert a table to a string for logging - function tableToString(tbl, indent) - if not indent then indent = 0 end - if type(tbl) ~= 'table' then return tostring(tbl) end - local lines = {} - for k, v in pairs(tbl) do - local formatting = string.rep(" ", indent) .. k .. ": " - if type(v) == "table" then - table.insert(lines, formatting) - table.insert(lines, tableToString(v, indent+1)) - else - table.insert(lines, formatting .. tostring(v)) - end + -- Process AAD v2 tokens + function processAADV2Token(payload, request_handle, managedIdentityClientId) + if payload["scp"] and payload["oid"] then + request_handle:headers():add("x-user-id", payload["oid"]) + request_handle:logWarn("x-user-id set from 'oid' claim with scp present") + elseif payload["azp"] then + if payload["azp"] == managedIdentityClientId then + local on_behalf_of = request_handle:headers():get("x-on-behalf-of") + if on_behalf_of and on_behalf_of ~= "" then + request_handle:headers():add("x-user-id", on_behalf_of) + request_handle:logWarn("x-user-id set from on-behalf-of header") + else + request_handle:headers():add("x-user-id", payload["azp"]) + request_handle:logWarn("x-user-id set from azp claim") + end + else + request_handle:headers():add("x-user-id", payload["azp"]) + request_handle:logWarn("x-user-id set from azp claim (non-delegated)") end - return table.concat(lines, "\n") + end + end + + -- Main processing function + function envoy_on_request(request_handle) + -- Step 1: Remove existing headers + request_handle:headers():remove("x-user-id") + request_handle:headers():remove("x-app-id") + request_handle:logInfo("x-user-id and x-app-id headers removed") + + -- Step 2: Retrieve JWT metadata + local meta = request_handle:streamInfo():dynamicMetadata():get("envoy.filters.http.jwt_authn") + if not meta or not meta["payload"] then + request_handle:logError("No JWT metadata or payload found") + return + end + local payload = meta["payload"] + + -- Step 3: Log raw payload for debugging + request_handle:logDebug("JWT payload: " .. tableToString(payload)) + + -- Step 4: Always set x-app-id from aud claim if present + if payload["aud"] then + request_handle:headers():add("x-app-id", payload["aud"]) + request_handle:logWarn("x-app-id set from 'aud' claim: " .. payload["aud"]) + end + + -- Step 5: Process based on issuer + if string.find(payload["iss"], AAD_V1_ISSUER) then + request_handle:logDebug("Processing AAD v1 token") + processAADV1Token(payload, request_handle, "{{ $managedIdentityClientId }}") + elseif string.find(payload["iss"], AAD_V2_ISSUER) then + request_handle:logDebug("Processing AAD v2 token") + processAADV2Token(payload, request_handle, "{{ $managedIdentityClientId }}") + else + request_handle:logError("Unknown issuer: " .. payload["iss"]) + end end \ No newline at end of file From 7f17e79bd79448ff714e182f4e864c227bd5c068 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 5 Jan 2025 19:02:41 -0600 Subject: [PATCH 094/122] Work fix for possible issuer problem. --- charts/osdu-developer-base/Chart.yaml | 2 +- charts/osdu-developer-base/envoy-filter.md | 20 +++++++++++++++++++ .../templates/request-authentication.yaml | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 charts/osdu-developer-base/envoy-filter.md diff --git a/charts/osdu-developer-base/Chart.yaml b/charts/osdu-developer-base/Chart.yaml index faf81a66..6c1c9d1d 100644 --- a/charts/osdu-developer-base/Chart.yaml +++ b/charts/osdu-developer-base/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-base type: application description: Installs the OSDU developer Base Components -version: 0.0.4 +version: 0.0.5 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/osdu-developer-base/envoy-filter.md b/charts/osdu-developer-base/envoy-filter.md new file mode 100644 index 00000000..fb59e6dc --- /dev/null +++ b/charts/osdu-developer-base/envoy-filter.md @@ -0,0 +1,20 @@ +```mermaid +graph TD + A[Start: Incoming Request] --> B[Step 1: Remove Headers] + B --> C[Step 2: Retrieve JWT Metadata] + C -->|Metadata Found| D[Step 3: Log Payload] + C -->|No Metadata Found| E[End: Request Processing Halted] + D --> F[Step 4: Set x-app-id from 'aud'] + F --> G{Step 5: Check Issuer} + G -->|Issuer: AAD v1 sts.windows.net| H[Process AAD v1 Token] + G -->|Issuer: AAD v2 login.microsoftonline.com| I[Process AAD v2 Token] + G -->|Unknown Issuer| J[Log Error: Unknown Issuer] + H --> H1[Set x-user-id using 'oid', fallback to 'upn' or 'unique_name'] + H1 --> K[Log Headers After AAD v1 Processing] + I --> I1[Set x-user-id using 'oid' or 'azp'] + I1 --> I2[Handle Delegation: Use x-on-behalf-of if Applicable] + I2 --> K[Log Headers After AAD v2 Processing] + J --> E + K --> L[Request Headers Modified] + L --> M[Request Forwarded] +``` \ No newline at end of file diff --git a/charts/osdu-developer-base/templates/request-authentication.yaml b/charts/osdu-developer-base/templates/request-authentication.yaml index af1b2dc1..f2366b7a 100644 --- a/charts/osdu-developer-base/templates/request-authentication.yaml +++ b/charts/osdu-developer-base/templates/request-authentication.yaml @@ -10,7 +10,7 @@ metadata: spec: jwtRules: - issuer: "https://sts.windows.net/{{ $tenantId }}/" - jwksUri: "https://login.microsoftonline.com:443/common/discovery/v2.0/keys" + jwksUri: "https://login.microsoftonline.com/common/discovery/v2.0/keys" audiences: - "{{ $appId }}" - "{{ $clientId }}" From 4b70f8f0eadedd798a893d4cab63dace38147d46 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 5 Jan 2025 19:19:42 -0600 Subject: [PATCH 095/122] Envoy Filter fixes --- charts/osdu-developer-base/Chart.yaml | 2 +- .../templates/envoy-filter.yaml | 24 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/charts/osdu-developer-base/Chart.yaml b/charts/osdu-developer-base/Chart.yaml index 6c1c9d1d..27bb298b 100644 --- a/charts/osdu-developer-base/Chart.yaml +++ b/charts/osdu-developer-base/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-base type: application description: Installs the OSDU developer Base Components -version: 0.0.5 +version: 0.0.6 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index 190e61f1..c51c303f 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -51,15 +51,13 @@ spec: -- Process AAD v1 tokens function processAADV1Token(payload, request_handle, managedIdentityClientId) - if payload["upn"] or payload["unique_name"] then - if payload["oid"] then - request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logWarn("x-user-id set from 'oid' claim") - else - local fallback = payload["upn"] or payload["unique_name"] - request_handle:headers():add("x-user-id", fallback) - request_handle:logWarn("x-user-id set from fallback claim: " .. fallback) - end + if payload["oid"] then + request_handle:headers():add("x-user-id", payload["oid"]) + request_handle:logWarn("x-user-id set from 'oid' claim") + elseif payload["upn"] or payload["unique_name"] then + local fallback = payload["upn"] or payload["unique_name"] + request_handle:headers():add("x-user-id", fallback) + request_handle:logWarn("x-user-id set from fallback claim: " .. fallback) elseif payload["appid"] then if payload["appid"] == managedIdentityClientId then local on_behalf_of = request_handle:headers():get("x-on-behalf-of") @@ -74,14 +72,16 @@ spec: request_handle:headers():add("x-user-id", payload["appid"]) request_handle:logWarn("x-user-id set from non-delegated appid claim") end + else + request_handle:logError("No valid claim for x-user-id found in AAD v1 token") end end -- Process AAD v2 tokens function processAADV2Token(payload, request_handle, managedIdentityClientId) - if payload["scp"] and payload["oid"] then + if payload["oid"] then request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logWarn("x-user-id set from 'oid' claim with scp present") + request_handle:logWarn("x-user-id set from 'oid' claim") elseif payload["azp"] then if payload["azp"] == managedIdentityClientId then local on_behalf_of = request_handle:headers():get("x-on-behalf-of") @@ -96,6 +96,8 @@ spec: request_handle:headers():add("x-user-id", payload["azp"]) request_handle:logWarn("x-user-id set from azp claim (non-delegated)") end + else + request_handle:logError("No valid claim for x-user-id found in AAD v2 token") end end From 973bf02e026be43814cedb38f9a2976727868f42 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 5 Jan 2025 19:53:59 -0600 Subject: [PATCH 096/122] Envoy Filter fixes --- charts/osdu-developer-base/Chart.yaml | 2 +- charts/osdu-developer-base/envoy-filter.md | 48 ++++++++++++++++++- .../templates/envoy-filter.yaml | 13 +++-- 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/charts/osdu-developer-base/Chart.yaml b/charts/osdu-developer-base/Chart.yaml index 27bb298b..b3f7b379 100644 --- a/charts/osdu-developer-base/Chart.yaml +++ b/charts/osdu-developer-base/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-base type: application description: Installs the OSDU developer Base Components -version: 0.0.6 +version: 0.0.7 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/osdu-developer-base/envoy-filter.md b/charts/osdu-developer-base/envoy-filter.md index fb59e6dc..fa9f622f 100644 --- a/charts/osdu-developer-base/envoy-filter.md +++ b/charts/osdu-developer-base/envoy-filter.md @@ -1,3 +1,37 @@ +# Microsoft Identity Filter for Istio Envoy + +This contains the configuration and logic for an EnvoyFilter that processes Microsoft Azure Active Directory (AAD) tokens. The filter handles both AAD v1 and AAD v2 tokens to set well-known headers for downstream services, enabling proper identity and authorization context propagation. + +## Features + +- **Header Removal**: Ensures `x-user-id` and `x-app-id` headers are reset at the start of request processing. +- **Token Support**: Processes AAD v1 (sts.windows.net) and AAD v2 (login.microsoftonline.com) tokens. +- **Delegation Handling**: Supports OAuth delegation via `x-on-behalf-of` header. +- **Flexible Issuer Recognition**: Handles tokens from multiple issuers without assuming a fixed tenant ID. +- **Dynamic Metadata Logging**: Logs JWT payload for debugging and troubleshooting. +- **Error Handling**: Logs detailed errors for malformed tokens or unknown issuers. + +## Token Scenarios Handled + +| Scenario | x-user-id | x-app-id | +|--------------------------------------------|-------------------------------------|-----------------------------------| +| **AAD v1 User Token (sts.windows.net)** | `oid` (fallback: `upn`/`unique_name`) | `aud` | +| **AAD v1 Service-to-Service Delegation** | `x-on-behalf-of` (fallback: `appid`) | `aud` | +| **AAD v1 Application (non-delegated)** | `appid` | `aud` | +| **AAD v2 User Token (login.microsoftonline.com)** | `oid` | `aud` | +| **AAD v2 Service-to-Service Delegation** | `x-on-behalf-of` (fallback: `azp`) | `aud` | +| **AAD v2 Application (non-delegated)** | `azp` (fallback: `oid`) | `aud` | + +## OAuth Delegation (On-Behalf-Of) Flow + +- Enables a service to act on behalf of a user in a multi-service architecture. +- Uses `x-on-behalf-of` header to maintain user identity through the service chain. +- Sets `x-user-id` based on the original user's identity. + +## Flow Diagram + +The following diagram illustrates the logical flow of the EnvoyFilter: + ```mermaid graph TD A[Start: Incoming Request] --> B[Step 1: Remove Headers] @@ -17,4 +51,16 @@ graph TD J --> E K --> L[Request Headers Modified] L --> M[Request Forwarded] -``` \ No newline at end of file +``` + +## Debugging and Logging + +### Increase Logging Level +Use the following Istio commands to increase the logging level for debugging: + +```bash +# Enable detailed logging for Lua, JWT, and RBAC +istioctl proxy-config log --level lua:debug +istioctl proxy-config log --level jwt:debug +istioctl proxy-config log --level rbac:debug +``` diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index c51c303f..e86e7772 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -3,13 +3,12 @@ # istioctl proxy-config log --level rbac:debug {{- $namespace := .Release.Namespace }} -{{- $managedIdentityClientId := .Values.azure.clientId }} apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: microsoft-identity-filter namespace: {{ $namespace }} + name: microsoft-identity-filter spec: configPatches: - applyTo: HTTP_FILTER @@ -24,7 +23,7 @@ spec: patch: operation: INSERT_BEFORE value: - name: envoy.lua.remove-user-appid-header + name: envoy.lua.microsoft-identity-filter typed_config: "@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua" inlineCode: | @@ -41,7 +40,7 @@ spec: local formatting = string.rep(" ", indent) .. k .. ": " if type(v) == "table" then table.insert(lines, formatting) - table.insert(lines, tableToString(v, indent+1)) + table.insert(lines, tableToString(v, indent + 1)) else table.insert(lines, formatting .. tostring(v)) end @@ -128,11 +127,11 @@ spec: -- Step 5: Process based on issuer if string.find(payload["iss"], AAD_V1_ISSUER) then request_handle:logDebug("Processing AAD v1 token") - processAADV1Token(payload, request_handle, "{{ $managedIdentityClientId }}") + processAADV1Token(payload, request_handle, "{{ .Values.azure.clientId }}") elseif string.find(payload["iss"], AAD_V2_ISSUER) then request_handle:logDebug("Processing AAD v2 token") - processAADV2Token(payload, request_handle, "{{ $managedIdentityClientId }}") + processAADV2Token(payload, request_handle, "{{ .Values.azure.clientId }}") else request_handle:logError("Unknown issuer: " .. payload["iss"]) end - end \ No newline at end of file + end From 35204c08b19dbd6f838610f0e391839c52d5e27b Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 5 Jan 2025 20:07:46 -0600 Subject: [PATCH 097/122] Envoy Filter fixes --- .../templates/envoy-filter.yaml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index e86e7772..0e270306 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -1,8 +1,5 @@ -# istioctl proxy-config log --level lua:debug -# istioctl proxy-config log --level jwt:debug -# istioctl proxy-config log --level rbac:debug - {{- $namespace := .Release.Namespace }} +{{- $managedIdentityClientId := .Values.azure.clientId }} apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter @@ -30,6 +27,7 @@ spec: -- Constants local AAD_V1_ISSUER = "sts.windows.net" local AAD_V2_ISSUER = "login.microsoftonline.com" + local managedIdentityClientId = "{{ $managedIdentityClientId }}" -- Helper function to log a table function tableToString(tbl, indent) @@ -49,7 +47,7 @@ spec: end -- Process AAD v1 tokens - function processAADV1Token(payload, request_handle, managedIdentityClientId) + function processAADV1Token(payload, request_handle) if payload["oid"] then request_handle:headers():add("x-user-id", payload["oid"]) request_handle:logWarn("x-user-id set from 'oid' claim") @@ -77,7 +75,7 @@ spec: end -- Process AAD v2 tokens - function processAADV2Token(payload, request_handle, managedIdentityClientId) + function processAADV2Token(payload, request_handle) if payload["oid"] then request_handle:headers():add("x-user-id", payload["oid"]) request_handle:logWarn("x-user-id set from 'oid' claim") @@ -127,11 +125,11 @@ spec: -- Step 5: Process based on issuer if string.find(payload["iss"], AAD_V1_ISSUER) then request_handle:logDebug("Processing AAD v1 token") - processAADV1Token(payload, request_handle, "{{ .Values.azure.clientId }}") + processAADV1Token(payload, request_handle) elseif string.find(payload["iss"], AAD_V2_ISSUER) then request_handle:logDebug("Processing AAD v2 token") - processAADV2Token(payload, request_handle, "{{ .Values.azure.clientId }}") + processAADV2Token(payload, request_handle) else request_handle:logError("Unknown issuer: " .. payload["iss"]) end - end + end \ No newline at end of file From f771f16af4d0a3cc01de53d1e9ba2c690f395130 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 5 Jan 2025 20:08:09 -0600 Subject: [PATCH 098/122] Envoy Filter fixes --- charts/osdu-developer-base/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-base/Chart.yaml b/charts/osdu-developer-base/Chart.yaml index b3f7b379..e646bd35 100644 --- a/charts/osdu-developer-base/Chart.yaml +++ b/charts/osdu-developer-base/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-base type: application description: Installs the OSDU developer Base Components -version: 0.0.7 +version: 0.0.8 appVersion: 0.0.1 maintainers: - name: danielscholl From 579b91a37d74131d0f2365904f6b69cb6ae8345d Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 5 Jan 2025 20:15:07 -0600 Subject: [PATCH 099/122] Envoy Filter fixes --- charts/osdu-developer-base/Chart.yaml | 2 +- .../osdu-developer-base/templates/envoy-filter.yaml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/charts/osdu-developer-base/Chart.yaml b/charts/osdu-developer-base/Chart.yaml index e646bd35..96bbc437 100644 --- a/charts/osdu-developer-base/Chart.yaml +++ b/charts/osdu-developer-base/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-base type: application description: Installs the OSDU developer Base Components -version: 0.0.8 +version: 0.0.9 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index 0e270306..b21f55c5 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -46,6 +46,14 @@ spec: return table.concat(lines, "\n") end + -- Function to log all headers + function logAllHeaders(request_handle) + local headers = request_handle:headers() + for key, value in pairs(headers) do + request_handle:logInfo("Header: " .. key .. " = " .. value) + end + end + -- Process AAD v1 tokens function processAADV1Token(payload, request_handle) if payload["oid"] then @@ -132,4 +140,8 @@ spec: else request_handle:logError("Unknown issuer: " .. payload["iss"]) end + + -- Step 6: Log all headers before leaving the filter + request_handle:logDebug("Logging all headers before leaving the filter:") + logAllHeaders(request_handle) end \ No newline at end of file From bbfc99f5eafdaf53a1cbc4e970d228b815d4bd9a Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 5 Jan 2025 20:26:15 -0600 Subject: [PATCH 100/122] Envoy Filter fixes --- .../templates/envoy-filter.yaml | 50 ++++++------------- 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index b21f55c5..1339916f 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -56,27 +56,15 @@ spec: -- Process AAD v1 tokens function processAADV1Token(payload, request_handle) - if payload["oid"] then + if payload["unique_name"] then + request_handle:headers():add("x-user-id", payload["unique_name"]) + request_handle:logWarn("x-user-id set from 'unique_name' claim") + elseif payload["oid"] then request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logWarn("x-user-id set from 'oid' claim") - elseif payload["upn"] or payload["unique_name"] then - local fallback = payload["upn"] or payload["unique_name"] - request_handle:headers():add("x-user-id", fallback) - request_handle:logWarn("x-user-id set from fallback claim: " .. fallback) - elseif payload["appid"] then - if payload["appid"] == managedIdentityClientId then - local on_behalf_of = request_handle:headers():get("x-on-behalf-of") - if on_behalf_of and on_behalf_of ~= "" then - request_handle:headers():add("x-user-id", on_behalf_of) - request_handle:logWarn("x-user-id set from on-behalf-of header") - else - request_handle:headers():add("x-user-id", payload["appid"]) - request_handle:logWarn("x-user-id set from appid claim") - end - else - request_handle:headers():add("x-user-id", payload["appid"]) - request_handle:logWarn("x-user-id set from non-delegated appid claim") - end + request_handle:logWarn("x-user-id set from 'oid' claim as fallback") + elseif payload["upn"] then + request_handle:headers():add("x-user-id", payload["upn"]) + request_handle:logWarn("x-user-id set from 'upn' claim as fallback") else request_handle:logError("No valid claim for x-user-id found in AAD v1 token") end @@ -84,23 +72,15 @@ spec: -- Process AAD v2 tokens function processAADV2Token(payload, request_handle) - if payload["oid"] then + if payload["unique_name"] then + request_handle:headers():add("x-user-id", payload["unique_name"]) + request_handle:logWarn("x-user-id set from 'unique_name' claim") + elseif payload["oid"] then request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logWarn("x-user-id set from 'oid' claim") + request_handle:logWarn("x-user-id set from 'oid' claim as fallback") elseif payload["azp"] then - if payload["azp"] == managedIdentityClientId then - local on_behalf_of = request_handle:headers():get("x-on-behalf-of") - if on_behalf_of and on_behalf_of ~= "" then - request_handle:headers():add("x-user-id", on_behalf_of) - request_handle:logWarn("x-user-id set from on-behalf-of header") - else - request_handle:headers():add("x-user-id", payload["azp"]) - request_handle:logWarn("x-user-id set from azp claim") - end - else - request_handle:headers():add("x-user-id", payload["azp"]) - request_handle:logWarn("x-user-id set from azp claim (non-delegated)") - end + request_handle:headers():add("x-user-id", payload["azp"]) + request_handle:logWarn("x-user-id set from 'azp' claim as fallback") else request_handle:logError("No valid claim for x-user-id found in AAD v2 token") end From c88e0b04244db249e01877035a11be7d67e92069 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 5 Jan 2025 20:26:31 -0600 Subject: [PATCH 101/122] Envoy Filter fixes --- charts/osdu-developer-base/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-base/Chart.yaml b/charts/osdu-developer-base/Chart.yaml index 96bbc437..f57d4b33 100644 --- a/charts/osdu-developer-base/Chart.yaml +++ b/charts/osdu-developer-base/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-base type: application description: Installs the OSDU developer Base Components -version: 0.0.9 +version: 0.0.10 appVersion: 0.0.1 maintainers: - name: danielscholl From f5ab91dd7ed00af7e6fd4a243225d1a45f9ecc3b Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 10:00:00 -0600 Subject: [PATCH 102/122] Removed init users. --- .../applications/osdu-core/user-init.yaml | 142 +++++++++--------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/software/applications/osdu-core/user-init.yaml b/software/applications/osdu-core/user-init.yaml index 33ac5b02..cfb542d0 100644 --- a/software/applications/osdu-core/user-init.yaml +++ b/software/applications/osdu-core/user-init.yaml @@ -1,71 +1,71 @@ ---- -# kubectl create job --namespace osdu-core --from=cronjob/user-init user-init-run-$(date +%s) -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-user - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-entitlements-init - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - jobs: - userInit: true - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: emailAddress - valuesKey: first_user_id ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-user-sp - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-entitlements-init - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - jobs: - userInit: true - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: emailAddress - valuesKey: client_id \ No newline at end of file +# --- +# # kubectl create job --namespace osdu-core --from=cronjob/user-init user-init-run-$(date +%s) +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-user +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-entitlements-init +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# jobs: +# userInit: true +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: emailAddress +# valuesKey: first_user_id +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-user-sp +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-entitlements-init +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# jobs: +# userInit: true +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: emailAddress +# valuesKey: client_id \ No newline at end of file From f8b89687005b89f27392c4556b7e712338984a31 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 10:05:01 -0600 Subject: [PATCH 103/122] Debug entitlements init --- .../applications/osdu-core/entitlements.yaml | 98 +++++++++---------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index b14e7ca3..a7e68b43 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -113,52 +113,52 @@ spec: - name: PARTITION_SERVICE_ENDPOINT value: "http://partition/api/partition/v1" --- -# Retrigger: kubectl annotate helmrelease osdu-init-entitlements fluxcd.io/retrigger=$(date +%s) -n osdu-core -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-entitlements-init - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" - fluxcd.io/retrigger: "initial" -spec: - dependsOn: - - name: osdu-entitlements - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - installationType: osduCore - jobs: - partitionInit: false - entitlementInit: true - userInit: false - elasticInit: false - schemaInit: false - partition: opendes - clientSecret: - name: active-directory - key: principal-clientpassword - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id +# Retrigger: kubectl annotate helmrelease osdu-entitlements-init fluxcd.io/retrigger=$(date +%s) -n osdu-core +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-entitlements-init +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# fluxcd.io/retrigger: "initial" +# spec: +# dependsOn: +# - name: osdu-entitlements +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# installationType: osduCore +# jobs: +# partitionInit: false +# entitlementInit: true +# userInit: false +# elasticInit: false +# schemaInit: false +# partition: opendes +# clientSecret: +# name: active-directory +# key: principal-clientpassword +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: clientId +# valuesKey: client_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: tenantId +# valuesKey: tenant_id From 9e110cc5fc4cf612e92e243c4d7703111bec1f60 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 10:09:23 -0600 Subject: [PATCH 104/122] Debug entitlements init --- .../applications/osdu-core/entitlements.yaml | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/software/applications/osdu-core/entitlements.yaml b/software/applications/osdu-core/entitlements.yaml index a7e68b43..8aba1e1d 100644 --- a/software/applications/osdu-core/entitlements.yaml +++ b/software/applications/osdu-core/entitlements.yaml @@ -114,51 +114,51 @@ spec: value: "http://partition/api/partition/v1" --- # Retrigger: kubectl annotate helmrelease osdu-entitlements-init fluxcd.io/retrigger=$(date +%s) -n osdu-core -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-entitlements-init -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# fluxcd.io/retrigger: "initial" -# spec: -# dependsOn: -# - name: osdu-entitlements -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# installationType: osduCore -# jobs: -# partitionInit: false -# entitlementInit: true -# userInit: false -# elasticInit: false -# schemaInit: false -# partition: opendes -# clientSecret: -# name: active-directory -# key: principal-clientpassword -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: clientId -# valuesKey: client_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: tenantId -# valuesKey: tenant_id +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-entitlements-init + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" + fluxcd.io/retrigger: "initial" +spec: + dependsOn: + - name: osdu-entitlements + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + installationType: osduCore + jobs: + partitionInit: false + entitlementInit: true + userInit: false + elasticInit: false + schemaInit: false + partition: opendes + clientSecret: + name: active-directory + key: principal-clientpassword + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: clientId + valuesKey: client_id + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id From 8880f2e358dd008b491c526013030582bbf0d8eb Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 10:26:49 -0600 Subject: [PATCH 105/122] Debug entitlements init --- charts/osdu-developer-init/Chart.yaml | 2 +- charts/osdu-developer-init/templates/entitlement-init.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/osdu-developer-init/Chart.yaml b/charts/osdu-developer-init/Chart.yaml index af8bcbbf..2f97a4f5 100644 --- a/charts/osdu-developer-init/Chart.yaml +++ b/charts/osdu-developer-init/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-init type: application description: Installs the OSDU developer Initialization resources -version: 0.0.1 +version: 0.0.2 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/osdu-developer-init/templates/entitlement-init.yaml b/charts/osdu-developer-init/templates/entitlement-init.yaml index e0f83f38..1a495080 100644 --- a/charts/osdu-developer-init/templates/entitlement-init.yaml +++ b/charts/osdu-developer-init/templates/entitlement-init.yaml @@ -70,7 +70,7 @@ data: -t ${AZURE_TENANT_ID} # Get token (no resource needed) - TOKEN=$(az account get-access-token --resource "https://management.azure.com/" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "api://${AZURE_CLIENT_ID}" --query accessToken -o tsv) OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://entitlements.{{ $namespace }}/api/entitlements/v2/tenant-provisioning \ From d6b4297170111568e1e814691687ac08e3beee61 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 10:31:09 -0600 Subject: [PATCH 106/122] Debug --- software/applications/osdu-core/file.yaml | 276 +++++------ software/applications/osdu-core/indexer.yaml | 452 +++++++++--------- software/applications/osdu-core/legal.yaml | 244 +++++----- software/applications/osdu-core/schema.yaml | 334 ++++++------- software/applications/osdu-core/search.yaml | 242 +++++----- software/applications/osdu-core/storage.yaml | 276 +++++------ software/applications/osdu-core/workflow.yaml | 390 +++++++-------- 7 files changed, 1107 insertions(+), 1107 deletions(-) diff --git a/software/applications/osdu-core/file.yaml b/software/applications/osdu-core/file.yaml index 34a020b1..7946c52f 100644 --- a/software/applications/osdu-core/file.yaml +++ b/software/applications/osdu-core/file.yaml @@ -1,138 +1,138 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-file - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-indexer-queue - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: file - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: file - path: /api/file/ - hosts: - - "*" - cors: - - "http://localhost:8080" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/system/file/file- - branch: release-0-26 - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - request: - cpu: 1000m - memory: 1Gi - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/file/v2/info" - - "/api/file/v2/swagger*" - - "/api/file/v2/api-docs*" - - "/api/file/v2/webjars/*" - env: - - name: KEYVAULT_URL - secret: - name: azure-resources - key: keyvault-uri - - name: AZURE_AD_APP_RESOURCE_ID - secret: - name: active-directory - key: principal-clientid - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - value: "true" - - name: SERVER_SERVLET_CONTEXTPATH - value: "/api/file/" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: "file" - - name: SPRING_CONFIG_NAME - value: "common,application" - - name: LOG_PREFIX - value: file - - name: LOGGING_LEVEL - value: INFO - - name: APPLICATION_PORT - value: 80 - - name: COSMOSDB_DATABASE - value: osdu-db - - name: OSDU_ENTITLEMENTS_APP_KEY - value: OBSOLETE - - name: PARTITION_SERVICE_ENDPOINT - value: "http://partition/api/partition/v1" - - name: OSDU_ENTITLEMENTS_URL - value: http://entitlements/api/entitlements/v2 - - name: authorizeAPI - value: http://entitlements/api/entitlements/v2 - - name: OSDU_STORAGE_URL - value: http://storage/api/storage/v2 - - name: SEARCH_HOST - value: http://search/api/search/v2 - - name: AZURE_PUBSUB_PUBLISH - value: "true" - - name: SERVICE_BUS_ENABLED_STATUS - value: "true" - - name: SERVICE_BUS_TOPIC_STATUS - value: "statuschangedtopic" - - name: BATCH_SIZE - value: "100" - - name: SEARCH_QUERY_LIMIT - value: "1000" - - name: FILE_CHECKSUM_CALCULATION_LIMIT - value: "5368709120L" \ No newline at end of file +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-file +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-indexer-queue +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: file +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: file +# path: /api/file/ +# hosts: +# - "*" +# cors: +# - "http://localhost:8080" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/system/file/file- +# branch: release-0-26 +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# request: +# cpu: 1000m +# memory: 1Gi +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/file/v2/info" +# - "/api/file/v2/swagger*" +# - "/api/file/v2/api-docs*" +# - "/api/file/v2/webjars/*" +# env: +# - name: KEYVAULT_URL +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AZURE_AD_APP_RESOURCE_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_SERVLET_CONTEXTPATH +# value: "/api/file/" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: "file" +# - name: SPRING_CONFIG_NAME +# value: "common,application" +# - name: LOG_PREFIX +# value: file +# - name: LOGGING_LEVEL +# value: INFO +# - name: APPLICATION_PORT +# value: 80 +# - name: COSMOSDB_DATABASE +# value: osdu-db +# - name: OSDU_ENTITLEMENTS_APP_KEY +# value: OBSOLETE +# - name: PARTITION_SERVICE_ENDPOINT +# value: "http://partition/api/partition/v1" +# - name: OSDU_ENTITLEMENTS_URL +# value: http://entitlements/api/entitlements/v2 +# - name: authorizeAPI +# value: http://entitlements/api/entitlements/v2 +# - name: OSDU_STORAGE_URL +# value: http://storage/api/storage/v2 +# - name: SEARCH_HOST +# value: http://search/api/search/v2 +# - name: AZURE_PUBSUB_PUBLISH +# value: "true" +# - name: SERVICE_BUS_ENABLED_STATUS +# value: "true" +# - name: SERVICE_BUS_TOPIC_STATUS +# value: "statuschangedtopic" +# - name: BATCH_SIZE +# value: "100" +# - name: SEARCH_QUERY_LIMIT +# value: "1000" +# - name: FILE_CHECKSUM_CALCULATION_LIMIT +# value: "5368709120L" \ No newline at end of file diff --git a/software/applications/osdu-core/indexer.yaml b/software/applications/osdu-core/indexer.yaml index f3623733..d1702f95 100644 --- a/software/applications/osdu-core/indexer.yaml +++ b/software/applications/osdu-core/indexer.yaml @@ -1,226 +1,226 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-indexer-service - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-legal - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: indexer - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: indexer - path: /api/indexer/v2/ - hosts: - - "*" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/system/indexer-service/indexer-service- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/indexer/v2/info" - - /api/indexer/v2/swagger* - - /api/indexer/v2/api-docs* - - "/api/indexer/v2/webjars/*" - - '*/index-worker' - - '*/_dps/task-handlers' - - '*/reindex' - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - value: "true" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SECURITY_HTTPS_CERTIFICATE_TRUST - value: "true" - - name: SPRING_APPLICATION_NAME - value: indexer - - name: SERVER_SERVLET_CONTEXTPATH - value: /api/indexer/v2/ - - name: COSMOSDB_DATABASE - value: osdu-db - - name: REDIS_DATABASE - value: "4" - - name: REDIS_TTL_SECONDS - value: "3600" - - name: SERVICEBUS_TOPIC_NAME - value: indexing-progress - - name: REINDEX_TOPIC_NAME - value: recordstopic - - name: PARTITION_SERVICE_ENDPOINT - value: http://partition/api/partition/v1 - - name: ENTITLEMENTS_SERVICE_ENDPOINT - value: http://entitlements/api/entitlements/v2 - - name: ENTITLEMENTS_SERVICE_API_KEY - value: "OBSOLETE" - - name: SCHEMA_SERVICE_URL - value: http://schema/api/schema-service/v1 - - name: STORAGE_SERVICE_URL - value: http://storage/api/storage/v2 - - name: STORAGE_SCHEMA_HOST - value: http://storage/api/storage/v2/schemas - - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST - value: http://storage/api/storage/v2/query/records:batch - - name: STORAGE_QUERY_RECORD_HOST - value: http://storage/api/storage/v2/query/records - - name: SEARCH_SERVICE_URL - value: http://search/api/search/v2 ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-indexer-queue - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-legal - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: indexer-queue - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: indexer-queue - repository: community.opengroup.org:5555/osdu/platform/system/indexer-queue/indexer-queue- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_APP_RESOURCE_ID - secret: - name: active-directory - key: principal-clientid - - name: AZURE_APPLICATION_INSIGHTS_INSTRUMENTATION_KEY - secret: - name: azure-resources - key: insights-key - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - value: "true" - - name: SERVER_PORT - value: "80" - - name: SPRING_APPLICATION_NAME - value: indexer-queue - - name: AZURE_SERVICEBUS_TOPIC_NAME - value: recordstopic - - name: AZURE_REINDEX_TOPIC_NAME - value: reindextopic - - name: AZURE_SERVICEBUS_TOPIC_SUBSCRIPTION - value: recordstopicsubscription - - name: AZURE_REINDEX_TOPIC_SUBSCRIPTION - value: reindextopicsubscription - - name: AZURE_SCHEMACHANGED_TOPIC_NAME - value: schemachangedtopic - - name: AZURE_SCHEMACHANGED_TOPIC_SUBSCRIPTION - value: schemachangedtopiceg - - name: MAX_CONCURRENT_CALLS - value: "32" - - name: MAX_DELIVERY_COUNT - value: "5" - - name: EXECUTOR_N_THREADS - value: "32" - - name: MAX_LOCK_RENEW_DURATION_SECONDS - value: "600" - - name: PARTITION_API - value: http://partition/api/partition/v1 - - name: INDEXER_WORKER_URL - value: http://indexer/api/indexer/v2/_dps/task-handlers/index-worker - - name: schema_worker_url - value: http://indexer-service/api/indexer/v2/_dps/task-handlers/schema-worker \ No newline at end of file +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-indexer-service +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-legal +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: indexer +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: indexer +# path: /api/indexer/v2/ +# hosts: +# - "*" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/system/indexer-service/indexer-service- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/indexer/v2/info" +# - /api/indexer/v2/swagger* +# - /api/indexer/v2/api-docs* +# - "/api/indexer/v2/webjars/*" +# - '*/index-worker' +# - '*/_dps/task-handlers' +# - '*/reindex' +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SECURITY_HTTPS_CERTIFICATE_TRUST +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: indexer +# - name: SERVER_SERVLET_CONTEXTPATH +# value: /api/indexer/v2/ +# - name: COSMOSDB_DATABASE +# value: osdu-db +# - name: REDIS_DATABASE +# value: "4" +# - name: REDIS_TTL_SECONDS +# value: "3600" +# - name: SERVICEBUS_TOPIC_NAME +# value: indexing-progress +# - name: REINDEX_TOPIC_NAME +# value: recordstopic +# - name: PARTITION_SERVICE_ENDPOINT +# value: http://partition/api/partition/v1 +# - name: ENTITLEMENTS_SERVICE_ENDPOINT +# value: http://entitlements/api/entitlements/v2 +# - name: ENTITLEMENTS_SERVICE_API_KEY +# value: "OBSOLETE" +# - name: SCHEMA_SERVICE_URL +# value: http://schema/api/schema-service/v1 +# - name: STORAGE_SERVICE_URL +# value: http://storage/api/storage/v2 +# - name: STORAGE_SCHEMA_HOST +# value: http://storage/api/storage/v2/schemas +# - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST +# value: http://storage/api/storage/v2/query/records:batch +# - name: STORAGE_QUERY_RECORD_HOST +# value: http://storage/api/storage/v2/query/records +# - name: SEARCH_SERVICE_URL +# value: http://search/api/search/v2 +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-indexer-queue +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-legal +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: indexer-queue +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: indexer-queue +# repository: community.opengroup.org:5555/osdu/platform/system/indexer-queue/indexer-queue- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: AZURE_APP_RESOURCE_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: AZURE_APPLICATION_INSIGHTS_INSTRUMENTATION_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_PORT +# value: "80" +# - name: SPRING_APPLICATION_NAME +# value: indexer-queue +# - name: AZURE_SERVICEBUS_TOPIC_NAME +# value: recordstopic +# - name: AZURE_REINDEX_TOPIC_NAME +# value: reindextopic +# - name: AZURE_SERVICEBUS_TOPIC_SUBSCRIPTION +# value: recordstopicsubscription +# - name: AZURE_REINDEX_TOPIC_SUBSCRIPTION +# value: reindextopicsubscription +# - name: AZURE_SCHEMACHANGED_TOPIC_NAME +# value: schemachangedtopic +# - name: AZURE_SCHEMACHANGED_TOPIC_SUBSCRIPTION +# value: schemachangedtopiceg +# - name: MAX_CONCURRENT_CALLS +# value: "32" +# - name: MAX_DELIVERY_COUNT +# value: "5" +# - name: EXECUTOR_N_THREADS +# value: "32" +# - name: MAX_LOCK_RENEW_DURATION_SECONDS +# value: "600" +# - name: PARTITION_API +# value: http://partition/api/partition/v1 +# - name: INDEXER_WORKER_URL +# value: http://indexer/api/indexer/v2/_dps/task-handlers/index-worker +# - name: schema_worker_url +# value: http://indexer-service/api/indexer/v2/_dps/task-handlers/schema-worker \ No newline at end of file diff --git a/software/applications/osdu-core/legal.yaml b/software/applications/osdu-core/legal.yaml index ef554df5..5b27f8b7 100644 --- a/software/applications/osdu-core/legal.yaml +++ b/software/applications/osdu-core/legal.yaml @@ -1,122 +1,122 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-legal - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-partition - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-repo-override - optional: true - valuesKey: repository.yaml - values: - nameOverride: legal - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: legal - path: /api/legal/v1/ - hosts: - - "*" - cors: - - "http://localhost:8080" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/legal/legal- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/legal/v1/info" - - "/api/legal/v1/swagger*" - - "/api/legal/v1/api-docs*" - - "/api/legal/v1/webjars/*" - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - value: "true" - - name: SERVER_SERVLET_CONTEXTPATH - value: "/api/legal/v1/" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: "legal" - - name: SPRING_CONFIG_NAME - value: "common,application" - - name: LOG_PREFIX - value: "legal" - - name: AZURE_STORAGE_ENABLE_HTTPS - value: "true" - - name: COSMOSDB_DATABASE - value: "osdu-db" - - name: AZURE_STORAGE_CONTAINER_NAME - value: "legal-service-azure-configuration" - - name: LEGAL_SERVICE_REGION - value: "us" - - name: SERVICEBUS_TOPIC_NAME - value: "legaltags" - - name: REDIS_DATABASE - value: "2" - - name: PARTITION_SERVICE_ENDPOINT - value: "http://partition/api/partition/v1" - - name: ENTITLEMENTS_SERVICE_ENDPOINT - value: "http://entitlements/api/entitlements/v2" - - name: ENTITLEMENTS_SERVICE_API_KEY - value: "OBSOLETE" \ No newline at end of file +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-legal +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-partition +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-repo-override +# optional: true +# valuesKey: repository.yaml +# values: +# nameOverride: legal +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: legal +# path: /api/legal/v1/ +# hosts: +# - "*" +# cors: +# - "http://localhost:8080" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/legal/legal- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/legal/v1/info" +# - "/api/legal/v1/swagger*" +# - "/api/legal/v1/api-docs*" +# - "/api/legal/v1/webjars/*" +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_SERVLET_CONTEXTPATH +# value: "/api/legal/v1/" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: "legal" +# - name: SPRING_CONFIG_NAME +# value: "common,application" +# - name: LOG_PREFIX +# value: "legal" +# - name: AZURE_STORAGE_ENABLE_HTTPS +# value: "true" +# - name: COSMOSDB_DATABASE +# value: "osdu-db" +# - name: AZURE_STORAGE_CONTAINER_NAME +# value: "legal-service-azure-configuration" +# - name: LEGAL_SERVICE_REGION +# value: "us" +# - name: SERVICEBUS_TOPIC_NAME +# value: "legaltags" +# - name: REDIS_DATABASE +# value: "2" +# - name: PARTITION_SERVICE_ENDPOINT +# value: "http://partition/api/partition/v1" +# - name: ENTITLEMENTS_SERVICE_ENDPOINT +# value: "http://entitlements/api/entitlements/v2" +# - name: ENTITLEMENTS_SERVICE_API_KEY +# value: "OBSOLETE" \ No newline at end of file diff --git a/software/applications/osdu-core/schema.yaml b/software/applications/osdu-core/schema.yaml index 98578b0f..a740fea8 100644 --- a/software/applications/osdu-core/schema.yaml +++ b/software/applications/osdu-core/schema.yaml @@ -1,167 +1,167 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-schema - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-indexer-service - namespace: osdu-core - - name: osdu-indexer-queue - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: schema - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: schema - path: /api/schema-service/v1/ - hosts: - - "*" - cors: - - "http://localhost:8080" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/system/schema-service/schema-service- - tag: latest - probe: - path: /actuator/health - port: 8081 - keyvault: true - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/schema-service/v1/info" - - "/api/schema-service/v1/swagger*" - - "/api/schema-service/v1/api-docs*" - - "/api/schema-service/v2/webjars/*" - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - value: "true" - - name: SERVER_SERVLET_CONTEXTPATH - value: "/api/schema-service/v1/" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: "schema" - - name: COSMOSDB_DATABASE - value: "osdu-db" - - name: LOG_PREFIX - value: "schema" - - name: AZURE_SYSTEM_STORAGECONTAINERNAME - value: "system" - - name: SERVICEBUS_TOPIC_NAME - value: "schemachangedtopic" - - name: EVENT_GRID_ENABLED - value: 'false' - - name: EVENT_GRID_TOPIC - value: "schemachangedtopic" - - name: SERVICE_BUS_ENABLED - value: 'true' - - name: PARTITION_SERVICE_ENDPOINT - value: "http://partition/api/partition/v1" - - name: ENTITLEMENTS_SERVICE_ENDPOINT - value: "http://entitlements/api/entitlements/v2" - - name: ENTITLEMENTS_SERVICE_API_KEY - value: "OBSOLETE" ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-schema - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-schema - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - installationType: osduCore - jobs: - partitionInit: false - entitlementInit: false - userInit: false - schemaInit: true - elasticInit: false - partition: opendes - clientSecret: - name: active-directory - key: principal-clientpassword - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id - - kind: ConfigMap - name: configmap-services - targetPath: serviceBus - valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-schema +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-indexer-service +# namespace: osdu-core +# - name: osdu-indexer-queue +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: schema +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: schema +# path: /api/schema-service/v1/ +# hosts: +# - "*" +# cors: +# - "http://localhost:8080" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/system/schema-service/schema-service- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# keyvault: true +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/schema-service/v1/info" +# - "/api/schema-service/v1/swagger*" +# - "/api/schema-service/v1/api-docs*" +# - "/api/schema-service/v2/webjars/*" +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_SERVLET_CONTEXTPATH +# value: "/api/schema-service/v1/" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: "schema" +# - name: COSMOSDB_DATABASE +# value: "osdu-db" +# - name: LOG_PREFIX +# value: "schema" +# - name: AZURE_SYSTEM_STORAGECONTAINERNAME +# value: "system" +# - name: SERVICEBUS_TOPIC_NAME +# value: "schemachangedtopic" +# - name: EVENT_GRID_ENABLED +# value: 'false' +# - name: EVENT_GRID_TOPIC +# value: "schemachangedtopic" +# - name: SERVICE_BUS_ENABLED +# value: 'true' +# - name: PARTITION_SERVICE_ENDPOINT +# value: "http://partition/api/partition/v1" +# - name: ENTITLEMENTS_SERVICE_ENDPOINT +# value: "http://entitlements/api/entitlements/v2" +# - name: ENTITLEMENTS_SERVICE_API_KEY +# value: "OBSOLETE" +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-schema +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-schema +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# installationType: osduCore +# jobs: +# partitionInit: false +# entitlementInit: false +# userInit: false +# schemaInit: true +# elasticInit: false +# partition: opendes +# clientSecret: +# name: active-directory +# key: principal-clientpassword +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: clientId +# valuesKey: client_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: tenantId +# valuesKey: tenant_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: serviceBus +# valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file diff --git a/software/applications/osdu-core/search.yaml b/software/applications/osdu-core/search.yaml index 864ac275..76b77283 100644 --- a/software/applications/osdu-core/search.yaml +++ b/software/applications/osdu-core/search.yaml @@ -1,121 +1,121 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-search - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-indexer-queue - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: search - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: search - path: /api/search/v2/ - hosts: - - "*" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/system/search-service/search-service- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - request: - cpu: 1000m - memory: 1Gi - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/search/v2/info" - - "/api/search/v2/swagger*" - - "/api/search/v2/api-docs*" - - "/api/search/v2/webjars/*" - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - value: "true" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: search - - name: SERVER_SERVLET_CONTEXTPATH - value: /api/search/v2/ - - name: LOG_PREFIX - value: "search" - - name: SEARCH_SERVICE_SPRING_LOGGING_LEVEL - value: "DEBUG" - - name: COSMOSDB_DATABASE - value: "osdu-db" - - name: REDIS_DATABASE - value: "5" - - name: ENVIRONMENT - value: "evt" - - name: ELASTIC_CACHE_EXPIRATION - value: 1 - - name: MAX_CACHE_VALUE_SIZE - value: 60 - - name: POLICY_SERVICE_ENABLED - value: "false" - - name: ENTITLEMENTS_SERVICE_API_KEY - value: "OBSOLETE" - - name: PARTITION_SERVICE_ENDPOINT - value: "http://partition/api/partition/v1" - - name: ENTITLEMENTS_SERVICE_ENDPOINT - value: "http://entitlements/api/entitlements/v2" - - name: POLICY_SERVICE_ENDPOINT - value: http://policy/api/policy/v1 \ No newline at end of file +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-search +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-indexer-queue +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: search +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: search +# path: /api/search/v2/ +# hosts: +# - "*" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/system/search-service/search-service- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# request: +# cpu: 1000m +# memory: 1Gi +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/search/v2/info" +# - "/api/search/v2/swagger*" +# - "/api/search/v2/api-docs*" +# - "/api/search/v2/webjars/*" +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: search +# - name: SERVER_SERVLET_CONTEXTPATH +# value: /api/search/v2/ +# - name: LOG_PREFIX +# value: "search" +# - name: SEARCH_SERVICE_SPRING_LOGGING_LEVEL +# value: "DEBUG" +# - name: COSMOSDB_DATABASE +# value: "osdu-db" +# - name: REDIS_DATABASE +# value: "5" +# - name: ENVIRONMENT +# value: "evt" +# - name: ELASTIC_CACHE_EXPIRATION +# value: 1 +# - name: MAX_CACHE_VALUE_SIZE +# value: 60 +# - name: POLICY_SERVICE_ENABLED +# value: "false" +# - name: ENTITLEMENTS_SERVICE_API_KEY +# value: "OBSOLETE" +# - name: PARTITION_SERVICE_ENDPOINT +# value: "http://partition/api/partition/v1" +# - name: ENTITLEMENTS_SERVICE_ENDPOINT +# value: "http://entitlements/api/entitlements/v2" +# - name: POLICY_SERVICE_ENDPOINT +# value: http://policy/api/policy/v1 \ No newline at end of file diff --git a/software/applications/osdu-core/storage.yaml b/software/applications/osdu-core/storage.yaml index f503481f..e7ed6a86 100644 --- a/software/applications/osdu-core/storage.yaml +++ b/software/applications/osdu-core/storage.yaml @@ -1,138 +1,138 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-storage - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-indexer-queue - namespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - targetNamespace: osdu-core - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: storage - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: storage - path: /api/storage/v2/ - hosts: - - "*" - cors: - - "http://localhost:8080" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/system/storage/storage- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - request: - cpu: 800m - memory: 1Gi - # limit: - # cpu: 1000m - # memory: 4Gi - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/storage/v2/info" - - "/api/storage/v2/swagger*" - - "/api/storage/v2/api-docs*" - - "/api/storage/v2/webjars/*" - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - value: "true" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: storage - - name: SERVER_SERVLET_CONTEXTPATH - value: /api/storage/v2/ - - name: COSMOSDB_DATABASE - value: osdu-db - - name: AZURE_EVENTGRID_ENABLED - value: "false" - - name: AZURE_SERVICEBUS_ENABLED - value: "true" - - name: SERVICEBUS_TOPIC_NAME - value: recordstopic - - name: SERVICEBUS_V2_TOPIC_NAME - value: recordstopic-v2 - - name: REDIS_DATABASE - value: "4" - - name: PARTITION_SERVICE_ENDPOINT - value: http://partition/api/partition/v1 - - name: ENTITLEMENTS_SERVICE_ENDPOINT - value: http://entitlements/api/entitlements/v2 - - name: ENTITLEMENTS_SERVICE_API_KEY - value: "OBSOLETE" - - name: LEGAL_SERVICE_ENDPOINT - value: http://legal/api/legal/v1 - - name: LEGAL_SERVICE_REGION - value: southcentralus - - name: LEGAL_SERVICEBUS_TOPIC_NAME - value: legaltagschangedtopiceg - - name: LEGAL_SERVICEBUS_TOPIC_SUBSCRIPTION - value: eg_sb_legaltagchangedsubscription - - name: CRS_CONVERSION_SERVICE_ENDPOINT - value: http://crs-conversion/api/crs/converter/v2 - - name: POLICY_SERVICE_ENDPOINT - value: http://policy/api/policy/v1 - - name: OPA_ENABLED - value: "false" - - name: REDIS_HOST_KEY - value: redis-hostname - - name: REDIS_PASSWORD_KEY - value: redis-password +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-storage +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-indexer-queue +# namespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# targetNamespace: osdu-core +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: storage +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: storage +# path: /api/storage/v2/ +# hosts: +# - "*" +# cors: +# - "http://localhost:8080" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/system/storage/storage- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# request: +# cpu: 800m +# memory: 1Gi +# # limit: +# # cpu: 1000m +# # memory: 4Gi +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/storage/v2/info" +# - "/api/storage/v2/swagger*" +# - "/api/storage/v2/api-docs*" +# - "/api/storage/v2/webjars/*" +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: storage +# - name: SERVER_SERVLET_CONTEXTPATH +# value: /api/storage/v2/ +# - name: COSMOSDB_DATABASE +# value: osdu-db +# - name: AZURE_EVENTGRID_ENABLED +# value: "false" +# - name: AZURE_SERVICEBUS_ENABLED +# value: "true" +# - name: SERVICEBUS_TOPIC_NAME +# value: recordstopic +# - name: SERVICEBUS_V2_TOPIC_NAME +# value: recordstopic-v2 +# - name: REDIS_DATABASE +# value: "4" +# - name: PARTITION_SERVICE_ENDPOINT +# value: http://partition/api/partition/v1 +# - name: ENTITLEMENTS_SERVICE_ENDPOINT +# value: http://entitlements/api/entitlements/v2 +# - name: ENTITLEMENTS_SERVICE_API_KEY +# value: "OBSOLETE" +# - name: LEGAL_SERVICE_ENDPOINT +# value: http://legal/api/legal/v1 +# - name: LEGAL_SERVICE_REGION +# value: southcentralus +# - name: LEGAL_SERVICEBUS_TOPIC_NAME +# value: legaltagschangedtopiceg +# - name: LEGAL_SERVICEBUS_TOPIC_SUBSCRIPTION +# value: eg_sb_legaltagchangedsubscription +# - name: CRS_CONVERSION_SERVICE_ENDPOINT +# value: http://crs-conversion/api/crs/converter/v2 +# - name: POLICY_SERVICE_ENDPOINT +# value: http://policy/api/policy/v1 +# - name: OPA_ENABLED +# value: "false" +# - name: REDIS_HOST_KEY +# value: redis-hostname +# - name: REDIS_PASSWORD_KEY +# value: redis-password diff --git a/software/applications/osdu-core/workflow.yaml b/software/applications/osdu-core/workflow.yaml index 752c5bd8..2b2b7ce1 100644 --- a/software/applications/osdu-core/workflow.yaml +++ b/software/applications/osdu-core/workflow.yaml @@ -1,195 +1,195 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-workflow - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" -spec: - dependsOn: - - name: osdu-partition - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-service - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - timeout: 6m - install: - remediation: - retries: 3 - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - values: - nameOverride: workflow - installationType: osduCore - subset: m24 - replicaCount: 1 - service: - type: ClusterIP - port: 80 - configuration: - - service: workflow - path: /api/workflow/ - hosts: - - "*" - cors: - - "http://localhost:8080" - gateways: - - istio-system/internal-gateway - - istio-system/external-gateway - repository: community.opengroup.org:5555/osdu/platform/data-flow/ingestion/ingestion-workflow/ingestion-workflow- - tag: latest - probe: - path: /actuator/health - port: 8081 - liveness: - delay: 250 - seconds: 10 - keyvault: true - auth: - disable: - - "*/actuator/health" - - "*/health" - - "*/_ah/**" - - "*/configuration/ui" - - "*/configuration/security" - - "/api/workflow/v3/info" - - "/api/workflow/v3/swagger*" - - "/api/workflow/v3/api-docs*" - - "/api/workflow/v3/webjars/*" - env: - - name: KEYVAULT_URI - secret: - name: azure-resources - key: keyvault-uri - - name: AAD_CLIENT_ID - secret: - name: active-directory - key: principal-clientid - - name: APPINSIGHTS_KEY - secret: - name: azure-resources - key: insights-key - - name: APPLICATIONINSIGHTS_CONNECTION_STRING - secret: - name: azure-resources - key: insights-connection - - name: AZURE_ISTIOAUTH_ENABLED - value: "true" - - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED - value: "true" - - name: SERVER_SERVLET_CONTEXTPATH - value: "/api/workflow/" - - name: SERVER_PORT - value: "80" - - name: ACCEPT_HTTP - value: "true" - - name: SPRING_APPLICATION_NAME - value: "workflow" - - name: SPRING_CONFIG_NAME - value: "common,application" - - name: LOG_PREFIX - value: "workflow" - - name: AZURE_STORAGE_ENABLE_HTTPS - value: "true" - - name: COSMOSDB_DATABASE - value: "osdu-db" - - name: COSMOSDB_SYSTEM_DATABASE - value: osdu-system-db - - name: AIRFLOW_STORAGE_ACCOUNT_NAME - secret: - name: azure-resources - key: azurestorageaccountname - - name: AIRFLOW_STORAGE_ACCOUNT_KEY - secret: - name: azure-resources - key: azurestorageaccountkey - - name: OSDU_AIRFLOW_USERNAME - secret: - name: azure-resources - key: airflow-username - - name: OSDU_AIRFLOW_PASSWORD - secret: - name: azure-resources - key: airflow-password - - name: AUTHORIZEAPI - value: http://entitlements/api/entitlements/v2 - - name: AUTHORIZEAPIKEY - value: "OBSOLETE" - - name: PARTITION_SERVICE_ENDPOINT - value: "http://partition/api/partition/v1" - - name: OSDU_ENTITLEMENTS_URL - value: "http://entitlements/api/entitlements/v2" - - name: OSDU_AIRFLOW_URL - value: "http://airflow-web.airflow.svc.cluster.local:8080/airflow" - - name: OSDU_ENTITLEMENTS_APPKEY - value: "OBSOLETE" - - name: OSDU_AIRFLOW_VERSION2_ENABLED - value: true - - name: DP_AIRFLOW_FOR_SYSTEM_DAG - value: "false" - - name: IGNORE_DAGCONTENT - value: "true" - - name: IGNORE_CUSTOMOPERATORCONTENT - value: "true" ---- -# Retrigger: kubectl annotate helmrelease osdu-init-workflow fluxcd.io/retrigger=$(date +%s) -n osdu-core -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: osdu-init-workflow - namespace: osdu-core - annotations: - clusterconfig.azure.com/use-managed-source: "true" - fluxcd.io/retrigger: "initial" -spec: - dependsOn: - - name: osdu-workflow - namespace: osdu-core - targetNamespace: osdu-core - chart: - spec: - chart: ./charts/osdu-developer-init - sourceRef: - kind: GitRepository - name: flux-system - namespace: flux-system - interval: 5m0s - install: - remediation: - retries: 3 - values: - installationType: osduCore - jobs: - workflowInit: true - workflows: - - name: "Osdu_ingest" - description: "Manifest Ingest workflow for OSDU" - - name: "Osdu_ingest_by_reference" - description: "Manifest Ingest by reference workflow for OSDU" - - name: 'csv-parser' - description: 'CSV Parser workflow for OSDU' - partition: opendes - clientSecret: - name: active-directory - key: principal-clientpassword - valuesFrom: - - kind: ConfigMap - name: configmap-software - valuesKey: value.yaml - - kind: ConfigMap - name: configmap-services - targetPath: clientId - valuesKey: client_id - - kind: ConfigMap - name: configmap-services - targetPath: tenantId - valuesKey: tenant_id +# --- +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-workflow +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# spec: +# dependsOn: +# - name: osdu-partition +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-service +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# timeout: 6m +# install: +# remediation: +# retries: 3 +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# values: +# nameOverride: workflow +# installationType: osduCore +# subset: m24 +# replicaCount: 1 +# service: +# type: ClusterIP +# port: 80 +# configuration: +# - service: workflow +# path: /api/workflow/ +# hosts: +# - "*" +# cors: +# - "http://localhost:8080" +# gateways: +# - istio-system/internal-gateway +# - istio-system/external-gateway +# repository: community.opengroup.org:5555/osdu/platform/data-flow/ingestion/ingestion-workflow/ingestion-workflow- +# tag: latest +# probe: +# path: /actuator/health +# port: 8081 +# liveness: +# delay: 250 +# seconds: 10 +# keyvault: true +# auth: +# disable: +# - "*/actuator/health" +# - "*/health" +# - "*/_ah/**" +# - "*/configuration/ui" +# - "*/configuration/security" +# - "/api/workflow/v3/info" +# - "/api/workflow/v3/swagger*" +# - "/api/workflow/v3/api-docs*" +# - "/api/workflow/v3/webjars/*" +# env: +# - name: KEYVAULT_URI +# secret: +# name: azure-resources +# key: keyvault-uri +# - name: AAD_CLIENT_ID +# secret: +# name: active-directory +# key: principal-clientid +# - name: APPINSIGHTS_KEY +# secret: +# name: azure-resources +# key: insights-key +# - name: APPLICATIONINSIGHTS_CONNECTION_STRING +# secret: +# name: azure-resources +# key: insights-connection +# - name: AZURE_ISTIOAUTH_ENABLED +# value: "true" +# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED +# value: "true" +# - name: SERVER_SERVLET_CONTEXTPATH +# value: "/api/workflow/" +# - name: SERVER_PORT +# value: "80" +# - name: ACCEPT_HTTP +# value: "true" +# - name: SPRING_APPLICATION_NAME +# value: "workflow" +# - name: SPRING_CONFIG_NAME +# value: "common,application" +# - name: LOG_PREFIX +# value: "workflow" +# - name: AZURE_STORAGE_ENABLE_HTTPS +# value: "true" +# - name: COSMOSDB_DATABASE +# value: "osdu-db" +# - name: COSMOSDB_SYSTEM_DATABASE +# value: osdu-system-db +# - name: AIRFLOW_STORAGE_ACCOUNT_NAME +# secret: +# name: azure-resources +# key: azurestorageaccountname +# - name: AIRFLOW_STORAGE_ACCOUNT_KEY +# secret: +# name: azure-resources +# key: azurestorageaccountkey +# - name: OSDU_AIRFLOW_USERNAME +# secret: +# name: azure-resources +# key: airflow-username +# - name: OSDU_AIRFLOW_PASSWORD +# secret: +# name: azure-resources +# key: airflow-password +# - name: AUTHORIZEAPI +# value: http://entitlements/api/entitlements/v2 +# - name: AUTHORIZEAPIKEY +# value: "OBSOLETE" +# - name: PARTITION_SERVICE_ENDPOINT +# value: "http://partition/api/partition/v1" +# - name: OSDU_ENTITLEMENTS_URL +# value: "http://entitlements/api/entitlements/v2" +# - name: OSDU_AIRFLOW_URL +# value: "http://airflow-web.airflow.svc.cluster.local:8080/airflow" +# - name: OSDU_ENTITLEMENTS_APPKEY +# value: "OBSOLETE" +# - name: OSDU_AIRFLOW_VERSION2_ENABLED +# value: true +# - name: DP_AIRFLOW_FOR_SYSTEM_DAG +# value: "false" +# - name: IGNORE_DAGCONTENT +# value: "true" +# - name: IGNORE_CUSTOMOPERATORCONTENT +# value: "true" +# --- +# # Retrigger: kubectl annotate helmrelease osdu-init-workflow fluxcd.io/retrigger=$(date +%s) -n osdu-core +# apiVersion: helm.toolkit.fluxcd.io/v2beta1 +# kind: HelmRelease +# metadata: +# name: osdu-init-workflow +# namespace: osdu-core +# annotations: +# clusterconfig.azure.com/use-managed-source: "true" +# fluxcd.io/retrigger: "initial" +# spec: +# dependsOn: +# - name: osdu-workflow +# namespace: osdu-core +# targetNamespace: osdu-core +# chart: +# spec: +# chart: ./charts/osdu-developer-init +# sourceRef: +# kind: GitRepository +# name: flux-system +# namespace: flux-system +# interval: 5m0s +# install: +# remediation: +# retries: 3 +# values: +# installationType: osduCore +# jobs: +# workflowInit: true +# workflows: +# - name: "Osdu_ingest" +# description: "Manifest Ingest workflow for OSDU" +# - name: "Osdu_ingest_by_reference" +# description: "Manifest Ingest by reference workflow for OSDU" +# - name: 'csv-parser' +# description: 'CSV Parser workflow for OSDU' +# partition: opendes +# clientSecret: +# name: active-directory +# key: principal-clientpassword +# valuesFrom: +# - kind: ConfigMap +# name: configmap-software +# valuesKey: value.yaml +# - kind: ConfigMap +# name: configmap-services +# targetPath: clientId +# valuesKey: client_id +# - kind: ConfigMap +# name: configmap-services +# targetPath: tenantId +# valuesKey: tenant_id From 9ba9122f852a56953bcb74c20edcc929da2cf68b Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 10:51:16 -0600 Subject: [PATCH 107/122] Debug --- charts/osdu-developer-init/templates/entitlement-init.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/osdu-developer-init/templates/entitlement-init.yaml b/charts/osdu-developer-init/templates/entitlement-init.yaml index 1a495080..e0f83f38 100644 --- a/charts/osdu-developer-init/templates/entitlement-init.yaml +++ b/charts/osdu-developer-init/templates/entitlement-init.yaml @@ -70,7 +70,7 @@ data: -t ${AZURE_TENANT_ID} # Get token (no resource needed) - TOKEN=$(az account get-access-token --resource "api://${AZURE_CLIENT_ID}" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://management.azure.com/" --query accessToken -o tsv) OUTPUT=$(curl -s -w "%{http_code}" --request POST \ --url http://entitlements.{{ $namespace }}/api/entitlements/v2/tenant-provisioning \ From 204ce29e0192239bece97be450d7bbc41945bf8f Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 10:54:31 -0600 Subject: [PATCH 108/122] Debug --- charts/osdu-developer-init/templates/entitlement-init.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/osdu-developer-init/templates/entitlement-init.yaml b/charts/osdu-developer-init/templates/entitlement-init.yaml index e0f83f38..f7922349 100644 --- a/charts/osdu-developer-init/templates/entitlement-init.yaml +++ b/charts/osdu-developer-init/templates/entitlement-init.yaml @@ -83,6 +83,8 @@ data: HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$') BODY=${OUTPUT%???} + sleep 1000000 + if [ "$HTTP_STATUS_CODE" == "200" ]; then echo "Success: $(echo "$BODY" | jq .)" else From f7db9d2f04e8af6494e41a4e25d0b4fde5a96839 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 13:00:37 -0600 Subject: [PATCH 109/122] Debug --- .../templates/envoy-filter.yaml | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index 1339916f..cecc7ee9 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -104,21 +104,40 @@ spec: -- Step 3: Log raw payload for debugging request_handle:logDebug("JWT payload: " .. tableToString(payload)) - -- Step 4: Always set x-app-id from aud claim if present - if payload["aud"] then - request_handle:headers():add("x-app-id", payload["aud"]) - request_handle:logWarn("x-app-id set from 'aud' claim: " .. payload["aud"]) + -- Step 4: Process audience (aud) claim + local aud = payload["aud"] + if aud then + request_handle:headers():add("x-app-id", aud) + request_handle:logWarn("x-app-id set from 'aud' claim: " .. aud) + -- Special handling for audience "https://management.azure.com/" + if aud == "https://management.azure.com/" then + -- Check for oid to set x-user-id + local oid = payload["oid"] + if oid then + request_handle:headers():add("x-user-id", oid) + request_handle:logWarn("x-user-id set from 'oid' claim for management.azure.com audience") + -- Override x-app-id to match x-user-id + request_handle:headers():replace("x-app-id", oid) + request_handle:logWarn("x-app-id overridden to match x-user-id for management.azure.com audience") + else + request_handle:logError("No 'oid' claim found in payload for management.azure.com audience") + end + return -- Exit early as we don't need further processing for this case + end + else + request_handle:logError("No 'aud' claim found in payload") end - -- Step 5: Process based on issuer - if string.find(payload["iss"], AAD_V1_ISSUER) then + -- Step 5: Process issuer (iss) for additional logic + local iss = payload["iss"] + if iss and string.find(iss, AAD_V1_ISSUER) then request_handle:logDebug("Processing AAD v1 token") processAADV1Token(payload, request_handle) - elseif string.find(payload["iss"], AAD_V2_ISSUER) then + elseif iss and string.find(iss, AAD_V2_ISSUER) then request_handle:logDebug("Processing AAD v2 token") processAADV2Token(payload, request_handle) else - request_handle:logError("Unknown issuer: " .. payload["iss"]) + request_handle:logError("Unknown issuer: " .. (iss or "nil")) end -- Step 6: Log all headers before leaving the filter From 01d95ed39c0a1454b0c0b2d45567c4b05770e38e Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 14:21:33 -0600 Subject: [PATCH 110/122] Debug --- charts/osdu-developer-base/envoy-filter.md | 20 +++++++++---------- .../templates/envoy-filter.yaml | 20 +++++++++---------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/charts/osdu-developer-base/envoy-filter.md b/charts/osdu-developer-base/envoy-filter.md index fa9f622f..1f89b055 100644 --- a/charts/osdu-developer-base/envoy-filter.md +++ b/charts/osdu-developer-base/envoy-filter.md @@ -13,14 +13,14 @@ This contains the configuration and logic for an EnvoyFilter that processes Micr ## Token Scenarios Handled -| Scenario | x-user-id | x-app-id | -|--------------------------------------------|-------------------------------------|-----------------------------------| -| **AAD v1 User Token (sts.windows.net)** | `oid` (fallback: `upn`/`unique_name`) | `aud` | -| **AAD v1 Service-to-Service Delegation** | `x-on-behalf-of` (fallback: `appid`) | `aud` | -| **AAD v1 Application (non-delegated)** | `appid` | `aud` | -| **AAD v2 User Token (login.microsoftonline.com)** | `oid` | `aud` | -| **AAD v2 Service-to-Service Delegation** | `x-on-behalf-of` (fallback: `azp`) | `aud` | -| **AAD v2 Application (non-delegated)** | `azp` (fallback: `oid`) | `aud` | +| Scenario | x-user-id | x-app-id | +|--------------------------------------------|---------------------------------------|-----------------------------------| +| **AAD v1 User Token (sts.windows.net)** | `oid` (fallback: `upn`/`unique_name`) | `aud` | +| **AAD v1 Service-to-Service Delegation** | `x-on-behalf-of` (fallback: `appid`) | `aud` | +| **AAD v1 Application (non-delegated)** | `appid` | `aud` | +| **AAD v2 User Token (login.microsoftonline.com)** | `oid` | `aud` | +| **AAD v2 Service-to-Service Delegation** | `x-on-behalf-of` (fallback: `azp`) | `aud` | +| **AAD v2 Application (non-delegated)** | `azp` (fallback: `oid`) | `aud` | ## OAuth Delegation (On-Behalf-Of) Flow @@ -60,7 +60,5 @@ Use the following Istio commands to increase the logging level for debugging: ```bash # Enable detailed logging for Lua, JWT, and RBAC -istioctl proxy-config log --level lua:debug -istioctl proxy-config log --level jwt:debug -istioctl proxy-config log --level rbac:debug +istioctl proxy-config log --level lua:debug,jwt:debug,rbac:debug ``` diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index cecc7ee9..c145ab56 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -1,5 +1,5 @@ {{- $namespace := .Release.Namespace }} -{{- $managedIdentityClientId := .Values.azure.clientId }} +{{- $entraClientId := .Values.azure.clientId }} apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter @@ -27,7 +27,7 @@ spec: -- Constants local AAD_V1_ISSUER = "sts.windows.net" local AAD_V2_ISSUER = "login.microsoftonline.com" - local managedIdentityClientId = "{{ $managedIdentityClientId }}" + local entraClientId = "{{ $entraClientId }}" -- Helper function to log a table function tableToString(tbl, indent) @@ -111,16 +111,14 @@ spec: request_handle:logWarn("x-app-id set from 'aud' claim: " .. aud) -- Special handling for audience "https://management.azure.com/" if aud == "https://management.azure.com/" then - -- Check for oid to set x-user-id - local oid = payload["oid"] - if oid then - request_handle:headers():add("x-user-id", oid) - request_handle:logWarn("x-user-id set from 'oid' claim for management.azure.com audience") - -- Override x-app-id to match x-user-id - request_handle:headers():replace("x-app-id", oid) - request_handle:logWarn("x-app-id overridden to match x-user-id for management.azure.com audience") + local managedClientId = payload["appid"] + if managedClientId then + -- Set x-user-id and x-app-id to match appid for this case + request_handle:headers():add("x-user-id", entraClientId) + request_handle:headers():replace("x-app-id", entraClientId) + request_handle:logWarn("x-user-id and x-app-id set to 'appid' claim for management.azure.com audience") else - request_handle:logError("No 'oid' claim found in payload for management.azure.com audience") + request_handle:logError("No 'appid' claim found for management.azure.com audience") end return -- Exit early as we don't need further processing for this case end From 50839eedafbc659745fbd227a9a615e44cdca7bf Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 14:34:17 -0600 Subject: [PATCH 111/122] Debug --- charts/osdu-developer-base/Chart.yaml | 2 +- .../templates/entitlement-init.yaml | 2 - .../applications/osdu-core/user-init.yaml | 141 +++++++++--------- 3 files changed, 71 insertions(+), 74 deletions(-) diff --git a/charts/osdu-developer-base/Chart.yaml b/charts/osdu-developer-base/Chart.yaml index f57d4b33..00b27091 100644 --- a/charts/osdu-developer-base/Chart.yaml +++ b/charts/osdu-developer-base/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: osdu-developer-base type: application description: Installs the OSDU developer Base Components -version: 0.0.10 +version: 0.0.2 appVersion: 0.0.1 maintainers: - name: danielscholl diff --git a/charts/osdu-developer-init/templates/entitlement-init.yaml b/charts/osdu-developer-init/templates/entitlement-init.yaml index f7922349..e0f83f38 100644 --- a/charts/osdu-developer-init/templates/entitlement-init.yaml +++ b/charts/osdu-developer-init/templates/entitlement-init.yaml @@ -83,8 +83,6 @@ data: HTTP_STATUS_CODE=$(echo $OUTPUT | grep -oE '[0-9]{3}$') BODY=${OUTPUT%???} - sleep 1000000 - if [ "$HTTP_STATUS_CODE" == "200" ]; then echo "Success: $(echo "$BODY" | jq .)" else diff --git a/software/applications/osdu-core/user-init.yaml b/software/applications/osdu-core/user-init.yaml index cfb542d0..d3799b37 100644 --- a/software/applications/osdu-core/user-init.yaml +++ b/software/applications/osdu-core/user-init.yaml @@ -1,71 +1,70 @@ -# --- -# # kubectl create job --namespace osdu-core --from=cronjob/user-init user-init-run-$(date +%s) -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-user -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-entitlements-init -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# jobs: -# userInit: true -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: emailAddress -# valuesKey: first_user_id -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-user-sp -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-entitlements-init -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# jobs: -# userInit: true -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: emailAddress -# valuesKey: client_id \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-user + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-entitlements-init + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + jobs: + userInit: true + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: emailAddress + valuesKey: first_user_id +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-user-sp + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-entitlements-init + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + jobs: + userInit: true + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: emailAddress + valuesKey: client_id \ No newline at end of file From 2ca426a9aad21a11dc88ccf7ad59c9d14cc8d968 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 15:28:43 -0600 Subject: [PATCH 112/122] Debug --- charts/osdu-developer-base/envoy-filter.md | 17 ++++++++------- .../templates/envoy-filter.yaml | 21 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/charts/osdu-developer-base/envoy-filter.md b/charts/osdu-developer-base/envoy-filter.md index 1f89b055..27aad864 100644 --- a/charts/osdu-developer-base/envoy-filter.md +++ b/charts/osdu-developer-base/envoy-filter.md @@ -13,14 +13,15 @@ This contains the configuration and logic for an EnvoyFilter that processes Micr ## Token Scenarios Handled -| Scenario | x-user-id | x-app-id | -|--------------------------------------------|---------------------------------------|-----------------------------------| -| **AAD v1 User Token (sts.windows.net)** | `oid` (fallback: `upn`/`unique_name`) | `aud` | -| **AAD v1 Service-to-Service Delegation** | `x-on-behalf-of` (fallback: `appid`) | `aud` | -| **AAD v1 Application (non-delegated)** | `appid` | `aud` | -| **AAD v2 User Token (login.microsoftonline.com)** | `oid` | `aud` | -| **AAD v2 Service-to-Service Delegation** | `x-on-behalf-of` (fallback: `azp`) | `aud` | -| **AAD v2 Application (non-delegated)** | `azp` (fallback: `oid`) | `aud` | +| Use Case | Scenario | x-user-id | x-app-id | +|----------|--------------------------------------------|----------------------------------------|-----------------------------------| +| **UC1** | **AAD v1 User Token (sts.windows.net)** | `unique_name` (fallback: `oid`/`upn`) | `aud` | +| **UC2** | **AAD v1 Service-to-Service Delegation** | `x-on-behalf-of` (fallback: `appid`) | `aud` | +| **UC3** | **AAD v1 Application (non-delegated)** | `appid` | `aud` | +| **UC4** | **AAD v2 User Token (login.microsoftonline.com)** | `unique_name` (fallback: `oid`) | `aud` | +| **UC5** | **AAD v2 Service-to-Service Delegation** | `x-on-behalf-of` (fallback: `azp`) | `aud` | +| **UC6** | **AAD v2 Application (non-delegated)** | `azp` (fallback: `oid`) | `aud` | +| **UC7** | **Management Audience (`management.azure.com`)** | `entraClientId` | `entraClientId` | ## OAuth Delegation (On-Behalf-Of) Flow diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index c145ab56..622cc73a 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -58,15 +58,15 @@ spec: function processAADV1Token(payload, request_handle) if payload["unique_name"] then request_handle:headers():add("x-user-id", payload["unique_name"]) - request_handle:logWarn("x-user-id set from 'unique_name' claim") + request_handle:logInfo("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'unique_name' claim") elseif payload["oid"] then request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logWarn("x-user-id set from 'oid' claim as fallback") + request_handle:logInfo("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'oid' claim as fallback") elseif payload["upn"] then request_handle:headers():add("x-user-id", payload["upn"]) - request_handle:logWarn("x-user-id set from 'upn' claim as fallback") + request_handle:logInfo("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'upn' claim as fallback") else - request_handle:logError("No valid claim for x-user-id found in AAD v1 token") + request_handle:logError("UC1-(AAD v1 User Token (sts.windows.net)): No valid claim for x-user-id found in AAD v1 token") end end @@ -74,15 +74,15 @@ spec: function processAADV2Token(payload, request_handle) if payload["unique_name"] then request_handle:headers():add("x-user-id", payload["unique_name"]) - request_handle:logWarn("x-user-id set from 'unique_name' claim") + request_handle:logInfo("UC4-(AAD v2 User Token (login.microsoftonline.com)): x-user-id set from 'unique_name' claim") elseif payload["oid"] then request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logWarn("x-user-id set from 'oid' claim as fallback") + request_handle:logInfo("UC4-(AAD v2 User Token (login.microsoftonline.com)): x-user-id set from 'oid' claim as fallback") elseif payload["azp"] then request_handle:headers():add("x-user-id", payload["azp"]) - request_handle:logWarn("x-user-id set from 'azp' claim as fallback") + request_handle:logInfo("UC4-(AAD v2 User Token (login.microsoftonline.com)): x-user-id set from 'azp' claim as fallback") else - request_handle:logError("No valid claim for x-user-id found in AAD v2 token") + request_handle:logError("UC4-(AAD v2 User Token (login.microsoftonline.com)): No valid claim for x-user-id found in AAD v2 token") end end @@ -108,15 +108,14 @@ spec: local aud = payload["aud"] if aud then request_handle:headers():add("x-app-id", aud) - request_handle:logWarn("x-app-id set from 'aud' claim: " .. aud) + request_handle:logInfo("x-app-id set from 'aud' claim: " .. aud) -- Special handling for audience "https://management.azure.com/" if aud == "https://management.azure.com/" then local managedClientId = payload["appid"] if managedClientId then - -- Set x-user-id and x-app-id to match appid for this case request_handle:headers():add("x-user-id", entraClientId) request_handle:headers():replace("x-app-id", entraClientId) - request_handle:logWarn("x-user-id and x-app-id set to 'appid' claim for management.azure.com audience") + request_handle:logInfo("UC7-(Management Audience): x-user-id and x-app-id set to 'appid' claim for management.azure.com audience") else request_handle:logError("No 'appid' claim found for management.azure.com audience") end From 784675b3bf273836317a40071585ceee9957d2e9 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 15:48:23 -0600 Subject: [PATCH 113/122] Adjusted log levels --- .../templates/envoy-filter.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index 622cc73a..c3420c99 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -58,13 +58,13 @@ spec: function processAADV1Token(payload, request_handle) if payload["unique_name"] then request_handle:headers():add("x-user-id", payload["unique_name"]) - request_handle:logInfo("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'unique_name' claim") + request_handle:logWarn("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'unique_name' claim") elseif payload["oid"] then request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logInfo("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'oid' claim as fallback") + request_handle:logWarn("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'oid' claim as fallback") elseif payload["upn"] then request_handle:headers():add("x-user-id", payload["upn"]) - request_handle:logInfo("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'upn' claim as fallback") + request_handle:logWarn("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'upn' claim as fallback") else request_handle:logError("UC1-(AAD v1 User Token (sts.windows.net)): No valid claim for x-user-id found in AAD v1 token") end @@ -74,13 +74,13 @@ spec: function processAADV2Token(payload, request_handle) if payload["unique_name"] then request_handle:headers():add("x-user-id", payload["unique_name"]) - request_handle:logInfo("UC4-(AAD v2 User Token (login.microsoftonline.com)): x-user-id set from 'unique_name' claim") + request_handle:logWarn("UC4-(AAD v2 User Token (login.microsoftonline.com)): x-user-id set from 'unique_name' claim") elseif payload["oid"] then request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logInfo("UC4-(AAD v2 User Token (login.microsoftonline.com)): x-user-id set from 'oid' claim as fallback") + request_handle:logWarn("UC4-(AAD v2 User Token (login.microsoftonline.com)): x-user-id set from 'oid' claim as fallback") elseif payload["azp"] then request_handle:headers():add("x-user-id", payload["azp"]) - request_handle:logInfo("UC4-(AAD v2 User Token (login.microsoftonline.com)): x-user-id set from 'azp' claim as fallback") + request_handle:logWarn("UC4-(AAD v2 User Token (login.microsoftonline.com)): x-user-id set from 'azp' claim as fallback") else request_handle:logError("UC4-(AAD v2 User Token (login.microsoftonline.com)): No valid claim for x-user-id found in AAD v2 token") end @@ -91,7 +91,7 @@ spec: -- Step 1: Remove existing headers request_handle:headers():remove("x-user-id") request_handle:headers():remove("x-app-id") - request_handle:logInfo("x-user-id and x-app-id headers removed") + request_handle:logWarn("x-user-id and x-app-id headers removed") -- Step 2: Retrieve JWT metadata local meta = request_handle:streamInfo():dynamicMetadata():get("envoy.filters.http.jwt_authn") @@ -108,14 +108,14 @@ spec: local aud = payload["aud"] if aud then request_handle:headers():add("x-app-id", aud) - request_handle:logInfo("x-app-id set from 'aud' claim: " .. aud) + request_handle:logDebug("x-app-id set from 'aud' claim: " .. aud) -- Special handling for audience "https://management.azure.com/" if aud == "https://management.azure.com/" then local managedClientId = payload["appid"] if managedClientId then request_handle:headers():add("x-user-id", entraClientId) request_handle:headers():replace("x-app-id", entraClientId) - request_handle:logInfo("UC7-(Management Audience): x-user-id and x-app-id set to 'appid' claim for management.azure.com audience") + request_handle:logWarn("UC7-(Management Audience): x-user-id and x-app-id set to 'appid' claim for management.azure.com audience") else request_handle:logError("No 'appid' claim found for management.azure.com audience") end From 4f0028590deb62904467072cb1f8a364cb1fa16c Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 15:49:53 -0600 Subject: [PATCH 114/122] Added back legal. --- software/applications/osdu-core/legal.yaml | 244 ++++++++++----------- 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/software/applications/osdu-core/legal.yaml b/software/applications/osdu-core/legal.yaml index 5b27f8b7..ef554df5 100644 --- a/software/applications/osdu-core/legal.yaml +++ b/software/applications/osdu-core/legal.yaml @@ -1,122 +1,122 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-legal -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-partition -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-repo-override -# optional: true -# valuesKey: repository.yaml -# values: -# nameOverride: legal -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: legal -# path: /api/legal/v1/ -# hosts: -# - "*" -# cors: -# - "http://localhost:8080" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/legal/legal- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/legal/v1/info" -# - "/api/legal/v1/swagger*" -# - "/api/legal/v1/api-docs*" -# - "/api/legal/v1/webjars/*" -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_SERVLET_CONTEXTPATH -# value: "/api/legal/v1/" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: "legal" -# - name: SPRING_CONFIG_NAME -# value: "common,application" -# - name: LOG_PREFIX -# value: "legal" -# - name: AZURE_STORAGE_ENABLE_HTTPS -# value: "true" -# - name: COSMOSDB_DATABASE -# value: "osdu-db" -# - name: AZURE_STORAGE_CONTAINER_NAME -# value: "legal-service-azure-configuration" -# - name: LEGAL_SERVICE_REGION -# value: "us" -# - name: SERVICEBUS_TOPIC_NAME -# value: "legaltags" -# - name: REDIS_DATABASE -# value: "2" -# - name: PARTITION_SERVICE_ENDPOINT -# value: "http://partition/api/partition/v1" -# - name: ENTITLEMENTS_SERVICE_ENDPOINT -# value: "http://entitlements/api/entitlements/v2" -# - name: ENTITLEMENTS_SERVICE_API_KEY -# value: "OBSOLETE" \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-legal + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-partition + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-repo-override + optional: true + valuesKey: repository.yaml + values: + nameOverride: legal + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: legal + path: /api/legal/v1/ + hosts: + - "*" + cors: + - "http://localhost:8080" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/security-and-compliance/legal/legal- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/legal/v1/info" + - "/api/legal/v1/swagger*" + - "/api/legal/v1/api-docs*" + - "/api/legal/v1/webjars/*" + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_SERVLET_CONTEXTPATH + value: "/api/legal/v1/" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: "legal" + - name: SPRING_CONFIG_NAME + value: "common,application" + - name: LOG_PREFIX + value: "legal" + - name: AZURE_STORAGE_ENABLE_HTTPS + value: "true" + - name: COSMOSDB_DATABASE + value: "osdu-db" + - name: AZURE_STORAGE_CONTAINER_NAME + value: "legal-service-azure-configuration" + - name: LEGAL_SERVICE_REGION + value: "us" + - name: SERVICEBUS_TOPIC_NAME + value: "legaltags" + - name: REDIS_DATABASE + value: "2" + - name: PARTITION_SERVICE_ENDPOINT + value: "http://partition/api/partition/v1" + - name: ENTITLEMENTS_SERVICE_ENDPOINT + value: "http://entitlements/api/entitlements/v2" + - name: ENTITLEMENTS_SERVICE_API_KEY + value: "OBSOLETE" \ No newline at end of file From 817c86d80b6456ed8833510641a7d47d88af271f Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 16:05:28 -0600 Subject: [PATCH 115/122] Working Admin of Entitlements issues. --- charts/osdu-developer-base/templates/envoy-filter.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/osdu-developer-base/templates/envoy-filter.yaml b/charts/osdu-developer-base/templates/envoy-filter.yaml index c3420c99..2a8e1ae3 100644 --- a/charts/osdu-developer-base/templates/envoy-filter.yaml +++ b/charts/osdu-developer-base/templates/envoy-filter.yaml @@ -60,8 +60,8 @@ spec: request_handle:headers():add("x-user-id", payload["unique_name"]) request_handle:logWarn("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'unique_name' claim") elseif payload["oid"] then - request_handle:headers():add("x-user-id", payload["oid"]) - request_handle:logWarn("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'oid' claim as fallback") + request_handle:headers():add("x-user-id", payload["appid"]) + request_handle:logWarn("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'appid' claim as fallback") elseif payload["upn"] then request_handle:headers():add("x-user-id", payload["upn"]) request_handle:logWarn("UC1-(AAD v1 User Token (sts.windows.net)): x-user-id set from 'upn' claim as fallback") From bab66f14ccf0b1ae5383c8667dabeee5560cc73e Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 16:13:18 -0600 Subject: [PATCH 116/122] New location for Legal_COO --- software/applications/osdu-core/base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/applications/osdu-core/base.yaml b/software/applications/osdu-core/base.yaml index 8466b94a..131d6b36 100644 --- a/software/applications/osdu-core/base.yaml +++ b/software/applications/osdu-core/base.yaml @@ -68,4 +68,4 @@ spec: items: - name: legal file: "Legal_COO.json" - url: "https://raw.githubusercontent.com/Azure/osdu-developer/refs/heads/main/bicep/modules/script-blob-upload/Legal_COO.json" + url: "https://raw.githubusercontent.com/Azure/osdu-developer/refs/heads/main/bicep/modules/deploy-scripts/Legal_COO.json" From 5fea007a79640f024a7d5958943c26eb69b26380 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 16:36:03 -0600 Subject: [PATCH 117/122] Readd services. --- charts/osdu-developer-base/envoy-filter.md | 19 +- software/applications/osdu-core/file.yaml | 276 +++++------ software/applications/osdu-core/indexer.yaml | 452 +++++++++--------- software/applications/osdu-core/schema.yaml | 334 ++++++------- software/applications/osdu-core/search.yaml | 242 +++++----- software/applications/osdu-core/storage.yaml | 276 +++++------ software/applications/osdu-core/workflow.yaml | 390 +++++++-------- 7 files changed, 995 insertions(+), 994 deletions(-) diff --git a/charts/osdu-developer-base/envoy-filter.md b/charts/osdu-developer-base/envoy-filter.md index 27aad864..6abb6dcb 100644 --- a/charts/osdu-developer-base/envoy-filter.md +++ b/charts/osdu-developer-base/envoy-filter.md @@ -40,18 +40,19 @@ graph TD C -->|Metadata Found| D[Step 3: Log Payload] C -->|No Metadata Found| E[End: Request Processing Halted] D --> F[Step 4: Set x-app-id from 'aud'] - F --> G{Step 5: Check Issuer} + F --> F1{Check Management Audience} + F1 -->|aud = management.azure.com| F2[Set x-user-id and x-app-id to entraClientId] + F2 --> E + F1 -->|Other aud| G{Step 5: Check Issuer} G -->|Issuer: AAD v1 sts.windows.net| H[Process AAD v1 Token] G -->|Issuer: AAD v2 login.microsoftonline.com| I[Process AAD v2 Token] G -->|Unknown Issuer| J[Log Error: Unknown Issuer] - H --> H1[Set x-user-id using 'oid', fallback to 'upn' or 'unique_name'] - H1 --> K[Log Headers After AAD v1 Processing] - I --> I1[Set x-user-id using 'oid' or 'azp'] - I1 --> I2[Handle Delegation: Use x-on-behalf-of if Applicable] - I2 --> K[Log Headers After AAD v2 Processing] - J --> E - K --> L[Request Headers Modified] - L --> M[Request Forwarded] + H --> H1[Set x-user-id using unique_name, appid, or upn] + H1 --> K[Step 6: Log All Headers] + I --> I1[Set x-user-id using unique_name, oid, or azp] + I1 --> K + J --> K + K --> M[End: Request Forwarded] ``` ## Debugging and Logging diff --git a/software/applications/osdu-core/file.yaml b/software/applications/osdu-core/file.yaml index 7946c52f..34a020b1 100644 --- a/software/applications/osdu-core/file.yaml +++ b/software/applications/osdu-core/file.yaml @@ -1,138 +1,138 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-file -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-indexer-queue -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: file -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: file -# path: /api/file/ -# hosts: -# - "*" -# cors: -# - "http://localhost:8080" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/system/file/file- -# branch: release-0-26 -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# request: -# cpu: 1000m -# memory: 1Gi -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/file/v2/info" -# - "/api/file/v2/swagger*" -# - "/api/file/v2/api-docs*" -# - "/api/file/v2/webjars/*" -# env: -# - name: KEYVAULT_URL -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AZURE_AD_APP_RESOURCE_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_SERVLET_CONTEXTPATH -# value: "/api/file/" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: "file" -# - name: SPRING_CONFIG_NAME -# value: "common,application" -# - name: LOG_PREFIX -# value: file -# - name: LOGGING_LEVEL -# value: INFO -# - name: APPLICATION_PORT -# value: 80 -# - name: COSMOSDB_DATABASE -# value: osdu-db -# - name: OSDU_ENTITLEMENTS_APP_KEY -# value: OBSOLETE -# - name: PARTITION_SERVICE_ENDPOINT -# value: "http://partition/api/partition/v1" -# - name: OSDU_ENTITLEMENTS_URL -# value: http://entitlements/api/entitlements/v2 -# - name: authorizeAPI -# value: http://entitlements/api/entitlements/v2 -# - name: OSDU_STORAGE_URL -# value: http://storage/api/storage/v2 -# - name: SEARCH_HOST -# value: http://search/api/search/v2 -# - name: AZURE_PUBSUB_PUBLISH -# value: "true" -# - name: SERVICE_BUS_ENABLED_STATUS -# value: "true" -# - name: SERVICE_BUS_TOPIC_STATUS -# value: "statuschangedtopic" -# - name: BATCH_SIZE -# value: "100" -# - name: SEARCH_QUERY_LIMIT -# value: "1000" -# - name: FILE_CHECKSUM_CALCULATION_LIMIT -# value: "5368709120L" \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-file + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-indexer-queue + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: file + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: file + path: /api/file/ + hosts: + - "*" + cors: + - "http://localhost:8080" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/system/file/file- + branch: release-0-26 + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + request: + cpu: 1000m + memory: 1Gi + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/file/v2/info" + - "/api/file/v2/swagger*" + - "/api/file/v2/api-docs*" + - "/api/file/v2/webjars/*" + env: + - name: KEYVAULT_URL + secret: + name: azure-resources + key: keyvault-uri + - name: AZURE_AD_APP_RESOURCE_ID + secret: + name: active-directory + key: principal-clientid + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_SERVLET_CONTEXTPATH + value: "/api/file/" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: "file" + - name: SPRING_CONFIG_NAME + value: "common,application" + - name: LOG_PREFIX + value: file + - name: LOGGING_LEVEL + value: INFO + - name: APPLICATION_PORT + value: 80 + - name: COSMOSDB_DATABASE + value: osdu-db + - name: OSDU_ENTITLEMENTS_APP_KEY + value: OBSOLETE + - name: PARTITION_SERVICE_ENDPOINT + value: "http://partition/api/partition/v1" + - name: OSDU_ENTITLEMENTS_URL + value: http://entitlements/api/entitlements/v2 + - name: authorizeAPI + value: http://entitlements/api/entitlements/v2 + - name: OSDU_STORAGE_URL + value: http://storage/api/storage/v2 + - name: SEARCH_HOST + value: http://search/api/search/v2 + - name: AZURE_PUBSUB_PUBLISH + value: "true" + - name: SERVICE_BUS_ENABLED_STATUS + value: "true" + - name: SERVICE_BUS_TOPIC_STATUS + value: "statuschangedtopic" + - name: BATCH_SIZE + value: "100" + - name: SEARCH_QUERY_LIMIT + value: "1000" + - name: FILE_CHECKSUM_CALCULATION_LIMIT + value: "5368709120L" \ No newline at end of file diff --git a/software/applications/osdu-core/indexer.yaml b/software/applications/osdu-core/indexer.yaml index d1702f95..f3623733 100644 --- a/software/applications/osdu-core/indexer.yaml +++ b/software/applications/osdu-core/indexer.yaml @@ -1,226 +1,226 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-indexer-service -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-legal -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: indexer -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: indexer -# path: /api/indexer/v2/ -# hosts: -# - "*" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/system/indexer-service/indexer-service- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/indexer/v2/info" -# - /api/indexer/v2/swagger* -# - /api/indexer/v2/api-docs* -# - "/api/indexer/v2/webjars/*" -# - '*/index-worker' -# - '*/_dps/task-handlers' -# - '*/reindex' -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SECURITY_HTTPS_CERTIFICATE_TRUST -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: indexer -# - name: SERVER_SERVLET_CONTEXTPATH -# value: /api/indexer/v2/ -# - name: COSMOSDB_DATABASE -# value: osdu-db -# - name: REDIS_DATABASE -# value: "4" -# - name: REDIS_TTL_SECONDS -# value: "3600" -# - name: SERVICEBUS_TOPIC_NAME -# value: indexing-progress -# - name: REINDEX_TOPIC_NAME -# value: recordstopic -# - name: PARTITION_SERVICE_ENDPOINT -# value: http://partition/api/partition/v1 -# - name: ENTITLEMENTS_SERVICE_ENDPOINT -# value: http://entitlements/api/entitlements/v2 -# - name: ENTITLEMENTS_SERVICE_API_KEY -# value: "OBSOLETE" -# - name: SCHEMA_SERVICE_URL -# value: http://schema/api/schema-service/v1 -# - name: STORAGE_SERVICE_URL -# value: http://storage/api/storage/v2 -# - name: STORAGE_SCHEMA_HOST -# value: http://storage/api/storage/v2/schemas -# - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST -# value: http://storage/api/storage/v2/query/records:batch -# - name: STORAGE_QUERY_RECORD_HOST -# value: http://storage/api/storage/v2/query/records -# - name: SEARCH_SERVICE_URL -# value: http://search/api/search/v2 -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-indexer-queue -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-legal -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: indexer-queue -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: indexer-queue -# repository: community.opengroup.org:5555/osdu/platform/system/indexer-queue/indexer-queue- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: AZURE_APP_RESOURCE_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: AZURE_APPLICATION_INSIGHTS_INSTRUMENTATION_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_PORT -# value: "80" -# - name: SPRING_APPLICATION_NAME -# value: indexer-queue -# - name: AZURE_SERVICEBUS_TOPIC_NAME -# value: recordstopic -# - name: AZURE_REINDEX_TOPIC_NAME -# value: reindextopic -# - name: AZURE_SERVICEBUS_TOPIC_SUBSCRIPTION -# value: recordstopicsubscription -# - name: AZURE_REINDEX_TOPIC_SUBSCRIPTION -# value: reindextopicsubscription -# - name: AZURE_SCHEMACHANGED_TOPIC_NAME -# value: schemachangedtopic -# - name: AZURE_SCHEMACHANGED_TOPIC_SUBSCRIPTION -# value: schemachangedtopiceg -# - name: MAX_CONCURRENT_CALLS -# value: "32" -# - name: MAX_DELIVERY_COUNT -# value: "5" -# - name: EXECUTOR_N_THREADS -# value: "32" -# - name: MAX_LOCK_RENEW_DURATION_SECONDS -# value: "600" -# - name: PARTITION_API -# value: http://partition/api/partition/v1 -# - name: INDEXER_WORKER_URL -# value: http://indexer/api/indexer/v2/_dps/task-handlers/index-worker -# - name: schema_worker_url -# value: http://indexer-service/api/indexer/v2/_dps/task-handlers/schema-worker \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-indexer-service + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-legal + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: indexer + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: indexer + path: /api/indexer/v2/ + hosts: + - "*" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/system/indexer-service/indexer-service- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/indexer/v2/info" + - /api/indexer/v2/swagger* + - /api/indexer/v2/api-docs* + - "/api/indexer/v2/webjars/*" + - '*/index-worker' + - '*/_dps/task-handlers' + - '*/reindex' + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SECURITY_HTTPS_CERTIFICATE_TRUST + value: "true" + - name: SPRING_APPLICATION_NAME + value: indexer + - name: SERVER_SERVLET_CONTEXTPATH + value: /api/indexer/v2/ + - name: COSMOSDB_DATABASE + value: osdu-db + - name: REDIS_DATABASE + value: "4" + - name: REDIS_TTL_SECONDS + value: "3600" + - name: SERVICEBUS_TOPIC_NAME + value: indexing-progress + - name: REINDEX_TOPIC_NAME + value: recordstopic + - name: PARTITION_SERVICE_ENDPOINT + value: http://partition/api/partition/v1 + - name: ENTITLEMENTS_SERVICE_ENDPOINT + value: http://entitlements/api/entitlements/v2 + - name: ENTITLEMENTS_SERVICE_API_KEY + value: "OBSOLETE" + - name: SCHEMA_SERVICE_URL + value: http://schema/api/schema-service/v1 + - name: STORAGE_SERVICE_URL + value: http://storage/api/storage/v2 + - name: STORAGE_SCHEMA_HOST + value: http://storage/api/storage/v2/schemas + - name: STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST + value: http://storage/api/storage/v2/query/records:batch + - name: STORAGE_QUERY_RECORD_HOST + value: http://storage/api/storage/v2/query/records + - name: SEARCH_SERVICE_URL + value: http://search/api/search/v2 +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-indexer-queue + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-legal + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: indexer-queue + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: indexer-queue + repository: community.opengroup.org:5555/osdu/platform/system/indexer-queue/indexer-queue- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: AZURE_APP_RESOURCE_ID + secret: + name: active-directory + key: principal-clientid + - name: AZURE_APPLICATION_INSIGHTS_INSTRUMENTATION_KEY + secret: + name: azure-resources + key: insights-key + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_PORT + value: "80" + - name: SPRING_APPLICATION_NAME + value: indexer-queue + - name: AZURE_SERVICEBUS_TOPIC_NAME + value: recordstopic + - name: AZURE_REINDEX_TOPIC_NAME + value: reindextopic + - name: AZURE_SERVICEBUS_TOPIC_SUBSCRIPTION + value: recordstopicsubscription + - name: AZURE_REINDEX_TOPIC_SUBSCRIPTION + value: reindextopicsubscription + - name: AZURE_SCHEMACHANGED_TOPIC_NAME + value: schemachangedtopic + - name: AZURE_SCHEMACHANGED_TOPIC_SUBSCRIPTION + value: schemachangedtopiceg + - name: MAX_CONCURRENT_CALLS + value: "32" + - name: MAX_DELIVERY_COUNT + value: "5" + - name: EXECUTOR_N_THREADS + value: "32" + - name: MAX_LOCK_RENEW_DURATION_SECONDS + value: "600" + - name: PARTITION_API + value: http://partition/api/partition/v1 + - name: INDEXER_WORKER_URL + value: http://indexer/api/indexer/v2/_dps/task-handlers/index-worker + - name: schema_worker_url + value: http://indexer-service/api/indexer/v2/_dps/task-handlers/schema-worker \ No newline at end of file diff --git a/software/applications/osdu-core/schema.yaml b/software/applications/osdu-core/schema.yaml index a740fea8..98578b0f 100644 --- a/software/applications/osdu-core/schema.yaml +++ b/software/applications/osdu-core/schema.yaml @@ -1,167 +1,167 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-schema -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-indexer-service -# namespace: osdu-core -# - name: osdu-indexer-queue -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: schema -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: schema -# path: /api/schema-service/v1/ -# hosts: -# - "*" -# cors: -# - "http://localhost:8080" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/system/schema-service/schema-service- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# keyvault: true -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/schema-service/v1/info" -# - "/api/schema-service/v1/swagger*" -# - "/api/schema-service/v1/api-docs*" -# - "/api/schema-service/v2/webjars/*" -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_SERVLET_CONTEXTPATH -# value: "/api/schema-service/v1/" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: "schema" -# - name: COSMOSDB_DATABASE -# value: "osdu-db" -# - name: LOG_PREFIX -# value: "schema" -# - name: AZURE_SYSTEM_STORAGECONTAINERNAME -# value: "system" -# - name: SERVICEBUS_TOPIC_NAME -# value: "schemachangedtopic" -# - name: EVENT_GRID_ENABLED -# value: 'false' -# - name: EVENT_GRID_TOPIC -# value: "schemachangedtopic" -# - name: SERVICE_BUS_ENABLED -# value: 'true' -# - name: PARTITION_SERVICE_ENDPOINT -# value: "http://partition/api/partition/v1" -# - name: ENTITLEMENTS_SERVICE_ENDPOINT -# value: "http://entitlements/api/entitlements/v2" -# - name: ENTITLEMENTS_SERVICE_API_KEY -# value: "OBSOLETE" -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-schema -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-schema -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# installationType: osduCore -# jobs: -# partitionInit: false -# entitlementInit: false -# userInit: false -# schemaInit: true -# elasticInit: false -# partition: opendes -# clientSecret: -# name: active-directory -# key: principal-clientpassword -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: clientId -# valuesKey: client_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: tenantId -# valuesKey: tenant_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: serviceBus -# valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-schema + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-indexer-service + namespace: osdu-core + - name: osdu-indexer-queue + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: schema + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: schema + path: /api/schema-service/v1/ + hosts: + - "*" + cors: + - "http://localhost:8080" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/system/schema-service/schema-service- + tag: latest + probe: + path: /actuator/health + port: 8081 + keyvault: true + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/schema-service/v1/info" + - "/api/schema-service/v1/swagger*" + - "/api/schema-service/v1/api-docs*" + - "/api/schema-service/v2/webjars/*" + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_SERVLET_CONTEXTPATH + value: "/api/schema-service/v1/" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: "schema" + - name: COSMOSDB_DATABASE + value: "osdu-db" + - name: LOG_PREFIX + value: "schema" + - name: AZURE_SYSTEM_STORAGECONTAINERNAME + value: "system" + - name: SERVICEBUS_TOPIC_NAME + value: "schemachangedtopic" + - name: EVENT_GRID_ENABLED + value: 'false' + - name: EVENT_GRID_TOPIC + value: "schemachangedtopic" + - name: SERVICE_BUS_ENABLED + value: 'true' + - name: PARTITION_SERVICE_ENDPOINT + value: "http://partition/api/partition/v1" + - name: ENTITLEMENTS_SERVICE_ENDPOINT + value: "http://entitlements/api/entitlements/v2" + - name: ENTITLEMENTS_SERVICE_API_KEY + value: "OBSOLETE" +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-schema + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-schema + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + installationType: osduCore + jobs: + partitionInit: false + entitlementInit: false + userInit: false + schemaInit: true + elasticInit: false + partition: opendes + clientSecret: + name: active-directory + key: principal-clientpassword + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: clientId + valuesKey: client_id + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id + - kind: ConfigMap + name: configmap-services + targetPath: serviceBus + valuesKey: partition_servicebus_name_0 ## This is the first data partition service bus name \ No newline at end of file diff --git a/software/applications/osdu-core/search.yaml b/software/applications/osdu-core/search.yaml index 76b77283..864ac275 100644 --- a/software/applications/osdu-core/search.yaml +++ b/software/applications/osdu-core/search.yaml @@ -1,121 +1,121 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-search -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-indexer-queue -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: search -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: search -# path: /api/search/v2/ -# hosts: -# - "*" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/system/search-service/search-service- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# request: -# cpu: 1000m -# memory: 1Gi -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/search/v2/info" -# - "/api/search/v2/swagger*" -# - "/api/search/v2/api-docs*" -# - "/api/search/v2/webjars/*" -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: search -# - name: SERVER_SERVLET_CONTEXTPATH -# value: /api/search/v2/ -# - name: LOG_PREFIX -# value: "search" -# - name: SEARCH_SERVICE_SPRING_LOGGING_LEVEL -# value: "DEBUG" -# - name: COSMOSDB_DATABASE -# value: "osdu-db" -# - name: REDIS_DATABASE -# value: "5" -# - name: ENVIRONMENT -# value: "evt" -# - name: ELASTIC_CACHE_EXPIRATION -# value: 1 -# - name: MAX_CACHE_VALUE_SIZE -# value: 60 -# - name: POLICY_SERVICE_ENABLED -# value: "false" -# - name: ENTITLEMENTS_SERVICE_API_KEY -# value: "OBSOLETE" -# - name: PARTITION_SERVICE_ENDPOINT -# value: "http://partition/api/partition/v1" -# - name: ENTITLEMENTS_SERVICE_ENDPOINT -# value: "http://entitlements/api/entitlements/v2" -# - name: POLICY_SERVICE_ENDPOINT -# value: http://policy/api/policy/v1 \ No newline at end of file +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-search + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-indexer-queue + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: search + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: search + path: /api/search/v2/ + hosts: + - "*" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/system/search-service/search-service- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + request: + cpu: 1000m + memory: 1Gi + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/search/v2/info" + - "/api/search/v2/swagger*" + - "/api/search/v2/api-docs*" + - "/api/search/v2/webjars/*" + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: search + - name: SERVER_SERVLET_CONTEXTPATH + value: /api/search/v2/ + - name: LOG_PREFIX + value: "search" + - name: SEARCH_SERVICE_SPRING_LOGGING_LEVEL + value: "DEBUG" + - name: COSMOSDB_DATABASE + value: "osdu-db" + - name: REDIS_DATABASE + value: "5" + - name: ENVIRONMENT + value: "evt" + - name: ELASTIC_CACHE_EXPIRATION + value: 1 + - name: MAX_CACHE_VALUE_SIZE + value: 60 + - name: POLICY_SERVICE_ENABLED + value: "false" + - name: ENTITLEMENTS_SERVICE_API_KEY + value: "OBSOLETE" + - name: PARTITION_SERVICE_ENDPOINT + value: "http://partition/api/partition/v1" + - name: ENTITLEMENTS_SERVICE_ENDPOINT + value: "http://entitlements/api/entitlements/v2" + - name: POLICY_SERVICE_ENDPOINT + value: http://policy/api/policy/v1 \ No newline at end of file diff --git a/software/applications/osdu-core/storage.yaml b/software/applications/osdu-core/storage.yaml index e7ed6a86..f503481f 100644 --- a/software/applications/osdu-core/storage.yaml +++ b/software/applications/osdu-core/storage.yaml @@ -1,138 +1,138 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-storage -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-indexer-queue -# namespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# targetNamespace: osdu-core -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: storage -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: storage -# path: /api/storage/v2/ -# hosts: -# - "*" -# cors: -# - "http://localhost:8080" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/system/storage/storage- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# request: -# cpu: 800m -# memory: 1Gi -# # limit: -# # cpu: 1000m -# # memory: 4Gi -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/storage/v2/info" -# - "/api/storage/v2/swagger*" -# - "/api/storage/v2/api-docs*" -# - "/api/storage/v2/webjars/*" -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: storage -# - name: SERVER_SERVLET_CONTEXTPATH -# value: /api/storage/v2/ -# - name: COSMOSDB_DATABASE -# value: osdu-db -# - name: AZURE_EVENTGRID_ENABLED -# value: "false" -# - name: AZURE_SERVICEBUS_ENABLED -# value: "true" -# - name: SERVICEBUS_TOPIC_NAME -# value: recordstopic -# - name: SERVICEBUS_V2_TOPIC_NAME -# value: recordstopic-v2 -# - name: REDIS_DATABASE -# value: "4" -# - name: PARTITION_SERVICE_ENDPOINT -# value: http://partition/api/partition/v1 -# - name: ENTITLEMENTS_SERVICE_ENDPOINT -# value: http://entitlements/api/entitlements/v2 -# - name: ENTITLEMENTS_SERVICE_API_KEY -# value: "OBSOLETE" -# - name: LEGAL_SERVICE_ENDPOINT -# value: http://legal/api/legal/v1 -# - name: LEGAL_SERVICE_REGION -# value: southcentralus -# - name: LEGAL_SERVICEBUS_TOPIC_NAME -# value: legaltagschangedtopiceg -# - name: LEGAL_SERVICEBUS_TOPIC_SUBSCRIPTION -# value: eg_sb_legaltagchangedsubscription -# - name: CRS_CONVERSION_SERVICE_ENDPOINT -# value: http://crs-conversion/api/crs/converter/v2 -# - name: POLICY_SERVICE_ENDPOINT -# value: http://policy/api/policy/v1 -# - name: OPA_ENABLED -# value: "false" -# - name: REDIS_HOST_KEY -# value: redis-hostname -# - name: REDIS_PASSWORD_KEY -# value: redis-password +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-storage + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-indexer-queue + namespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + targetNamespace: osdu-core + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: storage + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: storage + path: /api/storage/v2/ + hosts: + - "*" + cors: + - "http://localhost:8080" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/system/storage/storage- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + request: + cpu: 800m + memory: 1Gi + # limit: + # cpu: 1000m + # memory: 4Gi + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/storage/v2/info" + - "/api/storage/v2/swagger*" + - "/api/storage/v2/api-docs*" + - "/api/storage/v2/webjars/*" + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: storage + - name: SERVER_SERVLET_CONTEXTPATH + value: /api/storage/v2/ + - name: COSMOSDB_DATABASE + value: osdu-db + - name: AZURE_EVENTGRID_ENABLED + value: "false" + - name: AZURE_SERVICEBUS_ENABLED + value: "true" + - name: SERVICEBUS_TOPIC_NAME + value: recordstopic + - name: SERVICEBUS_V2_TOPIC_NAME + value: recordstopic-v2 + - name: REDIS_DATABASE + value: "4" + - name: PARTITION_SERVICE_ENDPOINT + value: http://partition/api/partition/v1 + - name: ENTITLEMENTS_SERVICE_ENDPOINT + value: http://entitlements/api/entitlements/v2 + - name: ENTITLEMENTS_SERVICE_API_KEY + value: "OBSOLETE" + - name: LEGAL_SERVICE_ENDPOINT + value: http://legal/api/legal/v1 + - name: LEGAL_SERVICE_REGION + value: southcentralus + - name: LEGAL_SERVICEBUS_TOPIC_NAME + value: legaltagschangedtopiceg + - name: LEGAL_SERVICEBUS_TOPIC_SUBSCRIPTION + value: eg_sb_legaltagchangedsubscription + - name: CRS_CONVERSION_SERVICE_ENDPOINT + value: http://crs-conversion/api/crs/converter/v2 + - name: POLICY_SERVICE_ENDPOINT + value: http://policy/api/policy/v1 + - name: OPA_ENABLED + value: "false" + - name: REDIS_HOST_KEY + value: redis-hostname + - name: REDIS_PASSWORD_KEY + value: redis-password diff --git a/software/applications/osdu-core/workflow.yaml b/software/applications/osdu-core/workflow.yaml index 2b2b7ce1..752c5bd8 100644 --- a/software/applications/osdu-core/workflow.yaml +++ b/software/applications/osdu-core/workflow.yaml @@ -1,195 +1,195 @@ -# --- -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-workflow -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# spec: -# dependsOn: -# - name: osdu-partition -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-service -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# timeout: 6m -# install: -# remediation: -# retries: 3 -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# values: -# nameOverride: workflow -# installationType: osduCore -# subset: m24 -# replicaCount: 1 -# service: -# type: ClusterIP -# port: 80 -# configuration: -# - service: workflow -# path: /api/workflow/ -# hosts: -# - "*" -# cors: -# - "http://localhost:8080" -# gateways: -# - istio-system/internal-gateway -# - istio-system/external-gateway -# repository: community.opengroup.org:5555/osdu/platform/data-flow/ingestion/ingestion-workflow/ingestion-workflow- -# tag: latest -# probe: -# path: /actuator/health -# port: 8081 -# liveness: -# delay: 250 -# seconds: 10 -# keyvault: true -# auth: -# disable: -# - "*/actuator/health" -# - "*/health" -# - "*/_ah/**" -# - "*/configuration/ui" -# - "*/configuration/security" -# - "/api/workflow/v3/info" -# - "/api/workflow/v3/swagger*" -# - "/api/workflow/v3/api-docs*" -# - "/api/workflow/v3/webjars/*" -# env: -# - name: KEYVAULT_URI -# secret: -# name: azure-resources -# key: keyvault-uri -# - name: AAD_CLIENT_ID -# secret: -# name: active-directory -# key: principal-clientid -# - name: APPINSIGHTS_KEY -# secret: -# name: azure-resources -# key: insights-key -# - name: APPLICATIONINSIGHTS_CONNECTION_STRING -# secret: -# name: azure-resources -# key: insights-connection -# - name: AZURE_ISTIOAUTH_ENABLED -# value: "true" -# - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED -# value: "true" -# - name: SERVER_SERVLET_CONTEXTPATH -# value: "/api/workflow/" -# - name: SERVER_PORT -# value: "80" -# - name: ACCEPT_HTTP -# value: "true" -# - name: SPRING_APPLICATION_NAME -# value: "workflow" -# - name: SPRING_CONFIG_NAME -# value: "common,application" -# - name: LOG_PREFIX -# value: "workflow" -# - name: AZURE_STORAGE_ENABLE_HTTPS -# value: "true" -# - name: COSMOSDB_DATABASE -# value: "osdu-db" -# - name: COSMOSDB_SYSTEM_DATABASE -# value: osdu-system-db -# - name: AIRFLOW_STORAGE_ACCOUNT_NAME -# secret: -# name: azure-resources -# key: azurestorageaccountname -# - name: AIRFLOW_STORAGE_ACCOUNT_KEY -# secret: -# name: azure-resources -# key: azurestorageaccountkey -# - name: OSDU_AIRFLOW_USERNAME -# secret: -# name: azure-resources -# key: airflow-username -# - name: OSDU_AIRFLOW_PASSWORD -# secret: -# name: azure-resources -# key: airflow-password -# - name: AUTHORIZEAPI -# value: http://entitlements/api/entitlements/v2 -# - name: AUTHORIZEAPIKEY -# value: "OBSOLETE" -# - name: PARTITION_SERVICE_ENDPOINT -# value: "http://partition/api/partition/v1" -# - name: OSDU_ENTITLEMENTS_URL -# value: "http://entitlements/api/entitlements/v2" -# - name: OSDU_AIRFLOW_URL -# value: "http://airflow-web.airflow.svc.cluster.local:8080/airflow" -# - name: OSDU_ENTITLEMENTS_APPKEY -# value: "OBSOLETE" -# - name: OSDU_AIRFLOW_VERSION2_ENABLED -# value: true -# - name: DP_AIRFLOW_FOR_SYSTEM_DAG -# value: "false" -# - name: IGNORE_DAGCONTENT -# value: "true" -# - name: IGNORE_CUSTOMOPERATORCONTENT -# value: "true" -# --- -# # Retrigger: kubectl annotate helmrelease osdu-init-workflow fluxcd.io/retrigger=$(date +%s) -n osdu-core -# apiVersion: helm.toolkit.fluxcd.io/v2beta1 -# kind: HelmRelease -# metadata: -# name: osdu-init-workflow -# namespace: osdu-core -# annotations: -# clusterconfig.azure.com/use-managed-source: "true" -# fluxcd.io/retrigger: "initial" -# spec: -# dependsOn: -# - name: osdu-workflow -# namespace: osdu-core -# targetNamespace: osdu-core -# chart: -# spec: -# chart: ./charts/osdu-developer-init -# sourceRef: -# kind: GitRepository -# name: flux-system -# namespace: flux-system -# interval: 5m0s -# install: -# remediation: -# retries: 3 -# values: -# installationType: osduCore -# jobs: -# workflowInit: true -# workflows: -# - name: "Osdu_ingest" -# description: "Manifest Ingest workflow for OSDU" -# - name: "Osdu_ingest_by_reference" -# description: "Manifest Ingest by reference workflow for OSDU" -# - name: 'csv-parser' -# description: 'CSV Parser workflow for OSDU' -# partition: opendes -# clientSecret: -# name: active-directory -# key: principal-clientpassword -# valuesFrom: -# - kind: ConfigMap -# name: configmap-software -# valuesKey: value.yaml -# - kind: ConfigMap -# name: configmap-services -# targetPath: clientId -# valuesKey: client_id -# - kind: ConfigMap -# name: configmap-services -# targetPath: tenantId -# valuesKey: tenant_id +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-workflow + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" +spec: + dependsOn: + - name: osdu-partition + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + timeout: 6m + install: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + values: + nameOverride: workflow + installationType: osduCore + subset: m24 + replicaCount: 1 + service: + type: ClusterIP + port: 80 + configuration: + - service: workflow + path: /api/workflow/ + hosts: + - "*" + cors: + - "http://localhost:8080" + gateways: + - istio-system/internal-gateway + - istio-system/external-gateway + repository: community.opengroup.org:5555/osdu/platform/data-flow/ingestion/ingestion-workflow/ingestion-workflow- + tag: latest + probe: + path: /actuator/health + port: 8081 + liveness: + delay: 250 + seconds: 10 + keyvault: true + auth: + disable: + - "*/actuator/health" + - "*/health" + - "*/_ah/**" + - "*/configuration/ui" + - "*/configuration/security" + - "/api/workflow/v3/info" + - "/api/workflow/v3/swagger*" + - "/api/workflow/v3/api-docs*" + - "/api/workflow/v3/webjars/*" + env: + - name: KEYVAULT_URI + secret: + name: azure-resources + key: keyvault-uri + - name: AAD_CLIENT_ID + secret: + name: active-directory + key: principal-clientid + - name: APPINSIGHTS_KEY + secret: + name: azure-resources + key: insights-key + - name: APPLICATIONINSIGHTS_CONNECTION_STRING + secret: + name: azure-resources + key: insights-connection + - name: AZURE_ISTIOAUTH_ENABLED + value: "true" + - name: AZURE_PAAS_WORKLOADIDENTITY_ISENABLED + value: "true" + - name: SERVER_SERVLET_CONTEXTPATH + value: "/api/workflow/" + - name: SERVER_PORT + value: "80" + - name: ACCEPT_HTTP + value: "true" + - name: SPRING_APPLICATION_NAME + value: "workflow" + - name: SPRING_CONFIG_NAME + value: "common,application" + - name: LOG_PREFIX + value: "workflow" + - name: AZURE_STORAGE_ENABLE_HTTPS + value: "true" + - name: COSMOSDB_DATABASE + value: "osdu-db" + - name: COSMOSDB_SYSTEM_DATABASE + value: osdu-system-db + - name: AIRFLOW_STORAGE_ACCOUNT_NAME + secret: + name: azure-resources + key: azurestorageaccountname + - name: AIRFLOW_STORAGE_ACCOUNT_KEY + secret: + name: azure-resources + key: azurestorageaccountkey + - name: OSDU_AIRFLOW_USERNAME + secret: + name: azure-resources + key: airflow-username + - name: OSDU_AIRFLOW_PASSWORD + secret: + name: azure-resources + key: airflow-password + - name: AUTHORIZEAPI + value: http://entitlements/api/entitlements/v2 + - name: AUTHORIZEAPIKEY + value: "OBSOLETE" + - name: PARTITION_SERVICE_ENDPOINT + value: "http://partition/api/partition/v1" + - name: OSDU_ENTITLEMENTS_URL + value: "http://entitlements/api/entitlements/v2" + - name: OSDU_AIRFLOW_URL + value: "http://airflow-web.airflow.svc.cluster.local:8080/airflow" + - name: OSDU_ENTITLEMENTS_APPKEY + value: "OBSOLETE" + - name: OSDU_AIRFLOW_VERSION2_ENABLED + value: true + - name: DP_AIRFLOW_FOR_SYSTEM_DAG + value: "false" + - name: IGNORE_DAGCONTENT + value: "true" + - name: IGNORE_CUSTOMOPERATORCONTENT + value: "true" +--- +# Retrigger: kubectl annotate helmrelease osdu-init-workflow fluxcd.io/retrigger=$(date +%s) -n osdu-core +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: osdu-init-workflow + namespace: osdu-core + annotations: + clusterconfig.azure.com/use-managed-source: "true" + fluxcd.io/retrigger: "initial" +spec: + dependsOn: + - name: osdu-workflow + namespace: osdu-core + targetNamespace: osdu-core + chart: + spec: + chart: ./charts/osdu-developer-init + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 5m0s + install: + remediation: + retries: 3 + values: + installationType: osduCore + jobs: + workflowInit: true + workflows: + - name: "Osdu_ingest" + description: "Manifest Ingest workflow for OSDU" + - name: "Osdu_ingest_by_reference" + description: "Manifest Ingest by reference workflow for OSDU" + - name: 'csv-parser' + description: 'CSV Parser workflow for OSDU' + partition: opendes + clientSecret: + name: active-directory + key: principal-clientpassword + valuesFrom: + - kind: ConfigMap + name: configmap-software + valuesKey: value.yaml + - kind: ConfigMap + name: configmap-services + targetPath: clientId + valuesKey: client_id + - kind: ConfigMap + name: configmap-services + targetPath: tenantId + valuesKey: tenant_id From e1caa4d5922a9aa328804360700234043973ad60 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Mon, 6 Jan 2025 17:14:26 -0600 Subject: [PATCH 118/122] Fixes for indexing properly. --- bicep/modules/blade_partition.bicep | 68 +++++++++++++++-------------- tools/rest-scripts/schema.http | 1 + tools/rest-scripts/storage.http | 6 +-- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/bicep/modules/blade_partition.bicep b/bicep/modules/blade_partition.bicep index 8df7b7e9..e8ff7836 100644 --- a/bicep/modules/blade_partition.bicep +++ b/bicep/modules/blade_partition.bicep @@ -1,5 +1,5 @@ ///////////////// -// Partition Blade +// Partition Blade ///////////////// @description('The configuration for the blade section.') @@ -25,7 +25,7 @@ param cmekConfiguration object = { } @description('The name of the Key Vault where the secret exists') -param kvName string +param kvName string @description('List of Data Partitions') param partitions array = [ @@ -41,7 +41,7 @@ param managedIdentityName string param natClusterIP string ///////////////////////////////// -// Configuration +// Configuration ///////////////////////////////// var partitionLayerConfig = { secrets: { @@ -434,7 +434,7 @@ var partitionLayerConfig = { } ] } - + ] } } @@ -463,12 +463,12 @@ resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = { } -/* _______.___________. ______ .______ ___ _______ _______ +/* _______.___________. ______ .______ ___ _______ _______ / | | / __ \ | _ \ / \ / _____|| ____| - | (----`---| |----`| | | | | |_) | / ^ \ | | __ | |__ - \ \ | | | | | | | / / /_\ \ | | |_ | | __| -.----) | | | | `--' | | |\ \----./ _____ \ | |__| | | |____ -|_______/ |__| \______/ | _| `._____/__/ \__\ \______| |_______| + | (----`---| |----`| | | | | |_) | / ^ \ | | __ | |__ + \ \ | | | | | | | / / /_\ \ | | |_ | | __| +.----) | | | | `--' | | |\ \----./ _____ \ | |__| | | |____ +|_______/ |__| \______/ | _| `._____/__/ \__\ \______| |_______| */ // AVM Module Customized due to required Secrets. @@ -489,7 +489,7 @@ module storage 'storage-account/main.bicep' = [for (partition, index) in partiti purpose: 'data' } ) - + // Hook up Diagnostics diagnosticSettings: [ { @@ -538,7 +538,7 @@ module storage 'storage-account/main.bicep' = [for (partition, index) in partiti ] accessKey1: [ '${partition.name}-${partitionLayerConfig.secrets.storageAccountKey}' - ] + ] blobEndpoint: [ '${partition.name}-${partitionLayerConfig.secrets.storageAccountBlob}' ] @@ -547,7 +547,7 @@ module storage 'storage-account/main.bicep' = [for (partition, index) in partiti }] -module partitionDb './cosmos-db/main.bicep' = [for (partition, index) in partitions: { +module partitionDb './cosmos-db/main.bicep' = [for (partition, index) in partitions: { name: '${bladeConfig.sectionName}-cosmos-db-${index}' params: { #disable-next-line BCP335 @@ -585,7 +585,7 @@ module partitionDb './cosmos-db/main.bicep' = [for (partition, index) in partiti array(systemDatabase), array(partitionDatabase) ) : array(partitionDatabase) - + maxThroughput: partitionLayerConfig.database.throughput backupPolicyType: partitionLayerConfig.database.backup @@ -606,6 +606,24 @@ module partitionDb './cosmos-db/main.bicep' = [for (partition, index) in partiti }] +// First, create a variable to handle the subscription mapping +var topicsWithSubscriptions = [for topic in partitionLayerConfig.servicebus.topics: { + name: topic.name + maxSizeInMegabytes: topic.maxSizeInMegabytes + authorizationRules: [ + { + name: 'RootManageSharedAccessKey' + rights: [ + 'Listen' + 'Manage' + 'Send' + ] + } + ] + subscriptions: topic.subscriptions ?? [] +}] + +// Then use this variable in the module module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.9.1' = [for (partition, index) in partitions: { name: '${bladeConfig.sectionName}-service-bus-${index}' params: { @@ -649,23 +667,7 @@ module partitonNamespace 'br/public:avm/res/service-bus/namespace:0.9.1' = [for } ] - topics: [ - for topic in partitionLayerConfig.servicebus.topics: { - name: topic.name - maxSizeInMegabytes: topic.maxSizeInMegabytes - authorizationRules: [ - { - name: 'RootManageSharedAccessKey' - rights: [ - 'Listen' - 'Manage' - 'Send' - ] - } - ] - subscriptions: topic.subscriptions - } - ] + topics: topicsWithSubscriptions } }] @@ -680,16 +682,16 @@ module blobUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for ( retentionInterval: 'PT1H' timeout: 'PT30M' runOnce: true - + managedIdentities: { userAssignedResourcesIds: [ stampIdentity.id ] - } + } kind: 'AzureCLI' azCliVersion: '2.63.0' - + environmentVariables: [ { name: 'CONTENT', value: loadTextContent('./deploy-scripts/Legal_COO.json') } { name: 'FILE_NAME', value: 'Legal_COO.json' } diff --git a/tools/rest-scripts/schema.http b/tools/rest-scripts/schema.http index 5e0fabf0..d4accfb6 100644 --- a/tools/rest-scripts/schema.http +++ b/tools/rest-scripts/schema.http @@ -138,6 +138,7 @@ Accept: application/json data-partition-id: {{DATA_PARTITION}} +### This call can't be done by a default user. ### # @name createSystemSchema PUT {{SCHEMA_HOST}}/schemas/system diff --git a/tools/rest-scripts/storage.http b/tools/rest-scripts/storage.http index 84a80184..6d102744 100644 --- a/tools/rest-scripts/storage.http +++ b/tools/rest-scripts/storage.http @@ -141,12 +141,11 @@ data-partition-id: {{DATA_PARTITION}} ### # @name getRecord -GET {{STORAGE_HOST}}/records/{{id}} +GET {{STORAGE_HOST}}/records/{{createRecord.response.body.recordIds[0]}} Authorization: Bearer {{access_token}} Accept: application/json data-partition-id: {{DATA_PARTITION}} -@recordversion = {{getRecord.response.body.version}} ### # @name getRecordIdByKind @@ -155,11 +154,10 @@ Authorization: Bearer {{access_token}} Accept: application/json data-partition-id: {{DATA_PARTITION}} -@id = {{getRecordIdByKind.response.body.results[0]}} ### # @name getRecordByVersion -GET {{STORAGE_HOST}}/records/{{id}}/{{recordversion}} +GET {{STORAGE_HOST}}/records/{{id}}/{{getRecord.response.body.version}} Authorization: Bearer {{access_token}} Accept: application/json data-partition-id: {{DATA_PARTITION}} From 05331088822320281b8d279f13953d8c40013ee2 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Tue, 7 Jan 2025 15:59:16 -0600 Subject: [PATCH 119/122] adding WI to airflow pod template. --- software/components/airflow/release.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/software/components/airflow/release.yaml b/software/components/airflow/release.yaml index 9c9c862d..28d419c4 100644 --- a/software/components/airflow/release.yaml +++ b/software/components/airflow/release.yaml @@ -83,6 +83,9 @@ spec: - "--extra-index-url=https://community.opengroup.org/api/v4/projects/823/packages/pypi/simple" - "osdu-ingestion>=0.27,<1.0.0" kubernetesPodTemplate: + serviceAccountName: workload-identity-sa + annotations: + azure.workload.identity/use: "true" extraPipPackages: # - "apache-airflow-providers-microsoft-azure" - "--extra-index-url=https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple" @@ -118,7 +121,7 @@ spec: AIRFLOW__WEBSERVER__WORKER_CLASS: "sync" # AIRFLOW__WEBSERVER__WORKERS: "8" AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth" - AIRFLOW_VAR_CORE__CONFIG__SHOW_SKIPPED_IDS: "True" + AIRFLOW_VAR_CORE__CONFIG__SHOW_SKIPPED_IDS: "True" # AIRFLOW_VAR_CORE__CONFIG__DATALOAD_CONFIG_PATH: "/opt/airflow/dags/configs/dataload.ini" extraEnv: From 06c2dd4e29fc0bd58bc23d9c370465625e78490b Mon Sep 17 00:00:00 2001 From: Daniel Scholl Date: Sun, 26 Jan 2025 15:56:49 +0000 Subject: [PATCH 120/122] Upgrade bicep providers and modules. Configure custom vm sizes. --- .github/workflows/test.yml | 4 +- bicep/main.bicep | 57 +++++++++---------- bicep/main.parameters.json | 7 +++ bicep/modules/blade_cluster.bicep | 50 ++++++++++------ .../managed-cluster/agent-pool/main.bicep | 4 +- .../aks_appconfig_extension.bicep | 2 +- .../modules/managed-cluster/aks_policy.bicep | 4 +- bicep/modules/managed-cluster/main.bicep | 4 +- .../maintenance-configurations/main.bicep | 4 +- scripts/pre-provision.ps1 | 5 ++ 10 files changed, 85 insertions(+), 56 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3e5da83b..2b2e9bbe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,7 +46,7 @@ on: required: false region: description: "Region (needs to be same as byo vnet location)" - default: "eastus2" + default: "centralus" type: string required: false doStandards: @@ -150,7 +150,7 @@ jobs: if [ -z "${{ github.event.inputs.region }}" ] then echo "Region parameter not available through GitHub event data, setting default" - REGION="eastus2" + REGION="centralus" else echo "Region parameter found in GitHub event (${{ github.event.inputs.region }})" REGION="${{ github.event.inputs.region }}" diff --git a/bicep/main.bicep b/bicep/main.bicep index a8f96d2c..3b3de060 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -10,16 +10,9 @@ param emailAddress string @description('Specify the Application Client Id. (This is the unique application ID of this application.)') param applicationClientId string -// @description('Specify the Application Client Secret. (A valid secret for the application client ID.)') -// @secure() -// param applicationClientSecret string - @description('Specify the Enterprise Application Object Id. (This is the unique ID of the service principal object associated with the application.)') param applicationClientPrincipalOid string -@description('The size of the VM to use for the cluster.') -param customVMSize string = '' - @allowed([ 'External' 'Internal' @@ -55,6 +48,13 @@ param clusterConfiguration object = { enableLockDown: false } +@description('Optional: Server Configuration Overrides - {system}-->(4x8 ARM:true) {zone}-->(2x8 ARM:true) {user}-->(4x16 ARM:false BURST:true)') +param serverConfiguration object = { + systemPool: 'Standard_D4pds_v6' + zonePool: 'Standard_D2pds_v6' + userPool: 'Standard_B4s_v2' +} + @description('Optional. Bring your own Virtual Network.') param vnetConfiguration object = { group: '' @@ -189,7 +189,7 @@ module stampIdentity 'br/public:avm/res/managed-identity/user-assigned-identity: / _____ \ | |\ | / _____ \ | `----. | | | | | | | `----.----) | /__/ \__\ |__| \__| /__/ \__\ |_______| |__| |__| |__| \______|_______/ */ -module logAnalytics 'br/public:avm/res/operational-insights/workspace:0.7.1' = { +module logAnalytics 'br/public:avm/res/operational-insights/workspace:0.9.1' = { name: '${configuration.name}-log-analytics' params: { name: rg_unique_id @@ -216,7 +216,7 @@ module logAnalytics 'br/public:avm/res/operational-insights/workspace:0.7.1' = { |__| |__| \__| |_______/ |__| \______| |__| |__| |__| |_______/ */ -module insights 'br/public:avm/res/insights/component:0.3.0' = { +module insights 'br/public:avm/res/insights/component:0.5.0' = { name: '${configuration.name}-insights' params: { name: '${replace(configuration.name, '-', '')}${uniqueString(resourceGroup().id, configuration.name)}' @@ -257,7 +257,7 @@ module insights 'br/public:avm/res/insights/component:0.3.0' = { \______/__/ \__\ \______||__| |__| |_______| */ // This takes a long time to deploy so we are starting as soon as possible. -module redis 'br/public:avm/res/cache/redis:0.3.2' = { +module redis 'br/public:avm/res/cache/redis:0.9.0' = { name: '${configuration.name}-cache' params: { name: '${replace(configuration.name, '-', '')}${uniqueString(resourceGroup().id, configuration.name)}' @@ -376,7 +376,9 @@ module clusterBlade 'modules/blade_cluster.bicep' = { aksSubnetId: enableVnetInjection ? networkBlade.outputs.aksSubnetId : '' podSubnetId: enableVnetInjection ? networkBlade.outputs.podSubnetId : '' - vmSize: customVMSize + vmSizeSystemPool: serverConfiguration.systemPool == '' ? 'Standard_D4pds_v6' : serverConfiguration.systemPool + vmSizeZonePool: serverConfiguration.zonePool == '' ? 'Standard_D2pds_v6' : serverConfiguration.zonePool + vmSizeUserPool: serverConfiguration.userPool == '' ? 'Standard_B4s_v2' : serverConfiguration.userPool } dependsOn: [ stampIdentity @@ -429,16 +431,16 @@ module fluxExtension 'modules/flux-extension/main.bicep' = { .----) | | `----.| |\ \----.| | | | | | |_______/ \______|| _| `._____||__| | _| |__| */ -module extensionClientId 'br/public:avm/res/resources/deployment-script:0.4.0' = { +module extensionClientId 'br/public:avm/res/resources/deployment-script:0.5.1' = { name: '${configuration.name}-script-clientId' params: { kind: 'AzureCLI' name: 'script-${configuration.name}-aks-extension' - azCliVersion: '2.63.0' location: location + azCliVersion: '2.64.0' managedIdentities: { - userAssignedResourcesIds: [ + userAssignedResourceIds: [ stampIdentity.outputs.resourceId ] } @@ -458,6 +460,7 @@ module extensionClientId 'br/public:avm/res/resources/deployment-script:0.4.0' = retentionInterval: 'PT1H' scriptContent: ''' + tdnf install -y jq az login --identity echo "Looking up client ID for $principalId in ResourceGroup $rgName" @@ -481,7 +484,7 @@ module extensionClientId 'br/public:avm/res/resources/deployment-script:0.4.0' = | |\ \----.| |____ | |__| | | | .----) | | | | |\ \----. | | | _| `._____||_______| \______| |__| |_______/ |__| | _| `._____| |__| */ -module registry 'br/public:avm/res/container-registry/registry:0.1.1' = { +module registry 'br/public:avm/res/container-registry/registry:0.7.0' = { name: '${configuration.name}-container-registry' params: { name: '${replace(configuration.name, '-', '')}${uniqueString(resourceGroup().id, configuration.name)}' @@ -553,12 +556,6 @@ var vaultSecrets = [ secretName: 'subscription-id' secretValue: subscription().subscriptionId } - // Azure AD Secrets - // { - // secretName: 'app-dev-sp-password' - // secretValue: 'dummy' - // // secretValue: applicationClientSecret == '' ? 'dummy' : applicationClientSecret - // } { secretName: 'app-dev-sp-id' secretValue: applicationClientId @@ -601,7 +598,7 @@ var vaultSecrets = [ } ] -module keyvault 'br/public:avm/res/key-vault/vault:0.5.1' = { +module keyvault 'br/public:avm/res/key-vault/vault:0.11.2' = { name: '${configuration.name}-keyvault' params: { name: length(name) > 24 ? substring(name, 0, 24) : name @@ -654,12 +651,14 @@ module keyvault 'br/public:avm/res/key-vault/vault:0.5.1' = { } // Configure Secrets - secrets: { - secureList: [for secret in vaultSecrets: { - name: secret.secretName - value: secret.secretValue - }] - } + secrets: [for secret in vaultSecrets: { + name: secret.secretName + value: secret.secretValue + contentType: 'text/plain' + attributes: { + enabled: true + } + }] } } @@ -924,7 +923,7 @@ var directoryUploads = [ ] @batchSize(1) -module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.4.0' = [for item in directoryUploads: if (clusterSoftware.private == 'true') { +module gitOpsUpload 'br/public:avm/res/resources/deployment-script:0.5.1' = [for item in directoryUploads: if (clusterSoftware.private == 'true') { name: '${configuration.name}-storage-${item.directory}-upload' params: { name: 'script-${storage.outputs.name}-${item.directory}' diff --git a/bicep/main.parameters.json b/bicep/main.parameters.json index 34aac859..28420de4 100644 --- a/bicep/main.parameters.json +++ b/bicep/main.parameters.json @@ -33,6 +33,13 @@ "enableLockDown": "${ENABLE_LOCK_DOWN}" } }, + "serverConfiguration": { + "value": { + "systemPool": "${VMSIZE_SYSTEM_POOL}", + "zonePool": "${VMSIZE_ZONE_POOL}", + "userPool": "${VMSIZE_USER_POOL}" + } + }, "vnetConfiguration": { "value": { "group": "${VIRTUAL_NETWORK_GROUP}", diff --git a/bicep/modules/blade_cluster.bicep b/bicep/modules/blade_cluster.bicep index 1f5111cc..b66da5b9 100644 --- a/bicep/modules/blade_cluster.bicep +++ b/bicep/modules/blade_cluster.bicep @@ -17,18 +17,30 @@ param enableTelemetry bool @description('The workspace resource Id for diagnostics') param workspaceResourceId string -@description('A Custom VM Size for Internal Pool') -param vmSize string +// D4pds v5 with 4 vCPUs and 16 GiB of memory. Available in 22 regions starting from $88.18 per month. +// D4s_v5 with 4 vCPUs and 16 GiB of memory. Available in 50 regions starting from $140.16 per month. +@description('A Custom VM Size for System Pool (4x8 ARM:true)') +param vmSizeSystemPool string = 'Standard_D4pds_v6' + +// D2pds v5 with 2 vCPUs and 8 GiB of memory. Available in 22 regions starting from $44.09 per month. +// D2s_v5 with 2 vCPUs and 8 GiB of memory. Available in 50 regions starting from $70.08 per month. +@description('A Custom VM Size for Zone Pool (2x8 ARM:true)') +param vmSizeZonePool string = 'Standard_D2pds_v6' + +// B4s_v2 with 4 vCPUs and 16 GiB of memory. Available in 49 regions starting from $16.64 per month. +// D4s_v5 with 4 vCPUs and 16 GiB of memory. Available in 50 regions starting from $140.16 per month. +@description('A Custom VM Size for User Pool (2x8 ARM:false BURST:true)') +param vmSizeUserPool string = 'Standard_B4s_v2' @minLength(9) @maxLength(18) @description('The address range to use for services') -param serviceCidr string = '172.16.0.0/16' +param serviceCidr string = '10.0.0.0/16' @minLength(7) @maxLength(15) @description('The IP address to reserve for DNS') -param dnsServiceIP string = '172.16.0.10' +param dnsServiceIP string = '10.0.0.10' @description('The id of the subnet to deploy the AKS nodes') param aksSubnetId string @@ -51,6 +63,8 @@ param enablePrivateCluster bool = true @description('Feature Flag to Enable Node Resource Group Lock Down') param nodeResourceGroupLockDown bool = true + + ///////////////////////////////// // Configuration ///////////////////////////////// @@ -61,15 +75,18 @@ var serviceLayerConfig = { } cluster: { tier: 'Standard' + sku: 'Base' aksVersion: '1.30' - // D2pds v5 with 2 vCPUs and 8 GiB of memory. Available in 22 regions starting from $44.09 per month. - // D4pds v5 with 4 vCPUs and 16 GiB of memory. Available in 22 regions starting from $88.18 per month. - // D2s_v5 with 2 vCPUs and 8 GiB of memory. Available in 50 regions starting from $70.08 per month. - // D4s_v5 with 4 vCPUs and 16 GiB of memory. Available in 50 regions starting from $140.16 per month. - vmSize: 'Standard_D4pds_v5' - poolSize: 'Standard_D2pds_v5' - defaultSize: 'Standard_D4s_v5' // OSDU Java Services don't run on ARM? + // // D2pds v5 with 2 vCPUs and 8 GiB of memory. Available in 22 regions starting from $44.09 per month. + // // D4pds v5 with 4 vCPUs and 16 GiB of memory. Available in 22 regions starting from $88.18 per month. + // // D2s_v5 with 2 vCPUs and 8 GiB of memory. Available in 50 regions starting from $70.08 per month. + // // D4s_v5 with 4 vCPUs and 16 GiB of memory. Available in 50 regions starting from $140.16 per month. + // // D4ps_v5 with 4 vCPUs and 16 GiB of memory. Available in 23 regions, starting from $73.73 per month. + // // B4s_v2 with 4 vCPUs and 16 GiB of memory. Available in 49 regions starting from $16.64 per month. + // vmSize: 'Standard_D4pds_v6' + // poolSize: 'Standard_D2pds_v6' + // defaultSize: 'Standard_B4s_v2' // OSDU Java Services don't run on ARM? } } @@ -96,6 +113,7 @@ module cluster './managed-cluster/main.bicep' = { name: '${replace(bladeConfig.sectionName, '-', '')}${uniqueString(resourceGroup().id, bladeConfig.sectionName)}' location: location skuTier: serviceLayerConfig.cluster.tier + skuName: serviceLayerConfig.cluster.sku kubernetesVersion: serviceLayerConfig.cluster.aksVersion // Assign Tags @@ -209,7 +227,7 @@ module cluster './managed-cluster/main.bicep' = { { name: 'system' mode: 'System' - vmSize: empty(vmSize) ? serviceLayerConfig.cluster.vmSize : vmSize + vmSize: vmSizeSystemPool enableAutoScaling: !enableNodeAutoProvisioning count: enableNodeAutoProvisioning ? 2 : null minCount: enableNodeAutoProvisioning ? null : 2 @@ -237,7 +255,7 @@ module cluster './managed-cluster/main.bicep' = { { name: 'default' mode: 'User' - vmSize: empty(vmSize) ? serviceLayerConfig.cluster.defaultSize : vmSize + vmSize: vmSizeUserPool enableAutoScaling: !enableNodeAutoProvisioning count: enableNodeAutoProvisioning ? 4 : null minCount: enableNodeAutoProvisioning ? null : 4 @@ -256,7 +274,7 @@ module cluster './managed-cluster/main.bicep' = { { name: 'poolz1' mode: 'User' - vmSize: empty(vmSize) ? serviceLayerConfig.cluster.poolSize : vmSize + vmSize: vmSizeZonePool enableAutoScaling: !enableNodeAutoProvisioning minCount: enableNodeAutoProvisioning ? null : 1 maxCount: enableNodeAutoProvisioning ? null : 3 @@ -277,7 +295,7 @@ module cluster './managed-cluster/main.bicep' = { { name: 'poolz2' mode: 'User' - vmSize: empty(vmSize) ? serviceLayerConfig.cluster.poolSize : vmSize + vmSize: vmSizeZonePool enableAutoScaling: !enableNodeAutoProvisioning minCount: enableNodeAutoProvisioning ? null : 1 maxCount: enableNodeAutoProvisioning ? null : 3 @@ -298,7 +316,7 @@ module cluster './managed-cluster/main.bicep' = { { name: 'poolz3' mode: 'User' - vmSize: empty(vmSize) ? serviceLayerConfig.cluster.poolSize : vmSize + vmSize: vmSizeZonePool enableAutoScaling: !enableNodeAutoProvisioning minCount: enableNodeAutoProvisioning ? null : 1 maxCount: enableNodeAutoProvisioning ? null : 3 diff --git a/bicep/modules/managed-cluster/agent-pool/main.bicep b/bicep/modules/managed-cluster/agent-pool/main.bicep index 920a9000..807b9176 100644 --- a/bicep/modules/managed-cluster/agent-pool/main.bicep +++ b/bicep/modules/managed-cluster/agent-pool/main.bicep @@ -153,11 +153,11 @@ param workloadRuntime string? ]) param sshAccess string = 'Disabled' -resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-03-02-preview' existing = { +resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-preview' existing = { name: managedClusterName } -resource agentPool 'Microsoft.ContainerService/managedClusters/agentPools@2024-04-02-preview' = { +resource agentPool 'Microsoft.ContainerService/managedClusters/agentPools@2024-09-02-preview' = { name: name parent: managedCluster properties: { diff --git a/bicep/modules/managed-cluster/aks_appconfig_extension.bicep b/bicep/modules/managed-cluster/aks_appconfig_extension.bicep index 59406d80..093450b4 100644 --- a/bicep/modules/managed-cluster/aks_appconfig_extension.bicep +++ b/bicep/modules/managed-cluster/aks_appconfig_extension.bicep @@ -1,7 +1,7 @@ @description('The name of the Managed Cluster resource.') param clusterName string -resource existingManagedCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-preview' existing = { +resource existingManagedCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-preview' existing = { name: clusterName } diff --git a/bicep/modules/managed-cluster/aks_policy.bicep b/bicep/modules/managed-cluster/aks_policy.bicep index f0da2abc..30e93bc4 100644 --- a/bicep/modules/managed-cluster/aks_policy.bicep +++ b/bicep/modules/managed-cluster/aks_policy.bicep @@ -1,12 +1,12 @@ @description('The name of the Azure Kubernetes Service Cluster') param clusterName string = '' -resource managedCluster 'Microsoft.ContainerService/managedClusters@2023-05-02-preview' existing = if (clusterName != '') { +resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-preview' existing = if (clusterName != '') { name: clusterName } var policyDefinitionId = '/providers/Microsoft.Authorization/policySetDefinitions/c047ea8e-9c78-49b2-958b-37e56d291a44' -resource policyAssignment 'Microsoft.Authorization/policyAssignments@2024-04-01' = { +resource policyAssignment 'Microsoft.Authorization/policyAssignments@2024-05-01' = { name: 'aksDeploymentSafeguardsAssignment' scope: managedCluster properties: { diff --git a/bicep/modules/managed-cluster/main.bicep b/bicep/modules/managed-cluster/main.bicep index ee46ce95..3a8f2fed 100644 --- a/bicep/modules/managed-cluster/main.bicep +++ b/bicep/modules/managed-cluster/main.bicep @@ -553,7 +553,7 @@ resource cMKKeyVault 'Microsoft.KeyVault/vaults@2023-02-01' existing = if (!empt // Main Resources // // ============== // -resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-preview' = { +resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-preview' = { name: name location: location tags: tags @@ -870,7 +870,7 @@ module managedCluster_agentPools 'agent-pool/main.bicep' = [ } ] -module managedCluster_extension 'br/public:avm/res/kubernetes-configuration/extension:0.2.0' = if (!empty(fluxExtension)) { +module managedCluster_extension 'br/public:avm/res/kubernetes-configuration/extension:0.3.5' = if (!empty(fluxExtension)) { name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxExtension' params: { clusterName: managedCluster.name diff --git a/bicep/modules/managed-cluster/maintenance-configurations/main.bicep b/bicep/modules/managed-cluster/maintenance-configurations/main.bicep index 52f609f0..e6978a8a 100644 --- a/bicep/modules/managed-cluster/maintenance-configurations/main.bicep +++ b/bicep/modules/managed-cluster/maintenance-configurations/main.bicep @@ -11,11 +11,11 @@ param managedClusterName string @description('Optional. Name of the maintenance configuration.') param name string = 'aksManagedAutoUpgradeSchedule' -resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-03-02-preview' existing = { +resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-09-02-preview' existing = { name: managedClusterName } -resource aksManagedAutoUpgradeSchedule 'Microsoft.ContainerService/managedClusters/maintenanceConfigurations@2023-10-01' = { +resource aksManagedAutoUpgradeSchedule 'Microsoft.ContainerService/managedClusters/maintenanceConfigurations@2024-09-02-preview' = { name: name parent: managedCluster properties: { diff --git a/scripts/pre-provision.ps1 b/scripts/pre-provision.ps1 index f2bf2805..39d1aeca 100644 --- a/scripts/pre-provision.ps1 +++ b/scripts/pre-provision.ps1 @@ -224,6 +224,11 @@ function Set-LocalAuth { try { $appConfig = az appconfig list -g $env:AZURE_RESOURCE_GROUP --query '[0].name' -o tsv + if (-not $appConfig) { + Write-Host "No App Configuration found in resource group: $env:AZURE_RESOURCE_GROUP" + return + } + Write-Host "`n==================================================================" Write-Host "Disabling Local Authentication for App Configuration: $appConfig" Write-Host "==================================================================" From 52bca1724860bd0ad9d86550338a9afb709623a6 Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 26 Jan 2025 11:30:42 -0600 Subject: [PATCH 121/122] Updated documentation. --- bicep/main.parameters.json | 3 --- docs/src/feature_flags.md | 5 ++-- docs/src/getting_started.md | 47 ++++++++++++++++++++++++++++++------- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/bicep/main.parameters.json b/bicep/main.parameters.json index 28420de4..909f8c70 100644 --- a/bicep/main.parameters.json +++ b/bicep/main.parameters.json @@ -14,9 +14,6 @@ "emailAddress": { "value": "${EMAIL_ADDRESS}" }, - "customVMSize": { - "value": "${CLUSTER_VM_SIZE}" - }, "ingressType": { "value": "${CLUSTER_INGRESS}" }, diff --git a/docs/src/feature_flags.md b/docs/src/feature_flags.md index b9afe571..c773c815 100644 --- a/docs/src/feature_flags.md +++ b/docs/src/feature_flags.md @@ -37,7 +37,6 @@ Application registrations are created automatically with a naming convention of | Feature Flag | Description | |---------------------------|-----------------------------------------------------------------------------| | AZURE_CLIENT_ID | Use an existing Azure AD App Client ID | -| AZURE_CLIENT_SECRET | Use an existing Azure AD Client Secret and don't reset it. | | AZURE_CLIENT_PRINCIPAL_OID| Skip Principal ID lookup and use provided. | | AZURE_TENANT_ID | Skip Tenant ID lookup and use provided. | @@ -49,7 +48,9 @@ Infrastructure customizations can be modified using the following feature flags. | Feature Flag | Description | |---------------------------|-----------------------------------------------------------------------------| | CLUSTER_INGRESS | Specifies the Ingress type for the cluster (External, Internal, or Both) | -| CLUSTER_VM_SIZE | Overrides the default server type with a custom VM size | +| VMSIZE_SYSTEM_POOL | Overrides the default server for the system pool. (4x8 ARM) | +| VMSIZE_ZONE_POOL | Overrides the default server for the zone pool. (2x8 ARM) | +| VMSIZE_USER_POOL | Overrides the default server for the default pool. (4x8 INTEL) | | ENABLE_NODE_AUTO_PROVISIONING | Enables node auto provisioning (True by default) | | ENABLE_PRIVATE_CLUSTER | Enables private cluster (False by default) | diff --git a/docs/src/getting_started.md b/docs/src/getting_started.md index 913901d4..8d17b026 100644 --- a/docs/src/getting_started.md +++ b/docs/src/getting_started.md @@ -6,19 +6,48 @@ Prerequisites and configuration steps for deploying personal OSDU™ instances i It is recommended to have at least 50 vCPUs in a region for vCPU families along with the ability to deploy Cosmos DB instances which can be resource constrained in some regions. Defaults can be increased by requesting a [quota increase](https://learn.microsoft.com/en-us/azure/quotas/regional-quota-requests). -!!! note "Ensure Sufficient Quota" - The deployment requires quota for the following VM families: +!!! note "Ensure Sufficient Compute Quota per Region" - - Standard_D4pds_v5 nodes for system workloads - - Standard_D2pds_v5 nodes for zonal workloads - - Standard_D4s_v5 nodes for default workloads + | VM Types | Compute Family Series | + |------------------|---------------------------------| + | Standard ARM Generation | Standard Dpdsv6 Family vCPUs | + | Burstable Intel Generation | Standard Bsv2 Family vCPUS | + + Use the following command to validate the availability of servers in a region: + === "Bash" + ```bash + LOCATION="eastus2" # ie: eastus2, centralus + VM_PATTERN="Standard_D" # ie: Standard_D, Standard_B + + az vm list-skus \ + --location "$LOCATION" \ + --query "[?resourceType=='virtualMachines'] \ + | [?contains(locationInfo[0].zones, '1') && contains(locationInfo[0].zones, '2') && contains(locationInfo[0].zones, '3')] \ + | [?restrictions[0]==null] \ + | [?starts_with(name, '$VM_PATTERN')].{ResourceType:resourceType, Locations:locations[0], Name:name, Zones:join(',', locationInfo[0].zones), Restrictions:join('; ', restrictions[*].reasonCode || ['None'])}" \ + -o table + ``` + + === "PowerShell" + ```powershell + $LOCATION="eastus2" # ie: eastus2, centralus + $VM_PATTERN="Standard_D" # ie: Standard_D, Standard_B + + az vm list-skus ` + --location "$LOCATION" ` + --query "[?resourceType=='virtualMachines'] ` + | [?contains(locationInfo[0].zones, '1') && contains(locationInfo[0].zones, '2') && contains(locationInfo[0].zones, '3')] ` + | [?restrictions[0]==null] ` + | [?starts_with(name, '$VM_PATTERN')].{ResourceType:resourceType, Locations:locations[0], Name:name, Zones:join(',', locationInfo[0].zones), Restrictions:join('; ', restrictions[*].reasonCode || ['None'])}" ` + -o table + ``` | Quota Name | Minimum Quantity | |------------|------------------| -| Total Regional vCPUs | 100 | -| Standard DPDSv5 Family vCPUs | 50 | -| Standard DSv5 Family vCPUs | 50 | +| Total Regional vCPUs | 100 | +| Standard Dpdsv6 Family vCPUs | 50 | +| Standard Bsv2 Family vCPUs | 50 | !!! tip "Available Cosmos DB Regions" @@ -126,7 +155,7 @@ These credentials will be used in your ARM template deployment to authenticate a | Name | Description/Value | -|------|-------------| +|------|-------------| | Directory (tenant) ID | Unique identifier for the Microsoft Entra tenant | | Application (client) ID | Unique identifier for the registered application | | Object ID | Unique identifier for the application object in Microsoft Entra | From 52d66344c04a57e671cc60415817b8045cb1de8e Mon Sep 17 00:00:00 2001 From: danielscholl Date: Sun, 26 Jan 2025 11:38:15 -0600 Subject: [PATCH 122/122] Fixing validate step to remove client secret. --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2b2e9bbe..3b0d9cb8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,7 +74,7 @@ env: AZCLIVERSION: 2.63.0 # https://github.com/Azure/azure-cli/issues/29828 ParamFilePath: ".github/parameters.json" DEPNAME: "dep${{ github.run_number }}" - + jobs: Standards: runs-on: ubuntu-latest @@ -270,8 +270,8 @@ jobs: azcliversion: ${{ env.AZCLIVERSION }} inlineScript: | DEPNAME='Dep${{ github.run_number }}' - - PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES }} applicationClientId=${{ env.AZURE_CLIENT_ID }} applicationClientSecret=${{ secrets.AZURE_CLIENT_SECRET }} applicationClientPrincipalOid=${{ env.AZURE_CLIENT_PRINCIPAL_OID }} emailAddress=${{ secrets.EMAIL_ADDRESS }}' + + PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES }} applicationClientId=${{ env.AZURE_CLIENT_ID }} applicationClientPrincipalOid=${{ env.AZURE_CLIENT_PRINCIPAL_OID }} emailAddress=${{ secrets.EMAIL_ADDRESS }}' echo $PARAMS az deployment group validate -f bicep/main.bicep -g $RESOURCE_GROUP -p ${{ env.ParamFilePath }} -p $PARAMS --verbose @@ -330,7 +330,7 @@ jobs: - name: Install azd uses: Azure/setup-azd@v2.1.0 - + - name: Log in with Azure (Federated Credentials) if: ${{ env.AZURE_CLIENT_ID != '' }} run: |