SFI Compliance

bicep/main.bicep

@@ -1,8 +1,10 @@
 targetScope = 'resourceGroup'
 
 
+// BUG RIGHT NOW.  https://github.com/Azure/azure-dev/pull/5061
 @description('Specify the Azure region to place the application definition.')
-param location string = resourceGroup().location
+// param location string = resourceGroup().location
+param location string
 
 @description('Specify the User Email.')
 param emailAddress string
@@ -137,14 +139,16 @@ var configuration = {
       'azure-webjobs-hosts'
       'azure-webjobs-eventhub'
       'gitops'
+      'airflow-logs'
+      'airflow-dags'
+      'share-unit'
+      'share-crs'
+      'share-crs-conversion'
     ]
     tables: [
       'partitionInfo'
     ]
-    shares: [
-      'airflow-logs'
-      'airflow-dags'
-    ]
+    shares: []
   }
   partitions: [
     {
@@ -776,11 +780,12 @@ module storage 'modules/storage-account/main.bicep' = {
     ]
 
     // Apply Security
-    allowBlobPublicAccess: false
+    // allowBlobPublicAccess: false`  
     publicNetworkAccess: 'Enabled'
+    enableHierarchicalNamespace: false    // <--- Airflow doesn't like hiearchical.
 
     // TODO: This is required for Partition Service to access the storage account. Issue: https://github.com/Azure/osdu-developer/issues/230
-    allowSharedKeyAccess: true
+    allowSharedKeyAccess: false
 
     // https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template?tabs=CLI#debug-deployment-scripts
     networkAcls: {
@@ -1062,6 +1067,7 @@ module configBlade 'modules/blade_configuration.bicep' = {
     managedIdentityName: stampIdentity.outputs.name
     kvName: keyvault.outputs.name
     kvUri: keyvault.outputs.uri
+    storageAccountName: storage.outputs.name
     appInsightsKey: insights.outputs.instrumentationKey
     partitionStorageNames: partitionBlade.outputs.partitionStorageNames
     partitionServiceBusNames: partitionBlade.outputs.partitionServiceBusNames

bicep/main.parameters.json

@@ -5,9 +5,6 @@
     "applicationClientId": {
       "value": "${AZURE_CLIENT_ID}"
     },
-    "applicationClientSecret": {
-      "value": "${AZURE_CLIENT_SECRET}"
-    },
     "applicationClientPrincipalOid": {
       "value": "${AZURE_CLIENT_PRINCIPAL_OID}"
     },

bicep/modules/blade_cluster.bicep

@@ -190,7 +190,7 @@ module cluster './managed-cluster/main.bicep' = {
     enableStorageProfileDiskCSIDriver: true
     enableStorageProfileFileCSIDriver: true
     enableStorageProfileSnapshotController: true
-    enableStorageProfileBlobCSIDriver: true    
+    enableStorageProfileBlobCSIDriver: false    
     enableKeyvaultSecretsProvider: true
     enableSecretRotation: true
     enableImageCleaner: true

bicep/modules/blade_configuration.bicep

@@ -81,6 +81,9 @@ param osduVersion string = 'master'
 @description('The managed identity name for deployment scripts')
 param managedIdentityName string
 
+@description('The name of the system storage account')
+param storageAccountName string
+
 @description('The name of the partition storage accounts')
 param partitionStorageNames string[]
 
@@ -392,7 +395,7 @@ var configMaps = {
 values.yaml: |
   serviceAccount:
     create: false
-    name: "workload-identity-sa"
+    name: workload-identity-sa
   azure:
     tenantId: {0}
     clientId: {1}
@@ -403,12 +406,15 @@ values.yaml: |
     appId: {6}
     appOid: {7}
     resourceGroup: {8}
+    storageAccountName: {11}
   ingress:
     internalGateway:
       enabled: {9}
     externalGateway:
       enabled: {10}
-  '''
+  workloadIdentity:
+    clientID: {1}
+'''
 }
 
 /*
@@ -445,7 +451,8 @@ module appConfigMap './aks-config-map/main.bicep' = {
              applicationClientPrincipalOid,
              resourceGroup().name,
              clusterIngress == 'Internal' || clusterIngress == 'Both' ? 'true' : 'false',
-             clusterIngress == 'External' || clusterIngress == 'Both' ? 'true' : 'false')
+             clusterIngress == 'External' || clusterIngress == 'Both' ? 'true' : 'false',
+             storageAccountName)
     ]
   }
 }

bicep/modules/blade_partition.bicep

@@ -518,7 +518,7 @@ module storage 'storage-account/main.bicep' = [for (partition, index) in partiti
     // Apply Security
     allowBlobPublicAccess: enableBlobPublicAccess
     publicNetworkAccess: 'Enabled'
-    allowSharedKeyAccess: true
+    allowSharedKeyAccess: false
     // https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template?tabs=CLI#debug-deployment-scripts
     networkAcls: {
       bypass: 'AzureServices'
@@ -602,9 +602,33 @@ module partitionDb './cosmos-db/main.bicep' = [for (partition, index) in partiti
     databaseEndpointSecretName: '${partition.name}-${partitionLayerConfig.secrets.cosmosEndpoint}'
     databasePrimaryKeySecretName: '${partition.name}-${partitionLayerConfig.secrets.cosmosPrimaryKey}'
     databaseConnectionStringSecretName: '${partition.name}-${partitionLayerConfig.secrets.cosmosConnectionString}'
+
+    roleAssignments: [
+      {
+        roleDefinitionIdOrName: 'Contributor'
+        principals: [
+          {
+            id: stampIdentity.properties.principalId
+          }
+        ]
+        principalType: 'ServicePrincipal'
+      }
+    ]
   }
 }]
 
+// Add SQL role assignment for the Cosmos DB account
+module sqlRoleAssignment './cosmosdb-sql-role-assignment.bicep' = [for (partition, index) in partitions: {
+  name: '${bladeConfig.sectionName}-cosmos-db-sql-role-${index}'
+  params: {
+    databaseAccountName: partitionDb[index].outputs.name
+    principalId: stampIdentity.properties.principalId
+    roleDefinitionId: '${partitionDb[index].outputs.id}/sqlRoleDefinitions/00000000-0000-0000-0000-000000000002' // Built-in Cosmos DB Built-in Data Contributor
+  }
+  dependsOn: [
+    partitionDb[index]
+  ]
+}]
 
 // First, create a variable to handle the subscription mapping
 var topicsWithSubscriptions = [for topic in partitionLayerConfig.servicebus.topics: {

bicep/modules/cosmos-db/main.bicep

@@ -340,7 +340,7 @@ var databaseAccount_properties = union({
 
 
 // Create Database Account
-resource databaseAccount 'Microsoft.DocumentDB/databaseAccounts@2022-08-15' = {
+resource databaseAccount 'Microsoft.DocumentDB/databaseAccounts@2024-11-15' = {
   name: length(name) > 26 ? substring(name, 0, 26) : name
   location: resourceLocation
   tags: tags

bicep/modules/cosmosdb-sql-role-assignment.bicep

@@ -0,0 +1,35 @@
+@description('Name of the Cosmos DB account')
+param databaseAccountName string
+
+@description('Name of the SQL database (if scope is at database level)')
+@allowed([ '', 'database' ])
+param databaseName string = ''
+
+@description('Optional custom scope. If provided, overrides databaseName logic.')
+param customScope string = ''
+
+@description('Principal (Object ID) of the identity to assign')
+param principalId string
+
+@description('Full resource ID of the Cosmos DB SQL role definition')
+param roleDefinitionId string
+
+resource cosmosDbAccount 'Microsoft.DocumentDB/databaseAccounts@2024-11-15' existing = {
+  name: databaseAccountName
+}
+
+var resolvedScope = !empty(customScope)
+  ? customScope
+  : !empty(databaseName)
+    ? '${cosmosDbAccount.id}/sqlDatabases/${databaseName}'
+    : cosmosDbAccount.id
+
+resource sqlRoleAssignment 'Microsoft.DocumentDB/databaseAccounts/sqlRoleAssignments@2024-11-15' = {
+  name: guid(cosmosDbAccount.id, principalId, roleDefinitionId)
+  parent: cosmosDbAccount
+  properties: {
+    roleDefinitionId: roleDefinitionId
+    principalId: principalId
+    scope: resolvedScope
+  }
+}

charts/osdu-developer-base/templates/storage-share-job.yaml

@@ -1,17 +1,4 @@
 {{- if .Values.share.enabled -}}
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: {{ .Release.Name }}-pvc
-  namespace: {{ .Release.Namespace }}
-spec:
-  storageClassName: osdu-managed-disk
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 1Gi
----
 {{- range $index, $item := .Values.share.items }}
 apiVersion: batch/v1
 kind: Job
@@ -28,9 +15,9 @@ spec:
         configMap:
           name: storage-share-script
           defaultMode: 0500
-      - name: {{ $.Release.Name }}-storage
+      - name: storage
         persistentVolumeClaim:
-          claimName: {{ $.Release.Name }}-pvc
+          claimName: {{ $item.pvc }}
       initContainers:
       - name: data-seed
         image: mcr.microsoft.com/cbl-mariner/base/core:2.0
@@ -43,7 +30,7 @@ spec:
         volumeMounts:
           - name: script
             mountPath: "/script"
-          - name: {{ $.Release.Name }}-storage
+          - name: storage
             mountPath: "/share"
         env:
           - name: URL
@@ -96,19 +83,16 @@ data:
       exit 1
     fi
 
-    # Create the target directory in /share mounted volume
-    mkdir -p /share/${SHARE}
-
     # Check if the URL indicates a tar.gz file
     if [[ ${URL} == *.tar.gz ]]; then
         echo "URL indicates a tar.gz archive. Extracting contents..."
-        
+
         # Create a directory for extracted files
         mkdir -p extracted_files
-        
+
         # Extract the tar.gz file
         tar -xzf ${url_basename} --strip-components=1 -C extracted_files
-        
+
         if [[ ${compress} == "True" ]]; then
             echo "Creating zip of contents of ${FILE} and copying it to /share"
             # Remove the original downloaded tar file
@@ -124,23 +108,33 @@ data:
 
             # Create the zip from the contents without including the extracted_files/${FILE} path itself
             zip -r ${original_dir}/${zip_filename} *
-    
+
             # Navigate back to the original directory
             cd ${original_dir}
-    
+
             # Copy the zip file to the /share mounted volume
-            cp ${zip_filename} /share/${SHARE}
+            cp ${zip_filename} /share/
             echo "Zip file ${zip_filename} copied to /share."
         else
-            # Batch copy the extracted files to the /share mounted volume
             echo "Copying extracted files to /share with pattern ${FILE}/**"
-            cp -r extracted_files/${FILE} /share/${SHARE}
+
+            SOURCE_DIR="extracted_files/${FILE}"
+            TARGET_DIR="/share/${FILE}"
+
+            cd "$SOURCE_DIR"
+
+            find . -type f | while read -r file; do
+              relative_path="${file#./}"
+              dest_path="${TARGET_DIR}/${relative_path}"
+              install -D "$file" "$dest_path"
+              echo "Copied: $file -> $dest_path"
+            done
         fi
         echo "Files from ${url_basename} copied to /share."
     else
         # Copy the file to the /share mounted volume
         echo "Copying file ${FILE} to /share"
-        cp ${FILE} /share/${SHARE}
+        cp ${FILE} /share/
         echo "File ${FILE} copied to /share."
     fi
 {{- end }}

charts/osdu-developer-base/values.yaml

@@ -1,4 +1,3 @@
-
 fullnameOverride: osdu-svc
 
 ################################################################################
@@ -27,9 +26,11 @@ share:
   enabled: false
   items:
     - name: folder
+      pvc: ""
       file: "file"
       url: "https://example.com/folder/file.json"
     - name: item2
+      pvc: ""
       url: "https://example.com/archive.tar.gz"
       file: "folder_in_archive"
       compress: true

charts/storage-volumes/Chart.yaml

@@ -0,0 +1,27 @@
+apiVersion: v2
+name: storage-volumes
+description: A Chart for creating kubernetes persistent volumes and claims
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+appVersion: 0.0.1
+
+maintainers:
+  - name: danielscholl
+    url: https://github.com/danielscholl