[patch] patch postgredb-tekton-results-postgres-0 PVC with storageclass when no default available

src/mas/devops/mas.py

@@ -12,6 +12,7 @@
 import re
 import yaml
 from os import path
+from time import sleep
 from types import SimpleNamespace
 from kubernetes.dynamic.resource import ResourceInstance
 from openshift.dynamic import DynamicClient
@@ -145,8 +146,8 @@ def verifyMasInstance(dynClient: DynamicClient, instanceId: str) -> bool:
     except ResourceNotFoundError:
         # The MAS Suite CRD has not even been installed in the cluster
         return False
-    except UnauthorizedError:
-        logger.error("Error: Unable to verify MAS instance due to failed authorization: {e}")
+    except UnauthorizedError as e:
+        logger.error(f"Error: Unable to verify MAS instance due to failed authorization: {e}")
         return False
 
 
@@ -194,3 +195,64 @@ def updateIBMEntitlementKey(dynClient: DynamicClient, namespace: str, icrUsernam
 
     secret = secretsAPI.apply(body=secret, namespace=namespace)
     return secret
+
+
+def waitForPVC(dynClient: DynamicClient, namespace: str, pvcName: str) -> bool:
+    pvcAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolumeClaim")
+    maxRetries = 60
+    foundReadyPVC = False
+    retries = 0
+    while not foundReadyPVC and retries < maxRetries:
+        retries += 1
+        try:
+            pvc = pvcAPI.get(name=pvcName, namespace=namespace)
+            if pvc.status.phase == "Bound":
+                foundReadyPVC = True
+            else:
+                logger.debug(f"Waiting 5s for PVC {pvcName} to be ready before checking again ...")
+                sleep(5)
+        except NotFoundError:
+            logger.debug(f"Waiting 5s for PVC {pvcName} to be created before checking again ...")
+            sleep(5)
+
+    return foundReadyPVC
+
+
+def patchPendingPVC(dynClient: DynamicClient, namespace: str, pvcName: str, storageClassName: str = None) -> bool:
+    pvcAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolumeClaim")
+    try:
+        pvc = pvcAPI.get(name=pvcName, namespace=namespace)
+        if pvc.status.phase == "Pending" and pvc.spec.storageClassName is None:
+            if storageClassName is not None and storageClassName(dynClient, name=storageClassName) is not None:
+                pvc.spec.storageClassName = storageClassName
+            else:
+                defaultStorageClasses = getDefaultStorageClasses(dynClient)
+                if defaultStorageClasses.provider is not None:
+                    pvc.spec.storageClassName = defaultStorageClasses.rwo
+                else:
+                    logger.error(f"Unable to set storageClassName in PVC {pvcName}.")
+                    return False
+
+            pvcAPI.patch(body=pvc, namespace=namespace)
+
+            maxRetries = 60
+            foundReadyPVC = False
+            retries = 0
+            while not foundReadyPVC and retries < maxRetries:
+                retries += 1
+                try:
+                    patchedPVC = pvcAPI.get(name=pvcName, namespace=namespace)
+                    if patchedPVC.status.phase == "Bound":
+                        foundReadyPVC = True
+                    else:
+                        logger.debug(f"Waiting 5s for PVC {pvcName} to be bound before checking again ...")
+                        sleep(5)
+                except NotFoundError:
+                    logger.error(f"The patched PVC {pvcName} does not exist.")
+                    return False
+
+            return foundReadyPVC
+
+    except NotFoundError:
+        logger.error(f"PVC {pvcName} does not exist")
+        return False

src/mas/devops/ocp.py

@@ -150,10 +150,10 @@ def waitForDeployment(dynClient: DynamicClient, namespace: str, deploymentName:
                 # NoneType and int comparison TypeError
                 foundReadyDeployment = True
             else:
-                logger.debug("Waiting 5s for deployment {deploymentName} to be ready before checking again ...")
+                logger.debug(f"Waiting 5s for deployment {deploymentName} to be ready before checking again ...")
                 sleep(5)
         except NotFoundError:
-            logger.debug("Waiting 5s for deployment {deploymentName} to be created before checking again ...")
+            logger.debug(f"Waiting 5s for deployment {deploymentName} to be created before checking again ...")
             sleep(5)
     return foundReadyDeployment
 

src/mas/devops/tekton.py

@@ -23,11 +23,14 @@
 from jinja2 import Environment, FileSystemLoader
 
 from .ocp import getConsoleURL, waitForCRD, waitForDeployment, crdExists
+from .mas import waitForPVC, patchPendingPVC
 
 logger = logging.getLogger(__name__)
 
 
-def installOpenShiftPipelines(dynClient: DynamicClient) -> bool:
+# customStorageClassName is used when no default Storageclass is available on cluster,
+# openshift-pipelines creates PVC which looks for default. customStorageClassName is patched into PVC when default is unavailable.
+def installOpenShiftPipelines(dynClient: DynamicClient, customStorageClassName: str = None) -> bool:
     """
     Install the OpenShift Pipelines Operator and wait for it to be ready to use
     """
@@ -79,11 +82,30 @@ def installOpenShiftPipelines(dynClient: DynamicClient) -> bool:
     foundReadyWebhook = waitForDeployment(dynClient, namespace="openshift-pipelines", deploymentName="tekton-pipelines-webhook")
     if foundReadyWebhook:
         logger.info("OpenShift Pipelines Webhook is installed and ready")
-        return True
     else:
         logger.error("OpenShift Pipelines Webhook is NOT installed and ready")
         return False
 
+    # Wait for the postgredb-tekton-results-postgres-0 PVC to be ready
+    # this PVC doesn't come up when there's no default storage class is in the cluster,
+    # this is causing the pvc to be in pending state and causing the tekton-results-postgres statefulSet in pending,
+    # due to these resources not coming up, the MAS pre-install check in the pipeline times out checking the health of this statefulSet,
+    # causing failure in pipeline.
+    # Refer https://github.com/ibm-mas/cli/issues/1511
+    logger.debug("Waiting for postgredb-tekton-results-postgres-0 PVC to be ready")
+    foundReadyPVC = waitForPVC(dynClient, namespace="openshift-pipelines", pvcName="postgredb-tekton-results-postgres-0")
+    if foundReadyPVC:
+        logger.info("OpenShift Pipelines postgres is installed and ready")
+        return True
+    else:
+        patchedPVC = patchPendingPVC(dynClient, namespace="openshift-pipelines", pvcName="postgredb-tekton-results-postgres-0", storageClassName=customStorageClassName)
+        if patchedPVC:
+            logger.info("OpenShift Pipelines postgres is installed and ready")
+            return True
+        else:
+            logger.error("OpenShift Pipelines postgres PVC is NOT ready")
+            return False
+
 
 def updateTektonDefinitions(namespace: str, yamlFile: str) -> None:
     """