Merge branch 'idesai_dg_bugs' into 'master'

Dataguard Bugs Resolution

See merge request rac-docker-dev/oracle-database-operator!249

Author: yunus-qureshi

commons/database/constants.go

@@ -104,6 +104,10 @@ const StandbyDatabasePrerequisitesSQL string = "ALTER SYSTEM SET db_create_file_
 	"\nALTER SYSTEM SET STANDBY_FILE_MANAGEMENT=AUTO;" +
 	"\nALTER SYSTEM SET dg_broker_start=TRUE;"
 
+const GetDBOpenMode string = "select open_mode from v\\$database;"
+
+const ModifyStdbyDBOpenMode string = "alter database recover managed standby database disconnect;"
+
 const StandbyTnsnamesEntry string = `
 ##STANDBYDATABASE_SID## =
 (DESCRIPTION =
@@ -208,6 +212,9 @@ const DataguardBrokerAddDBMaxAvailabilityCMD string = "ADD DATABASE ${ORACLE_SID
 	"(CONNECT_DATA=(SERVICE_NAME=${ORACLE_SID}_DGMGRL)(INSTANCE_NAME=${ORACLE_SID})(SERVER=DEDICATED)))';" +
 	"\nENABLE CONFIGURATION;"
 
+const RemoveStandbyDBFromDGConfgCMD string = "DISABLE DATABASE ${ORACLE_SID};" + 
+  	"\nREMOVE DATABASE ${ORACLE_SID};"
+
 const DBShowConfigCMD string = "SHOW CONFIGURATION;"
 
 const DataguardBrokerGetDatabaseCMD string = "SELECT DATABASE || ':' || DATAGUARD_ROLE AS DATABASE FROM V\\$DG_BROKER_CONFIG;"

commons/database/utils.go

@@ -505,6 +505,26 @@ func GetDatabaseRole(readyPod corev1.Pod, r client.Reader,
 	return "", errors.New("database role is nil")
 }
 
+func GetDatabaseOpenMode(readyPod corev1.Pod, r client.Reader,
+	config *rest.Config, ctx context.Context, req ctrl.Request, edition string) (string, error) {
+		log := ctrllog.FromContext(ctx).WithValues("GetDatabaseOpenMode",req.NamespacedName)
+
+		out,err := ExecCommand(r, config, readyPod.Name, readyPod.Namespace, "", ctx, req, false, "bash", "-c",
+			fmt.Sprintf("echo -e \"%s\" | %s",GetDBOpenMode,SQLPlusCLI))
+		if err != nil {
+			return "",err
+		}
+		log.Info(out)
+		if !strings.Contains(out, "no rows selected") && !strings.Contains(out, "ORA-") {
+			out1 := strings.Replace(out, " ", "_", -1)
+			// filtering output and storing databse_role in  "database_role"
+			databaseOpenMode := strings.Fields(out1)[2]
+			// first 2 values in the slice will be column name(DATABASE_ROLE) and a seperator(--------------) .
+			return databaseOpenMode, nil
+		}
+		return "", errors.New("database open mode is nil")
+	}
+
 // Returns true if any of the pod in 'pods' is with pod.Status.Phase == phase
 func IsAnyPodWithStatus(pods []corev1.Pod, phase corev1.PodPhase) (bool, corev1.Pod) {
 	anyPodWithPhase := false

config/samples/sidb/dataguardbroker.yaml

@@ -25,5 +25,5 @@ spec:
   ## Protection Mode for dg configuration . MaxAvailability or MaxPerformance
   protectionMode: MaxAvailability
 
-  ## Manual Switchover to this database to make it primary(if not already) . 
+  ## Manual Switchover to this database to make it primary(if not already), requires target Database SID . 
   setAsPrimaryDatabase: ""

controllers/database/dataguardbroker_controller.go

@@ -403,7 +403,7 @@ func (r *DataguardBrokerReconciler) setupDataguardBrokerConfiguration(m *dbapi.D
 
 		// Update Databases
 		r.updateReconcileStatus(m, sidbReadyPod, ctx, req)
-	}
+	}  
 
 	eventReason := "DG Configuration up to date"
 	eventMsg := ""
@@ -657,26 +657,28 @@ func (r *DataguardBrokerReconciler) setupDataguardBrokerConfigurationForGivenDB(
 	}
 
 	// ## SET PROPERTY FASTSTARTFAILOVERTARGET FOR EACH DATABASE TO ALL OTHER DATABASES IN DG CONFIG .
-	for i := 0; i < len(databases); i++ {
-		out, err = dbcommons.ExecCommand(r, r.Config, standbyDatabaseReadyPod.Name, standbyDatabaseReadyPod.Namespace, "", ctx, req, false, "bash", "-c",
-			fmt.Sprintf("dgmgrl sys@%s \"EDIT DATABASE %s SET PROPERTY FASTSTARTFAILOVERTARGET=%s\"< admin.pwd", primaryConnectString,
-				strings.Split(databases[i], ":")[0], getFSFOTargets(i, databases)))
-		if err != nil {
-			log.Error(err, err.Error())
-			return requeueY
-		}
-		log.Info("SETTING FSFO TARGET OUTPUT")
-		log.Info(out)
+	if (m.Spec.FastStartFailOver.Enable == true){
+		for i := 0; i < len(databases); i++ {
+			out, err = dbcommons.ExecCommand(r, r.Config, standbyDatabaseReadyPod.Name, standbyDatabaseReadyPod.Namespace, "", ctx, req, false, "bash", "-c",
+				fmt.Sprintf("dgmgrl sys@%s \"EDIT DATABASE %s SET PROPERTY FASTSTARTFAILOVERTARGET=%s\"< admin.pwd", primaryConnectString,
+					strings.Split(databases[i], ":")[0], getFSFOTargets(i, databases)))
+			if err != nil {
+				log.Error(err, err.Error())
+				return requeueY
+			}
+			log.Info("SETTING FSFO TARGET OUTPUT")
+			log.Info(out)
 
-		out, err = dbcommons.ExecCommand(r, r.Config, standbyDatabaseReadyPod.Name, standbyDatabaseReadyPod.Namespace, "", ctx, req, false, "bash", "-c",
-			fmt.Sprintf("dgmgrl sys@%s \"SHOW DATABASE %s FASTSTARTFAILOVERTARGET\" < admin.pwd", primaryConnectString, strings.Split(databases[i], ":")[0]))
-		if err != nil {
-			log.Error(err, err.Error())
-			return requeueY
-		}
-		log.Info("FSFO TARGETS OF " + databases[i])
-		log.Info(out)
+			out, err = dbcommons.ExecCommand(r, r.Config, standbyDatabaseReadyPod.Name, standbyDatabaseReadyPod.Namespace, "", ctx, req, false, "bash", "-c",
+				fmt.Sprintf("dgmgrl sys@%s \"SHOW DATABASE %s FASTSTARTFAILOVERTARGET\" < admin.pwd", primaryConnectString, strings.Split(databases[i], ":")[0]))
+			if err != nil {
+				log.Error(err, err.Error())
+				return requeueY
+			}
+			log.Info("FSFO TARGETS OF " + databases[i])
+			log.Info(out)
 
+		}
 	}
 	// Remove admin pwd file
 	_, err = dbcommons.ExecCommand(r, r.Config, standbyDatabaseReadyPod.Name, standbyDatabaseReadyPod.Namespace, "", ctx, req, true, "bash", "-c",
@@ -694,6 +696,59 @@ func (r *DataguardBrokerReconciler) setupDataguardBrokerConfigurationForGivenDB(
 	return requeueN
 }
 
+// #############################################################################
+//
+//	Remove up DG Configuration for a given StandbyDatabase - To be Tested
+//
+// #############################################################################
+func (r *DataguardBrokerReconciler) removeDataguardBrokerConfigurationForGivenDB( m *dbapi.DataguardBroker,n *dbapi.SingleInstanceDatabase,standbyDatabase *dbapi.SingleInstanceDatabase, standbyDatabaseReadyPod corev1.Pod, sidbReadyPod corev1.Pod, ctx context.Context, req ctrl.Request) ctrl.Result {
+	log := r.Log.WithValues("removeDataguardBrokerConfigurationForGivenDB", req.NamespacedName)
+
+	if standbyDatabaseReadyPod.Name == "" || sidbReadyPod.Name == ""{
+		return requeueY
+	}
+
+	// ## CHECK IF DG CONFIGURATION IS AVAILABLE IN PRIMARY DATABASE ##
+	out, err := dbcommons.ExecCommand(r, r.Config, sidbReadyPod.Name, sidbReadyPod.Namespace, "", ctx, req, false, "bash", "-c", 
+		fmt.Sprintf("echo -e \"%s\" | dgmgrl / as sysdba", dbcommons.DBShowConfigCMD))
+	
+	if err != nil {
+		log.Error(err, err.Error())
+		return requeueY
+	}
+	log.Info("Showconfiguration Output")
+	log.Info(out)
+
+	if strings.Contains(out, "ORA-16525") {
+		log.Info("ORA-16525: The Oracle Data Guard broker is not yet available on Primary")
+		return requeueY
+	}
+
+	// ## REMOVING STANDBY DATABASE FROM DG CONFIGURATION ##
+	out, err = dbcommons.ExecCommand(r, r.Config, sidbReadyPod.Name, sidbReadyPod.Namespace, "", ctx, req, false, "bash", "-c",
+		fmt.Sprintf(dbcommons.CreateDGMGRLScriptFile, dbcommons.RemoveStandbyDBFromDGConfgCMD))
+
+	if err != nil {
+		log.Error(err, err.Error())
+		return requeueY
+	}
+
+	// ## SHOW CONFIGURATION 
+	out, err = dbcommons.ExecCommand(r, r.Config, sidbReadyPod.Name, sidbReadyPod.Namespace, "", ctx, req, false, "bash", "-c",
+		fmt.Sprintf("echo -e \"%s\" | dgmgrl / as sysdba", dbcommons.DBShowConfigCMD))
+	if err != nil {
+		log.Error(err, err.Error())
+		return requeueY
+	}
+	log.Info("Showconfiguration Output")
+	log.Info(out)
+	// Set DG Configured status to false for this standbyDatabase. so that in next reconcilation, we dont configure this again
+	standbyDatabase.Status.DgBrokerConfigured = false
+	r.Status().Update(ctx, standbyDatabase)
+
+	return requeueN
+}
+
 // #############################################################################
 //
 //	Return FSFO targets of each StandbyDatabase
@@ -976,6 +1031,33 @@ func (r *DataguardBrokerReconciler) manageDataguardBrokerDeletion(req ctrl.Reque
 	// indicated by the deletion timestamp being set.
 	isDataguardBrokerMarkedToBeDeleted := m.GetDeletionTimestamp() != nil
 	if isDataguardBrokerMarkedToBeDeleted {
+
+		// Make a singleinstancedatabase with empty 
+		singleInstanceDatabase := &dbapi.SingleInstanceDatabase{}
+		err := r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: m.Spec.PrimaryDatabaseRef}, singleInstanceDatabase)
+		if err != nil {
+			log.Error(err,err.Error())
+			return requeueY,err
+		}
+		// Get its POD
+		// Validate if Primary Database Reference is ready
+		result, sidbReadyPod, _ := r.validateSidbReadiness(m, singleInstanceDatabase, ctx, req)
+		if result.Requeue {
+			log.Info("Reconcile queued")
+			return result, nil
+		}
+		// Get its Role 
+		out, err := dbcommons.GetDatabaseRole(sidbReadyPod, r, r.Config, ctx, req, singleInstanceDatabase.Spec.Edition)
+		// check if its PRIMARY
+		if strings.ToUpper(out) != "PRIMARY" {
+			eventReason := "Deletion"
+			eventMsg := "DataGuard Broker cannot be deleted since primaryDatabaseRef is not in PRIMARY role"
+			log.Info("DataGuard Broker cannot be deleted since primaryDatabaseRef is not in PRIMARY role")
+			r.Recorder.Eventf(m, corev1.EventTypeWarning, eventReason, eventMsg)
+			return requeueN, nil
+		}
+		// if not PRIMARY throw error and log it 
+
 		if controllerutil.ContainsFinalizer(m, dataguardBrokerFinalizer) {
 			// Run finalization logic for dataguardBrokerFinalizer. If the
 			// finalization logic fails, don't remove the finalizer so

controllers/database/singleinstancedatabase_controller.go

@@ -229,6 +229,24 @@ func (r *SingleInstanceDatabaseReconciler) Reconcile(ctx context.Context, req ct
 		if err != nil {
 			return requeueY, err
 		}
+		
+		databaseOpenMode,err := dbcommons.GetDatabaseOpenMode(readyPod, r, r.Config, ctx, req, singleInstanceDatabase.Spec.Edition)
+	
+		if err != nil {
+			r.Log.Error(err, err.Error())
+			return requeueY, err
+		}
+		r.Log.Info("DB openMode Output")
+		r.Log.Info(databaseOpenMode)
+		if databaseOpenMode == "READ_ONLY" {
+			out, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", ctx, req, false, "bash", "-c",fmt.Sprintf("echo -e  \"%s\"  | %s",dbcommons.ModifyStdbyDBOpenMode,dbcommons.SQLPlusCLI))
+			if err != nil {
+				r.Log.Error(err, err.Error())
+				return requeueY, err
+			}
+			r.Log.Info("Standby DB open mode modified")
+			r.Log.Info(out)
+		}
 
 		singleInstanceDatabase.Status.PrimaryDatabase = referredPrimaryDatabase.Name
 		// Store all standbyDatabase sid:name in a map to use it during manual switchover.
@@ -498,6 +516,7 @@ func (r *SingleInstanceDatabaseReconciler) validate(m *dbapi.SingleInstanceDatab
 		if m.Spec.Sid == rp.Spec.Sid {
 			r.Log.Info("Standby database SID can not be same as the Primary database SID")
 			r.Recorder.Eventf(m, corev1.EventTypeWarning, "Spec Error", "Standby and Primary database SID can not be same")
+			m.Status.Status = dbcommons.StatusError
 			return requeueY, err
 		}
 

docs/sidb/README.md

@@ -735,16 +735,16 @@ $ kubectl apply -f dataguardbroker.yaml
   dataguardbroker.database.oracle.com/dataguardbroker-sample apply
 
 ```
+Or use the patch command 
 
-#### Patching DataGuardBroker Attributes
+```sh
+$ kubectl --type=merge -p '{"spec":{"setAsPrimaryDatabase":"ORCLS1"}}' patch dataguardbroker dataguardbroker-sample
 
-The following attributes cannot be patched post DataguardBroker resource Creation : `primaryDatabaseRef, standbyDatabaseRefs, loadBalancer`.
+  dataguardbroker.database.oracle.com/dataguardbroker-sample patched
+```
 
-```sh
-$ kubectl --type=merge -p '{"spec":{"primaryDatabaseRef":"ORCL"}}' patch dataguardbroker dataguardbroker-sample 
+**NOTE :** The following attributes cannot be patched post DataguardBroker resource Creation : `primaryDatabaseRef,    protectionMode`.
 
-  The DataguardBroker "dataguardbroker-sample" is invalid: spec.sid: Forbidden: cannot be changed
-  ```
 
 #### Delete DataguardBroker Resource