DAOS-18487 rebuild: don't wait for discard

src/include/daos_srv/pool.h

@@ -88,13 +88,14 @@ struct ds_pool {
 	 */
 	uuid_t			sp_srv_cont_hdl;
 	uuid_t			sp_srv_pool_hdl;
-	uint32_t sp_stopping : 1, sp_cr_checked : 1, sp_immutable : 1, sp_need_discard : 1,
-	    sp_disable_rebuild : 1, sp_disable_dtx_resync : 1, sp_incr_reint : 1;
+	uint32_t sp_stopping : 1, sp_cr_checked : 1, sp_immutable : 1, sp_disable_rebuild : 1,
+	    sp_disable_dtx_resync : 1, sp_incr_reint : 1;
 	/* pool_uuid + map version + leader term + rebuild generation define a
 	 * rebuild job.
 	 */
 	uint32_t                 sp_rebuild_gen;
 	ATOMIC int               sp_rebuilding;
+	ATOMIC int               sp_discarding;
 	/**
 	 * someone has already messaged this pool to for rebuild scan,
 	 * NB: all xstreams can do lockless-write on it but it's OK
@@ -191,8 +192,7 @@ struct ds_pool_child {
 	int		spc_ref;
 	ABT_eventual	spc_ref_eventual;
 
-	uint64_t	spc_discard_done:1,
-			spc_no_storage:1; /* The pool shard has no storage. */
+	uint64_t                 spc_no_storage : 1; /* The pool shard has no storage. */
 
 	uint32_t	spc_reint_mode;
 	uint32_t	*spc_state;	/* Pointer to ds_pool->sp_states[i] */

src/object/srv_obj.c

@@ -2450,20 +2450,22 @@ static int
 obj_inflight_io_check(struct ds_cont_child *child, uint32_t opc,
 		      uint32_t rpc_map_ver, uint32_t flags)
 {
+	struct ds_pool *pool = child->sc_pool->spc_pool;
+
 	if (opc == DAOS_OBJ_RPC_ENUMERATE && flags & ORF_FOR_MIGRATION) {
 		/* EC aggregation is still inflight, rebuild should wait until it's paused */
 		if (ds_cont_child_ec_aggregating(child)) {
 			D_ERROR(DF_CONT " ec aggregate still active, rebuilding %d\n",
-				DP_CONT(child->sc_pool->spc_uuid, child->sc_uuid),
-				atomic_load(&child->sc_pool->spc_pool->sp_rebuilding));
+				DP_CONT(pool->sp_uuid, child->sc_uuid),
+				atomic_load(&pool->sp_rebuilding));
 			return -DER_UPDATE_AGAIN;
 		}
 	}
 
 	if (!obj_is_modification_opc(opc) && (opc != DAOS_OBJ_RPC_CPD || flags & ORF_CPD_RDONLY))
 		return 0;
 
-	if (atomic_load(&child->sc_pool->spc_pool->sp_rebuilding)) {
+	if (atomic_load(&pool->sp_rebuilding)) {
 		uint32_t version;
 
 		ds_rebuild_running_query(child->sc_pool_uuid, RB_OP_REBUILD,
@@ -2480,10 +2482,8 @@ obj_inflight_io_check(struct ds_cont_child *child, uint32_t opc,
 	 * vos discard to finish, which otherwise might discard these new in-flight
 	 * I/O update.
 	 */
-	if ((flags & ORF_REINTEGRATING_IO) &&
-	    (child->sc_pool->spc_pool->sp_need_discard &&
-	     child->sc_pool->spc_discard_done == 0)) {
-		D_ERROR("reintegrating "DF_UUID" retry.\n", DP_UUID(child->sc_pool->spc_uuid));
+	if ((flags & ORF_REINTEGRATING_IO) && atomic_load(&pool->sp_discarding) > 0) {
+		D_ERROR("reintegrating " DF_UUID " retry.\n", DP_UUID(pool->sp_uuid));
 		return -DER_UPDATE_AGAIN;
 	}
 

src/object/srv_obj_migrate.c

@@ -3230,6 +3230,7 @@ migrate_obj_ult(void *data)
 {
 	struct iter_obj_arg	*arg = data;
 	struct migrate_pool_tls	*tls = NULL;
+	struct ds_pool          *pool;
 	daos_epoch_range_t	epr;
 	daos_epoch_t		stable_epoch = 0;
 	daos_handle_t            coh          = DAOS_HDL_INVAL;
@@ -3249,20 +3250,22 @@ migrate_obj_ult(void *data)
 	 * discard, or discard has been done. spc_discard_done means
 	 * discarding has been done in the current VOS target.
 	 */
-	if (tls->mpt_pool->spc_pool->sp_need_discard) {
-		while(!tls->mpt_pool->spc_discard_done) {
-			D_DEBUG(DB_REBUILD, DF_RB ": wait for discard to finish.\n",
-				DP_RB_MPT(tls));
-			dss_sleep(2 * 1000);
-			if (tls->mpt_fini)
-				D_GOTO(free_notls, rc);
-		}
-		if (tls->mpt_pool->spc_pool->sp_discard_status) {
-			rc = tls->mpt_pool->spc_pool->sp_discard_status;
+	pool = tls->mpt_pool->spc_pool;
+	while (atomic_load(&pool->sp_discarding) > 0) {
+		D_DEBUG(DB_REBUILD, DF_RB ": wait for discard to finish.\n", DP_RB_MPT(tls));
+		dss_sleep(2 * 1000);
+		if (tls->mpt_fini)
+			D_GOTO(free_notls, rc);
+
+		ABT_mutex_lock(pool->sp_mutex);
+		if (pool->sp_discard_status) {
+			rc = pool->sp_discard_status;
+			ABT_mutex_unlock(pool->sp_mutex);
 			D_DEBUG(DB_REBUILD, DF_RB ": discard failure: " DF_RC "\n", DP_RB_MPT(tls),
 				DP_RC(rc));
 			D_GOTO(out, rc);
 		}
+		ABT_mutex_unlock(pool->sp_mutex);
 	}
 
 	if (tls->mpt_reintegrating) {