Skip to content

DAOS-19036 dtx: handle DTX race issues - b28 #18458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Nasf-Fan wants to merge 1 commit into
base: release/2.8
Choose a base branch
from
Open

DAOS-19036 dtx: handle DTX race issues - b28 #18458

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 13 additions & 10 deletions src/dtx/dtx_coll.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,13 @@
*/

struct dtx_coll_local_args {
uuid_t dcla_po_uuid;
uuid_t dcla_co_uuid;
struct dtx_id dcla_xid;
daos_epoch_t dcla_epoch;
uint32_t dcla_opc;
int *dcla_results;
uuid_t dcla_po_uuid;
uuid_t dcla_co_uuid;
struct dtx_id dcla_xid;
daos_epoch_t dcla_epoch;
uint32_t dcla_ver;
uint32_t dcla_opc;
int *dcla_results;
};

void
Expand Down Expand Up @@ -361,7 +362,7 @@ dtx_coll_local_one(void *args)
rc = vos_dtx_commit(cont->sc_hdl, &dcla->dcla_xid, 1, false, NULL);
break;
case DTX_COLL_ABORT:
rc = vos_dtx_abort(cont->sc_hdl, &dcla->dcla_xid, dcla->dcla_epoch);
rc = vos_dtx_abort(cont->sc_hdl, &dcla->dcla_xid, dcla->dcla_epoch, dcla->dcla_ver);
break;
case DTX_COLL_CHECK:
rc = vos_dtx_check(cont->sc_hdl, &dcla->dcla_xid, NULL, NULL, NULL, false);
Expand Down Expand Up @@ -397,7 +398,8 @@ dtx_coll_local_one(void *args)

int
dtx_coll_local_exec(uuid_t po_uuid, uuid_t co_uuid, struct dtx_id *xid, daos_epoch_t epoch,
uint32_t opc, uint32_t bitmap_sz, uint8_t *bitmap, int **p_results)
uint32_t version, uint32_t opc, uint32_t bitmap_sz, uint8_t *bitmap,
int **p_results)
{
struct dtx_coll_local_args dcla = { 0 };
struct dss_coll_ops coll_ops = { 0 };
Expand All @@ -410,9 +412,10 @@ dtx_coll_local_exec(uuid_t po_uuid, uuid_t co_uuid, struct dtx_id *xid, daos_epo

uuid_copy(dcla.dcla_po_uuid, po_uuid);
uuid_copy(dcla.dcla_co_uuid, co_uuid);
dcla.dcla_xid = *xid;
dcla.dcla_xid = *xid;
dcla.dcla_epoch = epoch;
dcla.dcla_opc = opc;
dcla.dcla_ver = version;
dcla.dcla_opc = opc;

coll_ops.co_func = dtx_coll_local_one;
coll_args.ca_func_args = &dcla;
Expand Down
5 changes: 3 additions & 2 deletions src/dtx/dtx_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/**
* (C) Copyright 2019-2024 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
* (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -303,7 +303,8 @@ int dtx_coll_prep(uuid_t po_uuid, daos_unit_oid_t oid, struct dtx_id *xid,
struct dtx_memberships *mbs, uint32_t my_tgtid, uint32_t dtx_ver,
uint32_t pm_ver, bool for_check, bool need_hint, struct dtx_coll_entry **p_dce);
int dtx_coll_local_exec(uuid_t po_uuid, uuid_t co_uuid, struct dtx_id *xid, daos_epoch_t epoch,
uint32_t opc, uint32_t bitmap_sz, uint8_t *bitmap, int **p_results);
uint32_t version, uint32_t opc, uint32_t bitmap_sz, uint8_t *bitmap,
int **p_results);
/* clang-format on */

enum dtx_status_handle_result {
Expand Down
37 changes: 27 additions & 10 deletions src/dtx/dtx_resync.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/**
* (C) Copyright 2019-2024 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
* (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -36,11 +36,12 @@ struct dtx_resync_head {
};

struct dtx_resync_args {
struct ds_cont_child *cont;
struct dtx_resync_head tables;
daos_epoch_t epoch;
uint32_t resync_version;
uint32_t discard_version;
struct ds_cont_child *cont;
struct dtx_resync_head tables;
daos_epoch_t epoch;
uint32_t resync_version;
uint32_t discard_version;
bool for_all;
};

static inline void
Expand Down Expand Up @@ -329,8 +330,8 @@ dtx_status_handle_one(struct ds_cont_child *cont, struct dtx_entry *dte, daos_un
else
rc = dtx_abort(cont, dte, epoch);

D_DEBUG(DB_TRACE, "As new leader for DTX "DF_DTI", abort it (2): "DF_RC"\n",
DP_DTI(&dte->dte_xid), DP_RC(rc));
DL_CDEBUG(rc != 0, DLOG_ERR, DB_TRACE, rc,
"As new leader for DTX " DF_DTI ", abort it (2)", DP_DTI(&dte->dte_xid));

if (rc < 0) {
if (err != NULL)
Expand Down Expand Up @@ -392,7 +393,8 @@ dtx_status_handle(struct dtx_resync_args *dra)
again:
d_list_for_each_entry_safe(dre, next, &drh->drh_list, dre_link) {
if (dre->dre_dte.dte_ver < dra->discard_version) {
err = vos_dtx_abort(cont->sc_hdl, &dre->dre_xid, dre->dre_epoch);
err = vos_dtx_abort(cont->sc_hdl, &dre->dre_xid, dre->dre_epoch,
dre->dre_dte.dte_ver);
if (err == -DER_NONEXIST)
err = 0;
if (err != 0)
Expand Down Expand Up @@ -538,7 +540,17 @@ dtx_iter_cb(uuid_t co_uuid, vos_iter_entry_t *ent, void *args)
if (dra->resync_version == dra->discard_version)
return 0;

/* Skip unprepared entry which version is at least not older than discard version. */
/*
* The DTX version maybe refreshed via obj_handle_resend(). It means that either the
* DTX is generated against the latest pool map or related IO RPC is resent by client
* after pool map changed. Under both cases, the DTX resync that is triggered for pool
* map changes (@for_all is false) should not handle such DTX to avoid making conflict
* commit/abort decision (against regular IO handler) by race.
*/
if ((ent->ie_dtx_ver > dra->resync_version) ||
(ent->ie_dtx_ver == dra->resync_version && !dra->for_all))
return 0;

if (ent->ie_dtx_tgt_cnt == 0)
return 0;

Expand Down Expand Up @@ -665,6 +677,11 @@ dtx_resync(daos_handle_t po_hdl, struct ds_cont_child *cont, uint32_t ver, bool
D_INIT_LIST_HEAD(&dra.tables.drh_list);
dra.tables.drh_count = 0;

if (block)
dra.for_all = false;
else
dra.for_all = true;

/*
* Trigger DTX reindex. That will avoid DTX_CHECK from others being blocked.
* It is harmless even if (committed) DTX entries have already been re-indexed.
Expand Down
19 changes: 10 additions & 9 deletions src/dtx/dtx_rpc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -961,7 +961,7 @@ dtx_abort(struct ds_cont_child *cont, struct dtx_entry *dte, daos_epoch_t epoch)
* to resend sometime later.
*/
if (epoch != 0)
rc1 = vos_dtx_abort(cont->sc_hdl, &dte->dte_xid, epoch);
rc1 = vos_dtx_abort(cont->sc_hdl, &dte->dte_xid, epoch, dte->dte_ver);
else
rc1 = vos_dtx_set_flags(cont->sc_hdl, &dte->dte_xid, 1, DTE_CORRUPTED);
if (rc1 > 0 || rc1 == -DER_NONEXIST)
Expand Down Expand Up @@ -1233,7 +1233,8 @@ dtx_refresh_internal(struct ds_cont_child *cont, int *check_count, d_list_t *che
d_list_del(&dsp->dsp_link);
dtx_dsp_free(dsp);
} else {
rc1 = vos_dtx_abort(cont->sc_hdl, &dsp->dsp_xid, dsp->dsp_epoch);
rc1 = vos_dtx_abort(cont->sc_hdl, &dsp->dsp_xid, dsp->dsp_epoch,
dsp->dsp_version);
D_ASSERT(rc1 != -DER_NO_PERM);

if (rc1 == 0 || !for_io) {
Expand Down Expand Up @@ -1653,8 +1654,8 @@ dtx_coll_commit(struct ds_cont_child *cont, struct dtx_coll_entry *dce, struct d
if (dce->dce_bitmap != NULL) {
clrbit(dce->dce_bitmap, dss_get_module_info()->dmi_tgt_id);
len = dtx_coll_local_exec(cont->sc_pool_uuid, cont->sc_uuid, &dce->dce_xid, 0,
DTX_COLL_COMMIT, dce->dce_bitmap_sz, dce->dce_bitmap,
&results);
dce->dce_ver, DTX_COLL_COMMIT, dce->dce_bitmap_sz,
dce->dce_bitmap, &results);
if (len < 0) {
rc1 = len;
} else {
Expand Down Expand Up @@ -1736,8 +1737,8 @@ dtx_coll_abort(struct ds_cont_child *cont, struct dtx_coll_entry *dce, daos_epoc
if (dce->dce_bitmap != NULL) {
clrbit(dce->dce_bitmap, dss_get_module_info()->dmi_tgt_id);
len = dtx_coll_local_exec(cont->sc_pool_uuid, cont->sc_uuid, &dce->dce_xid, epoch,
DTX_COLL_ABORT, dce->dce_bitmap_sz, dce->dce_bitmap,
&results);
dce->dce_ver, DTX_COLL_ABORT, dce->dce_bitmap_sz,
dce->dce_bitmap, &results);
if (len < 0) {
rc1 = len;
} else {
Expand All @@ -1757,7 +1758,7 @@ dtx_coll_abort(struct ds_cont_child *cont, struct dtx_coll_entry *dce, daos_epoc
}

if (epoch != 0)
rc2 = vos_dtx_abort(cont->sc_hdl, &dce->dce_xid, epoch);
rc2 = vos_dtx_abort(cont->sc_hdl, &dce->dce_xid, epoch, dce->dce_ver);
else
rc2 = vos_dtx_set_flags(cont->sc_hdl, &dce->dce_xid, 1, DTE_CORRUPTED);
if (rc2 > 0 || rc2 == -DER_NONEXIST)
Expand Down Expand Up @@ -1793,8 +1794,8 @@ dtx_coll_check(struct ds_cont_child *cont, struct dtx_coll_entry *dce, daos_epoc

if (dce->dce_bitmap != NULL) {
len = dtx_coll_local_exec(cont->sc_pool_uuid, cont->sc_uuid, &dce->dce_xid, epoch,
DTX_COLL_CHECK, dce->dce_bitmap_sz, dce->dce_bitmap,
&results);
dce->dce_ver, DTX_COLL_CHECK, dce->dce_bitmap_sz,
dce->dce_bitmap, &results);
if (len < 0) {
rc1 = len;
} else {
Expand Down
6 changes: 3 additions & 3 deletions src/dtx/dtx_srv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/**
* (C) Copyright 2019-2024 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
* (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -241,7 +241,7 @@ dtx_handler(crt_rpc_t *rpc)

rc = vos_dtx_abort(cont->sc_hdl,
(struct dtx_id *)din->di_dtx_array.ca_arrays,
din->di_epoch);
din->di_epoch, din->di_version);
} else {
rc = vos_dtx_set_flags(cont->sc_hdl,
(struct dtx_id *)din->di_dtx_array.ca_arrays,
Expand Down Expand Up @@ -464,7 +464,7 @@ dtx_coll_handler(crt_rpc_t *rpc)
}

len = dtx_coll_local_exec(dci->dci_po_uuid, dci->dci_co_uuid, &dci->dci_xid, dci->dci_epoch,
opc, bitmap_sz, bitmap, &results);
dci->dci_version, opc, bitmap_sz, bitmap, &results);
if (len < 0)
D_GOTO(out, rc = len);

Expand Down
11 changes: 6 additions & 5 deletions src/include/daos_srv/vos.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/**
* (C) Copyright 2015-2024 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP.
* (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP.
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -199,14 +199,15 @@ vos_dtx_commit(daos_handle_t coh, struct dtx_id dtis[], int count, bool keep_act
/**
* Abort the specified DTXs.
*
* \param coh [IN] Container open handle.
* \param dti [IN] The DTX identifiers to be aborted.
* \param epoch [IN] The max epoch for the DTX to be aborted.
* \param coh [IN] Container open handle.
* \param dti [IN] The DTX identifiers to be aborted.
* \param epoch [IN] The max epoch for the DTX to be aborted.
* \param version [IN] The max version for the DTX to be aborted.
*
* \return Zero on success, negative value if error.
*/
int
vos_dtx_abort(daos_handle_t coh, struct dtx_id *dti, daos_epoch_t epoch);
vos_dtx_abort(daos_handle_t coh, struct dtx_id *dti, daos_epoch_t epoch, uint32_t version);

/**
* Discard the active DTX entry's records if invalid.
Expand Down
Loading
Loading