From d1bccee11a7182aa27b0419ba0e066481c885ecb Mon Sep 17 00:00:00 2001 From: Damien Daspit Date: Thu, 25 Jun 2026 14:19:39 -0400 Subject: [PATCH 1/2] Add swAlignModel_getNumSentencePairs to C API --- src/shared_library/thot.cc | 6 ++++++ src/shared_library/thot.h | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/shared_library/thot.cc b/src/shared_library/thot.cc index f89625ef..d8029f21 100644 --- a/src/shared_library/thot.cc +++ b/src/shared_library/thot.cc @@ -555,6 +555,12 @@ extern "C" return alignmentModel->getMaxSentenceLength(); } + unsigned int swAlignModel_getNumSentencePairs(void* swAlignModelHandle) + { + auto alignmentModel = static_cast(swAlignModelHandle); + return alignmentModel->numSentencePairs(); + } + void swAlignModel_setVariationalBayes(void* swAlignModelHandle, bool variationalBayes) { auto alignmentModel = static_cast(swAlignModelHandle); diff --git a/src/shared_library/thot.h b/src/shared_library/thot.h index 9683693d..1f377ec2 100644 --- a/src/shared_library/thot.h +++ b/src/shared_library/thot.h @@ -108,6 +108,8 @@ extern "C" THOT_API unsigned int swAlignModel_getMaxSentenceLength(void* swAlignModelHandle); + THOT_API unsigned int swAlignModel_getNumSentencePairs(void* swAlignModelHandle); + THOT_API void swAlignModel_setVariationalBayes(void* swAlignModelHandle, bool variationalBayes); THOT_API bool swAlignModel_getVariationalBayes(void* swAlignModelHandle); From f23c75469628bb1f30bf4778f32255c95d7e25ec Mon Sep 17 00:00:00 2001 From: Damien Daspit Date: Tue, 30 Jun 2026 17:42:49 -0400 Subject: [PATCH 2/2] Add SymmetrizedAlignmentModel --- src/CMakeLists.txt | 2 + src/python_module/module.cc | 25 ++++ src/sw_models/SymmetrizedAligner.cc | 22 ++- src/sw_models/SymmetrizedAligner.h | 8 + src/sw_models/SymmetrizedAlignmentModel.cc | 49 ++++++ src/sw_models/SymmetrizedAlignmentModel.h | 48 ++++++ tests/CMakeLists.txt | 1 + .../SymmetrizedAlignmentModelTest.cc | 141 ++++++++++++++++++ tests/test_module.py | 69 +++++++++ thot/alignment/__init__.pyi | 8 + 10 files changed, 365 insertions(+), 8 deletions(-) create mode 100644 src/sw_models/SymmetrizedAlignmentModel.cc create mode 100644 src/sw_models/SymmetrizedAlignmentModel.h create mode 100644 tests/sw_models/SymmetrizedAlignmentModelTest.cc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a8e79408..2343e1fa 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -358,6 +358,8 @@ add_library(thot_lib OBJECT sw_models/SwDefs.h sw_models/SymmetrizedAligner.cc sw_models/SymmetrizedAligner.h + sw_models/SymmetrizedAlignmentModel.cc + sw_models/SymmetrizedAlignmentModel.h ) target_include_directories(thot_lib PUBLIC diff --git a/src/python_module/module.cc b/src/python_module/module.cc index 20efc81f..268ee873 100644 --- a/src/python_module/module.cc +++ b/src/python_module/module.cc @@ -20,6 +20,7 @@ #include "sw_models/NormalSentenceLengthModel.h" #include "sw_models/SentenceLengthModel.h" #include "sw_models/SymmetrizedAligner.h" +#include "sw_models/SymmetrizedAlignmentModel.h" #include #include @@ -310,6 +311,30 @@ PYBIND11_MODULE(thot, m) py::arg("inverse_aligner")) .def_property("heuristic", &SymmetrizedAligner::getHeuristic, &SymmetrizedAligner::setHeuristic); + py::class_>( + alignment, "SymmetrizedAlignmentModel") + .def(py::init, std::shared_ptr>(), py::arg("direct_model"), + py::arg("inverse_model")) + .def_property_readonly("num_sentence_pairs", &SymmetrizedAlignmentModel::numSentencePairs) + .def( + "get_sentence_pair", + [](SymmetrizedAlignmentModel& model, unsigned int n) { + std::vector srcSentence, trgSentence; + Count c; + if (model.getSentencePair(n, srcSentence, trgSentence, c) == THOT_ERROR) + throw std::out_of_range("The sentence pair index is out of range."); + return std::make_tuple(srcSentence, trgSentence, (float)c); + }, + py::arg("n")) + .def( + "get_training_alignment", + [](SymmetrizedAlignmentModel& model, size_t n) { + WordAlignmentMatrix waMatrix; + LgProb logProb = model.getTrainingAlignment(n, waMatrix); + return std::make_tuple((double)logProb, std::move(waMatrix)); + }, + py::arg("n")); + py::enum_(alignment, "AlignmentModelType") .value("IBM1", AlignmentModelType::Ibm1) .value("IBM2", AlignmentModelType::Ibm2) diff --git a/src/sw_models/SymmetrizedAligner.cc b/src/sw_models/SymmetrizedAligner.cc index 2e1cbd6d..de4980e0 100644 --- a/src/sw_models/SymmetrizedAligner.cc +++ b/src/sw_models/SymmetrizedAligner.cc @@ -53,33 +53,39 @@ LgProb SymmetrizedAligner::getBestAlignment(const vector& srcSentence WordAlignmentMatrix invMatrix; LgProb invLogProb = inverseAligner->getBestAlignment(trgSentence, srcSentence, invMatrix); invMatrix.transpose(); + applyHeuristic(bestWaMatrix, invMatrix, heuristic); + return max(logProb, invLogProb); +} + +void SymmetrizedAligner::applyHeuristic(WordAlignmentMatrix& matrix, const WordAlignmentMatrix& invMatrix, + SymmetrizationHeuristic heuristic) +{ switch (heuristic) { case SymmetrizationHeuristic::Union: - bestWaMatrix |= invMatrix; + matrix |= invMatrix; break; case SymmetrizationHeuristic::Intersection: - bestWaMatrix &= invMatrix; + matrix &= invMatrix; break; case SymmetrizationHeuristic::Och: - bestWaMatrix.symmetr1(invMatrix); + matrix.symmetr1(invMatrix); break; case SymmetrizationHeuristic::Grow: - bestWaMatrix.grow(invMatrix); + matrix.grow(invMatrix); break; case SymmetrizationHeuristic::GrowDiag: - bestWaMatrix.growDiag(invMatrix); + matrix.growDiag(invMatrix); break; case SymmetrizationHeuristic::GrowDiagFinal: - bestWaMatrix.growDiagFinal(invMatrix); + matrix.growDiagFinal(invMatrix); break; case SymmetrizationHeuristic::GrowDiagFinalAnd: - bestWaMatrix.growDiagFinalAnd(invMatrix); + matrix.growDiagFinalAnd(invMatrix); break; case SymmetrizationHeuristic::None: break; } - return max(logProb, invLogProb); } WordIndex SymmetrizedAligner::stringToSrcWordIndex(string s) const diff --git a/src/sw_models/SymmetrizedAligner.h b/src/sw_models/SymmetrizedAligner.h index 5c96d535..5c891ab0 100644 --- a/src/sw_models/SymmetrizedAligner.h +++ b/src/sw_models/SymmetrizedAligner.h @@ -1,3 +1,5 @@ +#pragma once + #include "sw_models/Aligner.h" #include @@ -38,6 +40,12 @@ class SymmetrizedAligner : public virtual Aligner { } +protected: + // Combines 'matrix' (direct, src x trg) with 'invMatrix' (inverse, already + // transposed to src x trg) in place according to the given heuristic. + static void applyHeuristic(WordAlignmentMatrix& matrix, const WordAlignmentMatrix& invMatrix, + SymmetrizationHeuristic heuristic); + private: std::shared_ptr directAligner; std::shared_ptr inverseAligner; diff --git a/src/sw_models/SymmetrizedAlignmentModel.cc b/src/sw_models/SymmetrizedAlignmentModel.cc new file mode 100644 index 00000000..ed491cbc --- /dev/null +++ b/src/sw_models/SymmetrizedAlignmentModel.cc @@ -0,0 +1,49 @@ +#include "sw_models/SymmetrizedAlignmentModel.h" + +using namespace std; + +SymmetrizedAlignmentModel::SymmetrizedAlignmentModel(shared_ptr directModel, + shared_ptr inverseModel) + : SymmetrizedAligner{directModel, inverseModel}, directModel{directModel}, inverseModel{inverseModel} +{ +} + +unsigned int SymmetrizedAlignmentModel::numSentencePairs() +{ + return directModel->numSentencePairs(); +} + +int SymmetrizedAlignmentModel::getSentencePair(unsigned int n, vector& srcSentStr, vector& trgSentStr, + Count& c) +{ + return directModel->getSentencePair(n, srcSentStr, trgSentStr, c); +} + +LgProb SymmetrizedAlignmentModel::getTrainingAlignment(size_t n, WordAlignmentMatrix& bestWaMatrix) +{ + LgProb logProb = directModel->getTrainingAlignment(n, bestWaMatrix); + if (getHeuristic() == SymmetrizationHeuristic::None) + return logProb; + + WordAlignmentMatrix invMatrix; + LgProb invLogProb = inverseModel->getTrainingAlignment(n, invMatrix); + invMatrix.transpose(); + + // Skip the combine when the matrices are degenerate or their dimensions don't + // line up (e.g. an out-of-range n, or a pair filtered out of training in only + // one direction): the heuristic operations require matching dimensions. + if (bestWaMatrix.get_I() == 0 || bestWaMatrix.get_J() == 0 || invMatrix.get_I() != bestWaMatrix.get_I() || + invMatrix.get_J() != bestWaMatrix.get_J()) + return logProb; + + applyHeuristic(bestWaMatrix, invMatrix, getHeuristic()); + return max(logProb, invLogProb); +} + +LgProb SymmetrizedAlignmentModel::getTrainingAlignment(size_t n, vector& alignment) +{ + WordAlignmentMatrix waMatrix; + LgProb logProb = getTrainingAlignment(n, waMatrix); + waMatrix.getAligVec(alignment); + return logProb; +} diff --git a/src/sw_models/SymmetrizedAlignmentModel.h b/src/sw_models/SymmetrizedAlignmentModel.h new file mode 100644 index 00000000..203cf80f --- /dev/null +++ b/src/sw_models/SymmetrizedAlignmentModel.h @@ -0,0 +1,48 @@ +#pragma once + +#include "nlp_common/Count.h" +#include "sw_models/AlignmentModel.h" +#include "sw_models/SymmetrizedAligner.h" + +#include + +// A symmetrized aligner that also supports transductive alignment, i.e. +// getTrainingAlignment. It combines the per-training-pair alignments inferred by +// a direct model (trained on src->trg) and an inverse model (trained on the same +// pairs with trg/src swapped) using a symmetrization heuristic. +// +// Both models must be trained on the same sentence pairs in the same order (the +// inverse with source and target swapped) and with setEmitTrainingAlignments(true), +// so that index n lines up across the two corpora. +// +// Inherits the symmetrized getBestAlignment overloads and the 'heuristic' +// property from SymmetrizedAligner. +class SymmetrizedAlignmentModel : public SymmetrizedAligner +{ +public: + SymmetrizedAlignmentModel(std::shared_ptr directModel, + std::shared_ptr inverseModel); + + unsigned int numSentencePairs(); + + // Returns the source/target sentences (and count) for training pair n, in + // src->trg order (delegates to the direct model). + int getSentencePair(unsigned int n, std::vector& srcSentStr, std::vector& trgSentStr, + Count& c); + + // The symmetrized alignment for training pair n, mirroring getBestAlignment: + // combines the direct and inverse models' training alignments under the current + // heuristic. Fills 'alignment' (per target token, 1-based source position, + // 0 = NULL) and returns the direction-max log-probability. + LgProb getTrainingAlignment(size_t n, std::vector& alignment); + // As above, but fills a WordAlignmentMatrix instead of a per-target vector. + LgProb getTrainingAlignment(size_t n, WordAlignmentMatrix& bestWaMatrix); + + virtual ~SymmetrizedAlignmentModel() + { + } + +private: + std::shared_ptr directModel; + std::shared_ptr inverseModel; +}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a1d94fe7..4c00c2ce 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -13,6 +13,7 @@ add_executable(thot_test sw_models/IncrHmmAlignmentModelTest.cc sw_models/LexTableTest.h sw_models/MemoryLexTableTest.cc + sw_models/SymmetrizedAlignmentModelTest.cc sw_models/TestUtils.cc sw_models/TestUtils.h ) diff --git a/tests/sw_models/SymmetrizedAlignmentModelTest.cc b/tests/sw_models/SymmetrizedAlignmentModelTest.cc new file mode 100644 index 00000000..d87226db --- /dev/null +++ b/tests/sw_models/SymmetrizedAlignmentModelTest.cc @@ -0,0 +1,141 @@ +#include "sw_models/SymmetrizedAlignmentModel.h" + +#include "TestUtils.h" +#include "nlp_common/ErrorDefs.h" +#include "nlp_common/MathDefs.h" +#include "sw_models/FastAlignModel.h" + +#include + +#include + +using namespace std; + +namespace +{ +// Builds a direct model (trained on the test corpus) and an inverse model trained +// on the same pairs with source/target swapped, both emitting training +// alignments, then wraps them in a SymmetrizedAlignmentModel. +shared_ptr buildSymmetrizedModel(shared_ptr& direct, + shared_ptr& inverse) +{ + direct = make_shared(); + direct->setEmitTrainingAlignments(true); + addTrainingData(*direct); + train(*direct, 2); + + inverse = make_shared(); + inverse->setEmitTrainingAlignments(true); + for (unsigned int n = 0; n < direct->numSentencePairs(); ++n) + { + vector src, trg; + Count c; + direct->getSentencePair(n, src, trg, c); + inverse->addSentencePair(trg, src, c); // swapped + } + train(*inverse, 2); + + return make_shared(direct, inverse); +} +} // namespace + +TEST(SymmetrizedAlignmentModelTest, sentencePairAccessors) +{ + shared_ptr direct, inverse; + shared_ptr model = buildSymmetrizedModel(direct, inverse); + + EXPECT_EQ(model->numSentencePairs(), direct->numSentencePairs()); + + // get_sentence_pair returns the direct model's pairs, in src->trg order (not + // the swapped inverse order). + for (unsigned int n = 0; n < model->numSentencePairs(); ++n) + { + vector src, trg, expectedSrc, expectedTrg; + Count c, expectedC; + direct->getSentencePair(n, expectedSrc, expectedTrg, expectedC); + model->getSentencePair(n, src, trg, c); + EXPECT_EQ(src, expectedSrc); + EXPECT_EQ(trg, expectedTrg); + } +} + +TEST(SymmetrizedAlignmentModelTest, heuristicNoneReturnsDirect) +{ + shared_ptr direct, inverse; + shared_ptr model = buildSymmetrizedModel(direct, inverse); + model->setHeuristic(SymmetrizationHeuristic::None); + + for (unsigned int n = 0; n < model->numSentencePairs(); ++n) + { + vector expected; + LgProb expectedProb = direct->getTrainingAlignment(n, expected); + + vector alig; + LgProb prob = model->getTrainingAlignment(n, alig); + EXPECT_EQ(alig, expected); + EXPECT_NEAR((double)prob, (double)expectedProb, EPSILON); + } +} + +TEST(SymmetrizedAlignmentModelTest, combinesDirectAndInverse) +{ + shared_ptr direct, inverse; + shared_ptr model = buildSymmetrizedModel(direct, inverse); + + for (SymmetrizationHeuristic h : + {SymmetrizationHeuristic::Union, SymmetrizationHeuristic::Intersection, SymmetrizationHeuristic::Och, + SymmetrizationHeuristic::GrowDiagFinalAnd}) + { + model->setHeuristic(h); + for (unsigned int n = 0; n < model->numSentencePairs(); ++n) + { + // Reconstruct the expected combination directly from the two underlying + // training alignments: direct (src x trg) and inverse transposed (src x trg). + WordAlignmentMatrix expected; + LgProb directProb = direct->getTrainingAlignment(n, expected); + WordAlignmentMatrix invMatrix; + LgProb invProb = inverse->getTrainingAlignment(n, invMatrix); + invMatrix.transpose(); + switch (h) + { + case SymmetrizationHeuristic::Union: + expected |= invMatrix; + break; + case SymmetrizationHeuristic::Intersection: + expected &= invMatrix; + break; + case SymmetrizationHeuristic::Och: + expected.symmetr1(invMatrix); + break; + case SymmetrizationHeuristic::GrowDiagFinalAnd: + expected.growDiagFinalAnd(invMatrix); + break; + default: + break; + } + + WordAlignmentMatrix actual; + LgProb prob = model->getTrainingAlignment(n, actual); + EXPECT_TRUE(actual == expected); + EXPECT_NEAR((double)prob, (double)max(directProb, invProb), EPSILON); + + // The per-target-vector overload agrees with the matrix overload. + vector aligVec, expectedVec; + model->getTrainingAlignment(n, aligVec); + expected.getAligVec(expectedVec); + EXPECT_EQ(aligVec, expectedVec); + } + } +} + +TEST(SymmetrizedAlignmentModelTest, outOfRangeIndex) +{ + shared_ptr direct, inverse; + shared_ptr model = buildSymmetrizedModel(direct, inverse); + model->setHeuristic(SymmetrizationHeuristic::GrowDiagFinalAnd); + + vector oob{1, 2, 3}; + LgProb prob = model->getTrainingAlignment(1000, oob); + EXPECT_TRUE(oob.empty()); + EXPECT_EQ((double)prob, (double)SMALL_LG_NUM); +} diff --git a/tests/test_module.py b/tests/test_module.py index 344bc1a9..433f0993 100644 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -10,6 +10,7 @@ NormalSentenceLengthModel, SymmetrizationHeuristic, SymmetrizedAligner, + SymmetrizedAlignmentModel, ) from thot.translation import SmtModel, SmtDecoder @@ -75,6 +76,74 @@ def test_alignment_model() -> None: assert np.array_equal(alignments[3][1].to_numpy(), _create_matrix(0, [])) +def test_symmetrized_alignment_model() -> None: + train_src_sentences = [ + "isthay isyay ayay esttay-N .", + "ouyay ouldshay esttay-V oftenyay .", + "isyay isthay orkingway ?", + "isthay ouldshay orkway-V .", + "ityay isyay orkingway .", + "orkway-N ancay ebay ardhay !", + "ayay esttay-N ancay ebay ardhay .", + "isthay isyay ayay ordway !", + ] + train_trg_sentences = [ + "this is a test N .", + "you should test V often .", + "is this working ?", + "this should work V .", + "it is working .", + "work N can be hard !", + "a test N can be hard .", + "this is a word !", + ] + + # Direct model (src -> trg) and inverse model (trained on the same pairs with + # src/trg swapped), both emitting training alignments. + direct_model = _train_aligned_hmm(train_src_sentences, train_trg_sentences) + inverse_model = _train_aligned_hmm(train_trg_sentences, train_src_sentences) + + model = SymmetrizedAlignmentModel(direct_model, inverse_model) + + # The corpus accessors delegate to the direct model: pairs come back in + # src -> trg order, not swapped. + assert model.num_sentence_pairs == direct_model.num_sentence_pairs + for n in range(model.num_sentence_pairs): + src, trg, _ = model.get_sentence_pair(n) + expected_src, expected_trg, _ = direct_model.get_sentence_pair(n) + assert list(src) == list(expected_src) + assert list(trg) == list(expected_trg) + + # heuristic NONE returns the direct model's training alignment unchanged. + model.heuristic = SymmetrizationHeuristic.NONE + for n in range(model.num_sentence_pairs): + _, matrix = model.get_training_alignment(n) + _, expected = direct_model.get_training_alignment(n) + assert np.array_equal(matrix.to_numpy(), expected.to_numpy()) + + # UNION combines the direct alignment with the transposed inverse alignment. + model.heuristic = SymmetrizationHeuristic.UNION + for n in range(model.num_sentence_pairs): + _, matrix = model.get_training_alignment(n) + _, direct_matrix = direct_model.get_training_alignment(n) + _, inverse_matrix = inverse_model.get_training_alignment(n) + inverse_matrix.transpose() + expected = direct_matrix.union(inverse_matrix) + assert np.array_equal(matrix.to_numpy(), expected.to_numpy()) + + +def _train_aligned_hmm(src_sentences: List[str], trg_sentences: List[str]) -> HmmAlignmentModel: + ibm1_model = Ibm1AlignmentModel() + _add_sentence_pairs(ibm1_model, src_sentences, trg_sentences) + _train_model(ibm1_model, 2) + + hmm_model = HmmAlignmentModel(ibm1_model) + hmm_model.hmm_p0 = 0.1 + hmm_model.emit_training_alignments = True + _train_model(hmm_model, 2) + return hmm_model + + def test_sentence_length_model() -> None: model = NormalSentenceLengthModel() model.train_sentence_pair(10, 20) diff --git a/thot/alignment/__init__.pyi b/thot/alignment/__init__.pyi index a09e0a9d..8f475865 100644 --- a/thot/alignment/__init__.pyi +++ b/thot/alignment/__init__.pyi @@ -42,6 +42,13 @@ class SymmetrizedAligner(Aligner): @heuristic.setter def heuristic(self, value: SymmetrizationHeuristic) -> None: ... +class SymmetrizedAlignmentModel(SymmetrizedAligner): + def __init__(self, direct_model: AlignmentModel, inverse_model: AlignmentModel) -> None: ... + @property + def num_sentence_pairs(self) -> int: ... + def get_sentence_pair(self, n: int) -> Tuple[Sequence[str], Sequence[str], float]: ... + def get_training_alignment(self, n: int) -> Tuple[float, WordAlignmentMatrix]: ... + class AlignmentModelType(Enum): IBM1 = ... IBM2 = ... @@ -285,4 +292,5 @@ __all__ = [ "SentenceLengthModel", "SymmetrizationHeuristic", "SymmetrizedAligner", + "SymmetrizedAlignmentModel", ]