From 32d24d8d38ed3e199899740e8ae693dd6bca5d05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Wed, 2 Jul 2025 21:07:14 +0200 Subject: [PATCH 1/4] Start adding binding for new IFileTree glob and walk. --- src/mobase/pybind11_all.h | 1 + src/mobase/wrappers/pyfiletree.cpp | 24 ++++++++++--- src/pybind11-utils/CMakeLists.txt | 1 + .../include/pybind11_utils/generator.h | 36 +++++++++++++++++++ 4 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 src/pybind11-utils/include/pybind11_utils/generator.h diff --git a/src/mobase/pybind11_all.h b/src/mobase/pybind11_all.h index 9a76ad2..a293fb8 100644 --- a/src/mobase/pybind11_all.h +++ b/src/mobase/pybind11_all.h @@ -13,6 +13,7 @@ #include "pybind11_qt/pybind11_qt.h" #include "pybind11_utils/functional.h" +#include "pybind11_utils/generator.h" #include "pybind11_utils/shared_cpp_owner.h" #include "pybind11_utils/smart_variant_wrapper.h" diff --git a/src/mobase/wrappers/pyfiletree.cpp b/src/mobase/wrappers/pyfiletree.cpp index 4657b8a..e3bb93b 100644 --- a/src/mobase/wrappers/pyfiletree.cpp +++ b/src/mobase/wrappers/pyfiletree.cpp @@ -84,6 +84,9 @@ namespace mo2::python { void add_ifiletree_bindings(pybind11::module_& m) { + // register generator for walk and glob + register_generator_type>(m); + // FileTreeEntry class: auto fileTreeEntryClass = py::class_>(m, @@ -164,6 +167,11 @@ namespace mo2::python { .value("SKIP", IFileTree::WalkReturn::SKIP) .export_values(); + py::enum_(iFileTreeClass, "GlobPatternType") + .value("GLOB", IFileTree::GlobPatternType::GLOB) + .value("REGEX", IFileTree::GlobPatternType::REGEX) + .export_values(); + // Non-mutable operations: iFileTreeClass.def("exists", py::overload_cast( @@ -175,10 +183,18 @@ namespace mo2::python { iFileTreeClass.def("pathTo", &IFileTree::pathTo, py::arg("entry"), py::arg("sep") = "\\"); - // Note: walk() would probably be better as a generator in python, but - // it is likely impossible to construct from the C++ walk() method. - iFileTreeClass.def("walk", &IFileTree::walk, py::arg("callback"), - py::arg("sep") = "\\"); + iFileTreeClass.def( + "walk", + py::overload_cast< + std::function)>, + QString>(&IFileTree::walk, py::const_), + py::arg("callback"), py::arg("sep") = "\\"); + + iFileTreeClass.def("walk", py::overload_cast<>(&IFileTree::walk, py::const_)); + + iFileTreeClass.def("glob", &IFileTree::glob, py::arg("pattern"), + py::arg("type") = IFileTree::GlobPatternType::GLOB); // Kind-of-static operations: iFileTreeClass.def("createOrphanTree", &IFileTree::createOrphanTree, diff --git a/src/pybind11-utils/CMakeLists.txt b/src/pybind11-utils/CMakeLists.txt index 372a9b7..77895b6 100644 --- a/src/pybind11-utils/CMakeLists.txt +++ b/src/pybind11-utils/CMakeLists.txt @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.16) add_library(pybind11-utils STATIC ./include/pybind11_utils/functional.h + ./include/pybind11_utils/generator.h ./include/pybind11_utils/shared_cpp_owner.h ./include/pybind11_utils/smart_variant_wrapper.h ./include/pybind11_utils/smart_variant.h diff --git a/src/pybind11-utils/include/pybind11_utils/generator.h b/src/pybind11-utils/include/pybind11_utils/generator.h new file mode 100644 index 0000000..9ab3827 --- /dev/null +++ b/src/pybind11-utils/include/pybind11_utils/generator.h @@ -0,0 +1,36 @@ +#ifndef PYTHON_PYBIND11_GENERATOR_H +#define PYTHON_PYBIND11_GENERATOR_H + +#include + +#include + +namespace mo2::python { + + // create a Python generator from a C++ generator + // + template + auto register_generator_type(pybind11::handle scope, + const char* name = "_mo2_generator") + { + namespace py = pybind11; + + return py::class_>(scope, name, py::module_local()) + .def("__iter__", + [](std::generator& gen) -> std::generator& { + return gen; + }) + .def("__next__", [](std::generator& gen) { + auto it = gen.begin(); + if (it != gen.end()) { + return *it; + } + else { + throw py::stop_iteration(); + } + }); + } + +} // namespace mo2::python + +#endif From 32eee30b6b4dd14dbd376002d0e10f3a0ffd2899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sat, 5 Jul 2025 11:41:53 +0200 Subject: [PATCH 2/4] Fix issue with generator bindings. --- src/mobase/wrappers/pyfiletree.cpp | 19 ++++--- .../include/pybind11_utils/generator.h | 50 +++++++++++++------ tests/python/CMakeLists.txt | 2 +- 3 files changed, 46 insertions(+), 25 deletions(-) diff --git a/src/mobase/wrappers/pyfiletree.cpp b/src/mobase/wrappers/pyfiletree.cpp index e3bb93b..b9a477a 100644 --- a/src/mobase/wrappers/pyfiletree.cpp +++ b/src/mobase/wrappers/pyfiletree.cpp @@ -49,7 +49,7 @@ namespace mo2::detail { return std::make_shared(parent, name, m_Callback); } - bool doPopulate(std::shared_ptr parent, + bool doPopulate([[maybe_unused]] std::shared_ptr parent, std::vector>&) const override { return true; @@ -83,10 +83,6 @@ namespace mo2::python { void add_ifiletree_bindings(pybind11::module_& m) { - - // register generator for walk and glob - register_generator_type>(m); - // FileTreeEntry class: auto fileTreeEntryClass = py::class_>(m, @@ -191,10 +187,17 @@ namespace mo2::python { QString>(&IFileTree::walk, py::const_), py::arg("callback"), py::arg("sep") = "\\"); - iFileTreeClass.def("walk", py::overload_cast<>(&IFileTree::walk, py::const_)); + iFileTreeClass.def("walk", [](IFileTree const* tree) { + return make_generator(tree->walk()); + }); - iFileTreeClass.def("glob", &IFileTree::glob, py::arg("pattern"), - py::arg("type") = IFileTree::GlobPatternType::GLOB); + iFileTreeClass.def( + "glob", // &IFileTree::glob, + [](IFileTree const* tree, QString pattern, + IFileTree::GlobPatternType patternType) { + return make_generator(tree->glob(pattern, patternType)); + }, + py::arg("pattern"), py::arg("type") = IFileTree::GlobPatternType::GLOB); // Kind-of-static operations: iFileTreeClass.def("createOrphanTree", &IFileTree::createOrphanTree, diff --git a/src/pybind11-utils/include/pybind11_utils/generator.h b/src/pybind11-utils/include/pybind11_utils/generator.h index 9ab3827..635b73b 100644 --- a/src/pybind11-utils/include/pybind11_utils/generator.h +++ b/src/pybind11-utils/include/pybind11_utils/generator.h @@ -7,28 +7,46 @@ namespace mo2::python { + // the code here is mostly taken from pybind11 itself, and relies on some pybind11 + // internals so might be subject to change when upgrading pybind11 versions + + namespace detail { + template + struct generator_state { + std::generator g; + decltype(g.begin()) it; + + generator_state(std::generator gen) : g(std::move(gen)), it(g.begin()) {} + }; + } // namespace detail + // create a Python generator from a C++ generator // template - auto register_generator_type(pybind11::handle scope, - const char* name = "_mo2_generator") + auto make_generator(std::generator g) { + using state = detail::generator_state; + namespace py = pybind11; + if (!py::detail::get_type_info(typeid(state), false)) { + py::class_(py::handle(), "iterator", pybind11::module_local()) + .def("__iter__", + [](state& s) -> state& { + return s; + }) + .def("__next__", [](state& s) { + if (s.it != s.g.end()) { + const auto v = *s.it; + s.it++; + return v; + } + else { + throw py::stop_iteration(); + } + }); + } - return py::class_>(scope, name, py::module_local()) - .def("__iter__", - [](std::generator& gen) -> std::generator& { - return gen; - }) - .def("__next__", [](std::generator& gen) { - auto it = gen.begin(); - if (it != gen.end()) { - return *it; - } - else { - throw py::stop_iteration(); - } - }); + return py::cast(state{std::move(g)}); } } // namespace mo2::python diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 6138c37..f473373 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -53,7 +53,7 @@ foreach (test_file ${test_files}) pybind11_add_module(${target} EXCLUDE_FROM_ALL THIN_LTO ${test_file}) set_target_properties(${target} PROPERTIES - CXX_STANDARD 20 + CXX_STANDARD 23 OUTPUT_NAME ${pymodule} FOLDER tests/python LIBRARY_OUTPUT_DIRECTORY "${PYLIB_DIR}/mobase_tests") From a698df0edc3b819c0bde225e6419b1ebc7f18467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sat, 5 Jul 2025 15:25:08 +0200 Subject: [PATCH 3/4] Fix issues with Version binding. --- src/mobase/wrappers/basic_classes.cpp | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/mobase/wrappers/basic_classes.cpp b/src/mobase/wrappers/basic_classes.cpp index ca5e8eb..5168cb9 100644 --- a/src/mobase/wrappers/basic_classes.cpp +++ b/src/mobase/wrappers/basic_classes.cpp @@ -54,7 +54,24 @@ namespace mo2::python { .value("NO_METADATA", Version::FormatMode::NoMetadata) .value("CONDENSED", static_cast(Version::FormatCondensed.toInt())) - .export_values(); + .export_values() + .def("__xor__", + py::overload_cast( + &operator^)) + .def("__and__", + py::overload_cast( + &operator&)) + .def("__or__", py::overload_cast( + &operator|)) + .def("__rxor__", + py::overload_cast( + &operator^)) + .def("__rand__", + py::overload_cast( + &operator&)) + .def("__ror__", + py::overload_cast( + &operator|)); pyVersion .def_static("parse", &Version::parse, "value"_a, @@ -86,8 +103,11 @@ namespace mo2::python { .def_property_readonly("subpatch", &Version::subpatch) .def_property_readonly("prereleases", &Version::preReleases) .def_property_readonly("build_metadata", &Version::buildMetadata) - .def("string", &Version::string, "mode"_a = Version::FormatCondensed) - .def("__str__", &Version::string) + .def("string", &Version::string, "mode"_a = Version::FormatModes{}) + .def("__str__", + [](Version const& version) { + return version.string(Version::FormatCondensed); + }) .def(py::self < py::self) .def(py::self > py::self) .def(py::self <= py::self) From b7f9a377d5173f0514e6527743b3e4b5c07ce6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=C3=ABl=20Capelle?= Date: Sat, 5 Jul 2025 15:38:54 +0200 Subject: [PATCH 4/4] Add a few tests to check binding for IFileTree glob and walk. --- tests/python/test_filetree.py | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/python/test_filetree.py b/tests/python/test_filetree.py index d062591..0322faa 100644 --- a/tests/python/test_filetree.py +++ b/tests/python/test_filetree.py @@ -1,3 +1,5 @@ +from typing import TypeAlias, cast + import mobase import mobase_tests.filetree as m @@ -34,3 +36,51 @@ def test_filetype(): assert m.is_file(FT.FILE_OR_DIRECTORY & ~FT.DIRECTORY) assert not m.is_directory(FT.FILE_OR_DIRECTORY & ~FT.DIRECTORY) + + +_tree_values: TypeAlias = list["str | tuple[str, _tree_values]"] + + +def make_tree( + values: _tree_values, root: mobase.IFileTree | None = None +) -> mobase.IFileTree: + if root is None: + root = cast(mobase.IFileTree, mobase.private.makeTree()) # type: ignore + + for value in values: + if isinstance(value, str): + root.addFile(value) + else: + sub_tree = root.addDirectory(value[0]) + make_tree(value[1], sub_tree) + + return root + + +def test_walk(): + tree = make_tree( + [("a", []), ("b", ["u", "v"]), "c.x", "d.y", ("e", [("q", ["c.t", ("p", [])])])] + ) + + assert {"a", "b", "b/u", "b/v", "c.x", "d.y", "e", "e/q", "e/q/c.t", "e/q/p"} == { + e.path("/") for e in tree.walk() + } + + entries: list[str] = [] + for e in tree.walk(): + if e.name() == "e": + break + entries.append(e.path("/")) + assert {"a", "b", "b/u", "b/v"} == set(entries) + + +def test_glob(): + tree = make_tree( + [("a", []), ("b", ["u", "v"]), "c.x", "d.y", ("e", [("q", ["c.t", ("p", [])])])] + ) + + assert {"a", "b", "b/u", "b/v", "c.x", "d.y", "e", "e/q", "e/q/c.t", "e/q/p"} == { + e.path("/") for e in tree.glob("**/*") + } + + assert {"d.y"} == {e.path("/") for e in tree.glob("**/*.y")}