From 7eb62dd2c2c33b6068043a4b850913891fda4d43 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Thu, 17 Oct 2024 16:43:27 -0700 Subject: [PATCH 01/24] Migrate to pyproject.toml, and convert the build to use py-build-cmake --- .gitmodules | 2 +- {amazon => src-python/amazon}/__init__.py | 0 {amazon => src-python/amazon}/ion/__init__.py | 0 {amazon => src-python/amazon}/ion/core.py | 0 {amazon => src-python/amazon}/ion/equivalence.py | 0 {amazon => src-python/amazon}/ion/exceptions.py | 0 {amazon => src-python/amazon}/ion/json_encoder.py | 0 {amazon => src-python/amazon}/ion/reader.py | 0 {amazon => src-python/amazon}/ion/reader_binary.py | 0 {amazon => src-python/amazon}/ion/reader_managed.py | 0 {amazon => src-python/amazon}/ion/reader_text.py | 0 {amazon => src-python/amazon}/ion/simple_types.py | 0 {amazon => src-python/amazon}/ion/simpleion.py | 0 {amazon => src-python/amazon}/ion/sliceable_buffer.py | 0 {amazon => src-python/amazon}/ion/symbols.py | 0 {amazon => src-python/amazon}/ion/util.py | 0 {amazon => src-python/amazon}/ion/writer.py | 0 {amazon => src-python/amazon}/ion/writer_binary.py | 0 {amazon => src-python/amazon}/ion/writer_binary_raw.py | 0 {amazon => src-python/amazon}/ion/writer_binary_raw_fields.py | 0 {amazon => src-python/amazon}/ion/writer_buffer.py | 0 {amazon => src-python/amazon}/ion/writer_text.py | 0 {amazon => src-python/amazon}/ionbenchmark/Format.py | 0 {amazon => src-python/amazon}/ionbenchmark/__init__.py | 0 {amazon => src-python/amazon}/ionbenchmark/benchmark_runner.py | 0 {amazon => src-python/amazon}/ionbenchmark/benchmark_spec.py | 0 {amazon => src-python/amazon}/ionbenchmark/cbor_load_dump.py | 0 {amazon => src-python/amazon}/ionbenchmark/ion_benchmark_cli.py | 0 {amazon => src-python/amazon}/ionbenchmark/ion_load_dump.py | 0 {amazon => src-python/amazon}/ionbenchmark/json_load_dump.py | 0 {amazon => src-python/amazon}/ionbenchmark/proto.py | 0 {amazon => src-python/amazon}/ionbenchmark/proto_tools.py | 0 {amazon => src-python/amazon}/ionbenchmark/report.py | 0 {amazon => src-python/amazon}/ionbenchmark/sample_dist.py | 0 .../amazon}/ionbenchmark/self_describing_proto.proto | 0 .../amazon}/ionbenchmark/self_describing_proto.py | 0 .../amazon}/ionbenchmark/self_describing_proto_pb2.py | 0 {amazon/ion => src}/_ioncmodule.h | 0 ion-c => src/ion-c | 0 {amazon/ion => src}/ioncmodule.c | 0 40 files changed, 1 insertion(+), 1 deletion(-) rename {amazon => src-python/amazon}/__init__.py (100%) rename {amazon => src-python/amazon}/ion/__init__.py (100%) rename {amazon => src-python/amazon}/ion/core.py (100%) rename {amazon => src-python/amazon}/ion/equivalence.py (100%) rename {amazon => src-python/amazon}/ion/exceptions.py (100%) rename {amazon => src-python/amazon}/ion/json_encoder.py (100%) rename {amazon => src-python/amazon}/ion/reader.py (100%) rename {amazon => src-python/amazon}/ion/reader_binary.py (100%) rename {amazon => src-python/amazon}/ion/reader_managed.py (100%) rename {amazon => src-python/amazon}/ion/reader_text.py (100%) rename {amazon => src-python/amazon}/ion/simple_types.py (100%) rename {amazon => src-python/amazon}/ion/simpleion.py (100%) rename {amazon => src-python/amazon}/ion/sliceable_buffer.py (100%) rename {amazon => src-python/amazon}/ion/symbols.py (100%) rename {amazon => src-python/amazon}/ion/util.py (100%) rename {amazon => src-python/amazon}/ion/writer.py (100%) rename {amazon => src-python/amazon}/ion/writer_binary.py (100%) rename {amazon => src-python/amazon}/ion/writer_binary_raw.py (100%) rename {amazon => src-python/amazon}/ion/writer_binary_raw_fields.py (100%) rename {amazon => src-python/amazon}/ion/writer_buffer.py (100%) rename {amazon => src-python/amazon}/ion/writer_text.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/Format.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/__init__.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/benchmark_runner.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/benchmark_spec.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/cbor_load_dump.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/ion_benchmark_cli.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/ion_load_dump.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/json_load_dump.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/proto.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/proto_tools.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/report.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/sample_dist.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/self_describing_proto.proto (100%) rename {amazon => src-python/amazon}/ionbenchmark/self_describing_proto.py (100%) rename {amazon => src-python/amazon}/ionbenchmark/self_describing_proto_pb2.py (100%) rename {amazon/ion => src}/_ioncmodule.h (100%) rename ion-c => src/ion-c (100%) rename {amazon/ion => src}/ioncmodule.c (100%) diff --git a/.gitmodules b/.gitmodules index 9a36d8eb9..ddbc09763 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,6 +3,6 @@ url = https://github.com/amazon-ion/ion-tests branch = master [submodule "ion-c"] - path = ion-c + path = src/ion-c url = https://github.com/amazon-ion/ion-c branch = master diff --git a/amazon/__init__.py b/src-python/amazon/__init__.py similarity index 100% rename from amazon/__init__.py rename to src-python/amazon/__init__.py diff --git a/amazon/ion/__init__.py b/src-python/amazon/ion/__init__.py similarity index 100% rename from amazon/ion/__init__.py rename to src-python/amazon/ion/__init__.py diff --git a/amazon/ion/core.py b/src-python/amazon/ion/core.py similarity index 100% rename from amazon/ion/core.py rename to src-python/amazon/ion/core.py diff --git a/amazon/ion/equivalence.py b/src-python/amazon/ion/equivalence.py similarity index 100% rename from amazon/ion/equivalence.py rename to src-python/amazon/ion/equivalence.py diff --git a/amazon/ion/exceptions.py b/src-python/amazon/ion/exceptions.py similarity index 100% rename from amazon/ion/exceptions.py rename to src-python/amazon/ion/exceptions.py diff --git a/amazon/ion/json_encoder.py b/src-python/amazon/ion/json_encoder.py similarity index 100% rename from amazon/ion/json_encoder.py rename to src-python/amazon/ion/json_encoder.py diff --git a/amazon/ion/reader.py b/src-python/amazon/ion/reader.py similarity index 100% rename from amazon/ion/reader.py rename to src-python/amazon/ion/reader.py diff --git a/amazon/ion/reader_binary.py b/src-python/amazon/ion/reader_binary.py similarity index 100% rename from amazon/ion/reader_binary.py rename to src-python/amazon/ion/reader_binary.py diff --git a/amazon/ion/reader_managed.py b/src-python/amazon/ion/reader_managed.py similarity index 100% rename from amazon/ion/reader_managed.py rename to src-python/amazon/ion/reader_managed.py diff --git a/amazon/ion/reader_text.py b/src-python/amazon/ion/reader_text.py similarity index 100% rename from amazon/ion/reader_text.py rename to src-python/amazon/ion/reader_text.py diff --git a/amazon/ion/simple_types.py b/src-python/amazon/ion/simple_types.py similarity index 100% rename from amazon/ion/simple_types.py rename to src-python/amazon/ion/simple_types.py diff --git a/amazon/ion/simpleion.py b/src-python/amazon/ion/simpleion.py similarity index 100% rename from amazon/ion/simpleion.py rename to src-python/amazon/ion/simpleion.py diff --git a/amazon/ion/sliceable_buffer.py b/src-python/amazon/ion/sliceable_buffer.py similarity index 100% rename from amazon/ion/sliceable_buffer.py rename to src-python/amazon/ion/sliceable_buffer.py diff --git a/amazon/ion/symbols.py b/src-python/amazon/ion/symbols.py similarity index 100% rename from amazon/ion/symbols.py rename to src-python/amazon/ion/symbols.py diff --git a/amazon/ion/util.py b/src-python/amazon/ion/util.py similarity index 100% rename from amazon/ion/util.py rename to src-python/amazon/ion/util.py diff --git a/amazon/ion/writer.py b/src-python/amazon/ion/writer.py similarity index 100% rename from amazon/ion/writer.py rename to src-python/amazon/ion/writer.py diff --git a/amazon/ion/writer_binary.py b/src-python/amazon/ion/writer_binary.py similarity index 100% rename from amazon/ion/writer_binary.py rename to src-python/amazon/ion/writer_binary.py diff --git a/amazon/ion/writer_binary_raw.py b/src-python/amazon/ion/writer_binary_raw.py similarity index 100% rename from amazon/ion/writer_binary_raw.py rename to src-python/amazon/ion/writer_binary_raw.py diff --git a/amazon/ion/writer_binary_raw_fields.py b/src-python/amazon/ion/writer_binary_raw_fields.py similarity index 100% rename from amazon/ion/writer_binary_raw_fields.py rename to src-python/amazon/ion/writer_binary_raw_fields.py diff --git a/amazon/ion/writer_buffer.py b/src-python/amazon/ion/writer_buffer.py similarity index 100% rename from amazon/ion/writer_buffer.py rename to src-python/amazon/ion/writer_buffer.py diff --git a/amazon/ion/writer_text.py b/src-python/amazon/ion/writer_text.py similarity index 100% rename from amazon/ion/writer_text.py rename to src-python/amazon/ion/writer_text.py diff --git a/amazon/ionbenchmark/Format.py b/src-python/amazon/ionbenchmark/Format.py similarity index 100% rename from amazon/ionbenchmark/Format.py rename to src-python/amazon/ionbenchmark/Format.py diff --git a/amazon/ionbenchmark/__init__.py b/src-python/amazon/ionbenchmark/__init__.py similarity index 100% rename from amazon/ionbenchmark/__init__.py rename to src-python/amazon/ionbenchmark/__init__.py diff --git a/amazon/ionbenchmark/benchmark_runner.py b/src-python/amazon/ionbenchmark/benchmark_runner.py similarity index 100% rename from amazon/ionbenchmark/benchmark_runner.py rename to src-python/amazon/ionbenchmark/benchmark_runner.py diff --git a/amazon/ionbenchmark/benchmark_spec.py b/src-python/amazon/ionbenchmark/benchmark_spec.py similarity index 100% rename from amazon/ionbenchmark/benchmark_spec.py rename to src-python/amazon/ionbenchmark/benchmark_spec.py diff --git a/amazon/ionbenchmark/cbor_load_dump.py b/src-python/amazon/ionbenchmark/cbor_load_dump.py similarity index 100% rename from amazon/ionbenchmark/cbor_load_dump.py rename to src-python/amazon/ionbenchmark/cbor_load_dump.py diff --git a/amazon/ionbenchmark/ion_benchmark_cli.py b/src-python/amazon/ionbenchmark/ion_benchmark_cli.py similarity index 100% rename from amazon/ionbenchmark/ion_benchmark_cli.py rename to src-python/amazon/ionbenchmark/ion_benchmark_cli.py diff --git a/amazon/ionbenchmark/ion_load_dump.py b/src-python/amazon/ionbenchmark/ion_load_dump.py similarity index 100% rename from amazon/ionbenchmark/ion_load_dump.py rename to src-python/amazon/ionbenchmark/ion_load_dump.py diff --git a/amazon/ionbenchmark/json_load_dump.py b/src-python/amazon/ionbenchmark/json_load_dump.py similarity index 100% rename from amazon/ionbenchmark/json_load_dump.py rename to src-python/amazon/ionbenchmark/json_load_dump.py diff --git a/amazon/ionbenchmark/proto.py b/src-python/amazon/ionbenchmark/proto.py similarity index 100% rename from amazon/ionbenchmark/proto.py rename to src-python/amazon/ionbenchmark/proto.py diff --git a/amazon/ionbenchmark/proto_tools.py b/src-python/amazon/ionbenchmark/proto_tools.py similarity index 100% rename from amazon/ionbenchmark/proto_tools.py rename to src-python/amazon/ionbenchmark/proto_tools.py diff --git a/amazon/ionbenchmark/report.py b/src-python/amazon/ionbenchmark/report.py similarity index 100% rename from amazon/ionbenchmark/report.py rename to src-python/amazon/ionbenchmark/report.py diff --git a/amazon/ionbenchmark/sample_dist.py b/src-python/amazon/ionbenchmark/sample_dist.py similarity index 100% rename from amazon/ionbenchmark/sample_dist.py rename to src-python/amazon/ionbenchmark/sample_dist.py diff --git a/amazon/ionbenchmark/self_describing_proto.proto b/src-python/amazon/ionbenchmark/self_describing_proto.proto similarity index 100% rename from amazon/ionbenchmark/self_describing_proto.proto rename to src-python/amazon/ionbenchmark/self_describing_proto.proto diff --git a/amazon/ionbenchmark/self_describing_proto.py b/src-python/amazon/ionbenchmark/self_describing_proto.py similarity index 100% rename from amazon/ionbenchmark/self_describing_proto.py rename to src-python/amazon/ionbenchmark/self_describing_proto.py diff --git a/amazon/ionbenchmark/self_describing_proto_pb2.py b/src-python/amazon/ionbenchmark/self_describing_proto_pb2.py similarity index 100% rename from amazon/ionbenchmark/self_describing_proto_pb2.py rename to src-python/amazon/ionbenchmark/self_describing_proto_pb2.py diff --git a/amazon/ion/_ioncmodule.h b/src/_ioncmodule.h similarity index 100% rename from amazon/ion/_ioncmodule.h rename to src/_ioncmodule.h diff --git a/ion-c b/src/ion-c similarity index 100% rename from ion-c rename to src/ion-c diff --git a/amazon/ion/ioncmodule.c b/src/ioncmodule.c similarity index 100% rename from amazon/ion/ioncmodule.c rename to src/ioncmodule.c From c5c55905843948fd346ef307989291ed4a5e2800 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Thu, 17 Oct 2024 16:49:16 -0700 Subject: [PATCH 02/24] Switch to pkgutil.extend_path (revisit PEP420, this may be unneeded), and update paths for ionc headers in extension --- src-python/amazon/__init__.py | 8 +++++++- src/_ioncmodule.h | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src-python/amazon/__init__.py b/src-python/amazon/__init__.py index 88e019661..54bff0954 100644 --- a/src-python/amazon/__init__.py +++ b/src-python/amazon/__init__.py @@ -13,4 +13,10 @@ # License. # XXX Setuptools Namespace Package -__import__("pkg_resources").declare_namespace(__name__) +#__import__("pkg_resources").declare_namespace(__name__) +""" +Amazon Ion +""" +__version__ = "0.12.0" +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/src/_ioncmodule.h b/src/_ioncmodule.h index e991ec5ce..0e681b95e 100644 --- a/src/_ioncmodule.h +++ b/src/_ioncmodule.h @@ -2,8 +2,8 @@ #define _IONCMODULE_H_ #include "structmember.h" -#include "decimal128.h" -#include "ion.h" +#include "decNumber/decimal128.h" +#include "ionc/ion.h" PyObject* ionc_init_module(void); iERR ionc_write_value(hWRITER writer, PyObject* obj, PyObject* tuple_as_sexp); From f143a42adbb7fb77724fa1d090b22d06ca0d653b Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Thu, 17 Oct 2024 16:51:14 -0700 Subject: [PATCH 03/24] Remove install.py; Update ion-c to latest (fixes sub-directory builds) --- install.py | 205 ----------------------------------------------------- 1 file changed, 205 deletions(-) delete mode 100644 install.py diff --git a/install.py b/install.py deleted file mode 100644 index a8aca322e..000000000 --- a/install.py +++ /dev/null @@ -1,205 +0,0 @@ -# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at: -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS -# OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the -# License. - -# Python 2/3 compatibility -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import platform -import shutil -import sys -from subprocess import check_call -from os.path import join, abspath, isdir, dirname - -_PYPY = hasattr(sys, 'pypy_translation_info') -_OS = platform.system() -_WIN = _OS == 'Windows' -_MAC = _OS == 'Darwin' -_LINUX = _OS == 'Linux' - -_C_EXT_DEPENDENCY_DIR = abspath(join(dirname(os.path.abspath(__file__)), 'amazon/ion/ion-c-build')) -_C_EXT_DEPENDENCY_LIB_LOCATION = abspath(join(_C_EXT_DEPENDENCY_DIR, 'lib')) -_C_EXT_DEPENDENCY_INCLUDES_DIR = abspath(join(_C_EXT_DEPENDENCY_DIR, 'include')) -_C_EXT_DEPENDENCY_INCLUDES_LOCATIONS = { - 'ionc': abspath(join(_C_EXT_DEPENDENCY_INCLUDES_DIR, 'ionc')), - 'decNumber': abspath(join(_C_EXT_DEPENDENCY_INCLUDES_DIR, 'decNumber')) -} -_CURRENT_ION_C_DIR = './ion-c' - -_IONC_REPO_URL = "https://github.com/amazon-ion/ion-c.git" -_IONC_DIR = abspath(join(dirname(os.path.abspath(__file__)), 'ion-c')) -_IONC_LOCATION = abspath(join(dirname(os.path.abspath(__file__)), 'ion-c', 'build', 'release')) -_IONC_INCLUDES_LOCATIONS = { - 'ionc': abspath(join(dirname(os.path.abspath(__file__)), 'ion-c', 'ionc', 'include', 'ionc')), - 'decNumber': abspath(join(dirname(os.path.abspath(__file__)), 'ion-c', 'decNumber', 'include', 'decNumber')) -} - -_LIB_PREFIX = 'lib' - -_LIB_SUFFIX_MAC = '.dylib' -_LIB_SUFFIX_WIN = '.lib' -_LIB_SUFFIX_LINUX = '.so' - - -def _get_lib_name(name): - if _MAC: - return '%s%s%s' % (_LIB_PREFIX, name, _LIB_SUFFIX_MAC) - elif _LINUX: - return '%s%s%s' % (_LIB_PREFIX, name, _LIB_SUFFIX_LINUX) - elif _WIN: - return '%s%s' % (name, _LIB_SUFFIX_WIN) - - -def _library_exists(): - return _library_exists_helper('ionc') and _library_exists_helper('decNumber') - - -def _library_exists_helper(name): - return os.path.exists(join(_IONC_INCLUDES_LOCATIONS[name])) \ - and os.path.exists(join(_IONC_LOCATION, name)) - - -def readonly_handler(func, path, execinfo): - os.chmod(path, 128) - func(path) - - -def _download_ionc(): - try: - # Install ion-c. - if isdir(_CURRENT_ION_C_DIR): - shutil.rmtree(_CURRENT_ION_C_DIR, onerror=readonly_handler) - - # TODO submodule does not work, need to be fixed. - # check_call(['git', 'submodule', 'update', '--init', '--recursive']) - check_call(['git', 'clone', '--recurse-submodules', _IONC_REPO_URL, 'ion-c']) - - os.chdir(_CURRENT_ION_C_DIR) - - # Initialize submodule. - check_call(['git', 'submodule', 'update', '--init']) - - # Build ion-c. - _build_ionc() - - os.chdir('../') - return True - except: - if isdir(_IONC_DIR): - shutil.rmtree(_IONC_DIR, onerror=readonly_handler) - print('ionc build error: Unable to build ion-c library.') - return False - - -def _build_ionc(): - if _WIN: - _build_ionc_win() - elif _MAC or _LINUX: - _build_ionc_mac_and_linux() - - -def _move_ionc(): - # move ion-c to output dir. - if _WIN: - _move_lib_win('ionc') - _move_lib_win('decNumber') - elif _MAC or _LINUX: - _move_lib_mac_and_linux('ionc') - _move_lib_mac_and_linux('decNumber') - - -def _build_ionc_win(): - # check_call('cmake -G \"Visual Studio 15 2017 Win64\"') - # check_call('cmake -G \"Visual Studio 16 2019\"') - check_call('cmake -G \"Visual Studio 17 2022\"') - check_call('cmake --build . --config Release --target decNumber --target ion') - - -def _move_lib_win(name): - """ - Move library and its include files to ion-c-build/lib and ion-c-build/include respectively. - """ - for f in os.listdir(_IONC_INCLUDES_LOCATIONS[name]): - shutil.copy(join(_IONC_INCLUDES_LOCATIONS[name], f), _C_EXT_DEPENDENCY_INCLUDES_LOCATIONS[name]) - - lib_path = join(_IONC_DIR, name, 'Release', '%s%s' % (name, _LIB_SUFFIX_WIN)) - shutil.copy(lib_path, _C_EXT_DEPENDENCY_LIB_LOCATION) - - -def _build_ionc_mac_and_linux(): - # build ion-c. - check_call(['./build-release.sh']) - - -def _move_lib_mac_and_linux(name): - """ - Move library and its include files to ion-c-build/lib and ion-c-build/include respectively. - """ - for f in os.listdir(_IONC_INCLUDES_LOCATIONS[name]): - shutil.copy(join(_IONC_INCLUDES_LOCATIONS[name], f), _C_EXT_DEPENDENCY_INCLUDES_LOCATIONS[name]) - - dir_path = join(_IONC_LOCATION, name) - for file in os.listdir(dir_path): - file_path = join(dir_path, file) - if _LINUX: - if file.startswith('%s%s%s' % (_LIB_PREFIX, name, _LIB_SUFFIX_LINUX)): - shutil.copy(file_path, _C_EXT_DEPENDENCY_LIB_LOCATION) - elif _MAC: - if file.endswith(_LIB_SUFFIX_MAC): - shutil.copy(file_path, _C_EXT_DEPENDENCY_LIB_LOCATION) - - -def move_build_lib_for_distribution(): - # Create a directory to store build output. - if isdir(_C_EXT_DEPENDENCY_DIR): - shutil.rmtree(_C_EXT_DEPENDENCY_DIR, onerror=readonly_handler) - os.mkdir(_C_EXT_DEPENDENCY_DIR) - os.mkdir(_C_EXT_DEPENDENCY_LIB_LOCATION) - os.mkdir(_C_EXT_DEPENDENCY_INCLUDES_DIR) - os.mkdir(_C_EXT_DEPENDENCY_INCLUDES_LOCATIONS['ionc']) - os.mkdir(_C_EXT_DEPENDENCY_INCLUDES_LOCATIONS['decNumber']) - # Move ion-c binaries to ion-c-build - _move_ionc() - - -def _check_dependencies(): - try: - check_call(['git', '--version']) - check_call(['cmake', '--version']) - # TODO add more dependency check here. - except: - print('ion-c build error: Missing dependencies.') - return False - return True - - -def _install_ionc(): - if _PYPY: # This is pointless if running with PyPy, which doesn't support CPython extensions anyway. - return False - - if not _check_dependencies(): - return False - - if not _library_exists(): - if not _download_ionc(): - return False - move_build_lib_for_distribution() - - return True - - -if __name__ == '__main__': - _install_ionc() From bff1587b3183bef5f59adc9e8f2ed22bb7c775f0 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Fri, 18 Oct 2024 00:56:32 -0700 Subject: [PATCH 04/24] Update benchmark-cli to use new python code path --- tests/test_benchmark_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_benchmark_cli.py b/tests/test_benchmark_cli.py index 913b74cbb..9d948b53a 100644 --- a/tests/test_benchmark_cli.py +++ b/tests/test_benchmark_cli.py @@ -17,7 +17,7 @@ def generate_test_path(p): def run_cli(c): import subprocess - cmd = ["python", abspath(join(dirname(os.path.abspath(__file__)), '../amazon/ionbenchmark/ion_benchmark_cli.py'))] + c + cmd = ["python", abspath(join(dirname(os.path.abspath(__file__)), '../src-python/amazon/ionbenchmark/ion_benchmark_cli.py'))] + c proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, text=True) error_code = proc.wait() (out, err) = proc.communicate() From 36eeb759d0c7b7cde0f582a984577cb6c425fa66 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Fri, 18 Oct 2024 17:18:01 -0700 Subject: [PATCH 05/24] Add new pyproject.toml and CMakeLists --- CMakeLists.txt | 6 +++++ pyproject.toml | 59 ++++++++++++++++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 24 +++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 CMakeLists.txt create mode 100644 pyproject.toml create mode 100644 src/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..991041e1f --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.6 FATAL_ERROR) +project(amazon) + +option(IONC_BUILD_TESTS OFF) + +add_subdirectory(src) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..08bc84fac --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,59 @@ +[project] +name = "amazon-ion" +readme = "README.md" +requires-python = ">=3.8" +license = { "file" = "LICENSE" } +authors = [{ "name" = "Amazon Ion Team", "email" = "ion-team@amazon.com" }] +keywords = [] +classifiers = [] +urls = { "Documentation" = "https://ion-python.readthedocs.io/en/latest/?badge=latest" } +dependencies = [ + "appdirs==1.4.4", + "attrs==21.2.0", + "distlib==0.3.8", + "filelock==3.14.0", + "iniconfig==1.1.1", + "jsonconversion==0.2.12", + "packaging==20.9", + "pluggy==0.13.1", + "py==1.11.0", + "pyparsing==3.2.0", + "pytest==6.2.4", + "pytest-runner==5.3.1", + "toml==0.10.2", +] +dynamic = ["version", "description"] + +[project.optional-dependencies] +benchmarking = [ + "docopt==0.6.2", + "tabulate==0.9.0", + "simplejson~=3.18.3", + "six~=1.16.0", + "cbor~=1.0.0", + "cbor2~=5.4.6", + "python-rapidjson~=1.19", + "ujson~=5.7.0", + "protobuf>=4.0.0", +] +dev = [ + "tox==3.23.1", + "virtualenv==20.26.2", +] + +[build-system] +requires = ["py-build-cmake~=0.1.8"] +build-backend = "py_build_cmake.build" + +[tool.py-build-cmake.module] +name = "amazon" +directory = "src-python" + +[tool.py-build-cmake.sdist] +include = ["CMakeLists.txt", "src/*"] + +[tool.py-build-cmake.cmake] +build_type = "Release" +source_path = "src" +build_args = [] +install_components = ["python_module"] diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 000000000..e66a88ffb --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.6 FATAL_ERROR) +project(ion) + +find_package(Python3 REQUIRED COMPONENTS Development.Module) + +set(IONC_BUILD_TESTS OFF) +add_subdirectory(ion-c/ EXCLUDE_FROM_ALL) + +Python3_add_library(_ioncmodule MODULE "ioncmodule.c") +target_link_libraries(_ioncmodule PRIVATE objlib decNumber m) +target_include_directories(_ioncmodule PUBLIC + $ + $ +) + +install(TARGETS _ioncmodule + EXCLUDE_FROM_ALL + COMPONENT python_module + DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}) + +# install(FILES _ioncmodule.pyi +# EXCLUDE_FROM_ALL +# COMPONENT python_module +# DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}) From f70105d91bd52573e0c3b90cb7bcb8a02b7e6980 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Mon, 21 Oct 2024 02:38:09 -0700 Subject: [PATCH 06/24] Adjust IONC_BUILD_TESTS override --- CMakeLists.txt | 2 -- src/CMakeLists.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 991041e1f..edb952270 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,4 @@ cmake_minimum_required(VERSION 3.6 FATAL_ERROR) project(amazon) -option(IONC_BUILD_TESTS OFF) - add_subdirectory(src) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e66a88ffb..16ff07d8d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,7 +3,7 @@ project(ion) find_package(Python3 REQUIRED COMPONENTS Development.Module) -set(IONC_BUILD_TESTS OFF) +set(IONC_BUILD_TESTS OFF CACHE BOOL "disable ionc test builds") add_subdirectory(ion-c/ EXCLUDE_FROM_ALL) Python3_add_library(_ioncmodule MODULE "ioncmodule.c") From 41b60fed6ab540a12c654a1b82e63bd6bd23fd28 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Mon, 21 Oct 2024 02:44:58 -0700 Subject: [PATCH 07/24] Update main github actions workflow --- .github/workflows/main.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1ac77020f..4af52a82c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,11 +21,9 @@ jobs: - name: Create a virtual environment run: git submodule init && git submodule update && python3 -m venv ./venv && . venv/bin/activate - - run: pip install --upgrade setuptools - - run: pip install -r requirements.txt - - run: pip install -e . - # Run our tests, but do not run the benchmark tests since they pull in libraries that do not have pypy support. - - run: py.test --ignore tests/test_benchmark_cli.py --ignore tests/test_benchmark_spec.py + - run: python -m build + - run: python -m pip install -e . + - run: py.test test-windows: runs-on: windows-latest @@ -42,8 +40,6 @@ jobs: - name: Create a virtual environment run: git submodule init && git submodule update && python3 -m venv ./venv && . venv/Scripts/activate - - run: pip install --upgrade setuptools - - run: pip install -r requirements.txt - - run: pip install -e . - # Run our tests, but do not run the benchmark tests since they pull in libraries that do not have pypy support. - - run: py.test --ignore tests/test_benchmark_cli.py --ignore tests/test_benchmark_spec.py + - run: python -m build + - run: python -m pip install -e . + - run: py.test From a358f91ce0b4cc18bb6aa10e3dee083609a02bec Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Mon, 21 Oct 2024 03:03:49 -0700 Subject: [PATCH 08/24] Install build module --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4af52a82c..f226e183a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,8 @@ jobs: - name: Create a virtual environment run: git submodule init && git submodule update && python3 -m venv ./venv && . venv/bin/activate - - run: python -m build + - run: python -m pip install build + - run: python -m build . - run: python -m pip install -e . - run: py.test @@ -40,6 +41,7 @@ jobs: - name: Create a virtual environment run: git submodule init && git submodule update && python3 -m venv ./venv && . venv/Scripts/activate + - run: python -m pip install build - run: python -m build - run: python -m pip install -e . - run: py.test From f69ecf715ef82e2f63e0132a2095b2bad27b080d Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Mon, 21 Oct 2024 03:15:35 -0700 Subject: [PATCH 09/24] Lower pyparsing version to support 3.8 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 08bc84fac..ac81b269e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ "packaging==20.9", "pluggy==0.13.1", "py==1.11.0", - "pyparsing==3.2.0", + "pyparsing==3.0.0", "pytest==6.2.4", "pytest-runner==5.3.1", "toml==0.10.2", From fd2ff2f1878f25e83ec1f5f8fec20803636e827d Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Fri, 8 Nov 2024 02:52:21 -0800 Subject: [PATCH 10/24] Further migration to pyproject.toml; move 3rd party configs into pyproject, update release workflow, and address max-int digits issue for test vectors --- .github/workflows/release.yml | 17 ++++----- pyproject.toml | 59 +++++++++++++++++++++------- pytest.ini | 3 -- setup.cfg | 2 - setup.py | 72 ----------------------------------- src-python/amazon/__init__.py | 2 - tests/test_vectors.py | 3 ++ tox.ini | 8 ---- 8 files changed, 56 insertions(+), 110 deletions(-) delete mode 100644 pytest.ini delete mode 100644 setup.cfg delete mode 100644 setup.py delete mode 100644 tox.ini diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c30578c31..078b12c32 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,13 +20,10 @@ jobs: - name: Create a virtual environment run: git submodule init && git submodule update && python3 -m venv ./venv && . venv/bin/activate - - name: Install dependencies.. - run: | - pip install --upgrade setuptools - pip install -r requirements.txt - pip install -e . - - name: Run Tests - run: py.test --ignore tests/test_benchmark_cli.py --ignore tests/test_benchmark_spec.py + - run: python -m pip install build + - run: python -m build . + - run: python -m pip install -e . + - run: py.test --ignore tests/test_benchmark_cli.py --ignore tests/test_benchmark_spec.py source-distribution: if: startsWith(github.ref, 'refs/tags/') @@ -40,13 +37,13 @@ jobs: - uses: actions/checkout@v4 - name: Build sdist - run: python3 setup.py sdist + run: python -m pip install build && python -m build -s - name: Upload source to Github uses: actions/upload-artifact@v4 with: name: source - path: ./dist/amazon.ion-*.tar.gz + path: ./dist/amazon_ion-*.tar.gz - uses: aws-actions/configure-aws-credentials@v4 with: @@ -60,7 +57,7 @@ jobs: - name: Upload source to s3 run: | - zip ion-python-source ./dist/amazon.ion-*.tar.gz + zip ion-python-source ./dist/amazon_ion-*.tar.gz aws s3 mv ion-python-source.zip ${{ secrets.AWS_SOURCE_BUCKET_URL }} --acl bucket-owner-full-control - name: Install the released source from PYPI diff --git a/pyproject.toml b/pyproject.toml index ac81b269e..c8bebeb61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,23 +8,17 @@ keywords = [] classifiers = [] urls = { "Documentation" = "https://ion-python.readthedocs.io/en/latest/?badge=latest" } dependencies = [ - "appdirs==1.4.4", "attrs==21.2.0", - "distlib==0.3.8", - "filelock==3.14.0", - "iniconfig==1.1.1", - "jsonconversion==0.2.12", - "packaging==20.9", - "pluggy==0.13.1", - "py==1.11.0", + "setuptools==71.0.0", # Needed for jsonconversion, it seems + "jsonconversion==0.2.13", # 1.0.1 tightens the requirements for pytest, and 1.0.0 is broken. "pyparsing==3.0.0", - "pytest==6.2.4", - "pytest-runner==5.3.1", - "toml==0.10.2", ] dynamic = ["version", "description"] [project.optional-dependencies] +test = [ + "pytest==8.3.3" +] benchmarking = [ "docopt==0.6.2", "tabulate==0.9.0", @@ -34,17 +28,22 @@ benchmarking = [ "cbor2~=5.4.6", "python-rapidjson~=1.19", "ujson~=5.7.0", - "protobuf>=4.0.0", + "protobuf>=4.0.0" ] dev = [ "tox==3.23.1", - "virtualenv==20.26.2", + "virtualenv==20.27.0" ] +# Build System ################################################################################ [build-system] requires = ["py-build-cmake~=0.1.8"] build-backend = "py_build_cmake.build" +# py-build-cmake ############################################################################## +# We use CMake for our extension's build. py-build-cmake implements the build backend to build +# our extension, and the underlying ion-c libraries it depends on, using cmake as well as provide +# the mechanisms to produce the sdist as well. [tool.py-build-cmake.module] name = "amazon" directory = "src-python" @@ -57,3 +56,37 @@ build_type = "Release" source_path = "src" build_args = [] install_components = ["python_module"] + +# cibuildwheel ################################################################################ +# Used to build the various wheels we want to support so that our users do not need to build +# the extension themselves. cibuildwheel can handle building the extension for multiple python +# versions and architectures. +[tool.cibuildwheel] +before-all = "uname -a" + +# pytest ###################################################################################### +# Unit testing options and support. +[tool.pytest.ini_options] +minversion = "6.0" +# addopts = "-rw --strict-markers" +testpaths = ["tests"] + +# tox ######################################################################################### +# Virtual Environment Management; When using tox the envlist defined here will be used for +# running our default unit tests (minus benchmarking related tests). This ensures that we +# can easily test against supported python versions. +[tool.tox] +envlist = ["py3.8", "py3.9", "pypy3"] # default environments to run. + +[tool.tox.env_run_base] +extras = ["test"] +requires = ["pip >= 21.1.2", "setuptools >= 57.0.0"] +commands = [ + ["py.test", "--ignore", "tests/test_benchmark_cli.py", "--ignore", "tests/test_benchmark_spec.py", { replace = "posargs", extend = true}] +] + +[tool.tox.env.benchmark_tests] +extras = ["test", "benchmarking"] +commands = [ + ["py.test", "tests/test_benchmark_cli.py", "tests/test_benchmark_spec.py", { replace = "posargs", extend = true}] +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 99adee9b4..000000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -testpaths=tests -addopts=-rw --strict diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b7e478982..000000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[aliases] -test=pytest diff --git a/setup.py b/setup.py deleted file mode 100644 index 74ff24854..000000000 --- a/setup.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at: -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS -# OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the -# License. - -import sys -from setuptools import setup, find_packages, Extension -from install import _install_ionc - -C_EXT = True if not hasattr(sys, 'pypy_translation_info') else False - - -def run_setup(): - if C_EXT and _install_ionc(): - print('C extension is enabled!') - kw = dict( - ext_modules=[ - Extension( - 'amazon.ion.ionc', - sources=['amazon/ion/ioncmodule.c'], - include_dirs=['amazon/ion/ion-c-build/include', - 'amazon/ion/ion-c-build/include/ionc', - 'amazon/ion/ion-c-build/include/decNumber'], - libraries=['ionc', 'decNumber'], - library_dirs=['amazon/ion/ion-c-build/lib'], - extra_link_args=['-Wl,-rpath,%s' % '$ORIGIN/ion-c-build/lib', # LINUX - '-Wl,-rpath,%s' % '@loader_path/ion-c-build/lib' # MAC - ], - extra_compile_args=['-std=c99'], - ), - ], - ) - else: - print('Using pure python implementation.') - kw = dict() - - - setup( - name='amazon.ion', - version='0.13.0', - description='A Python implementation of Amazon Ion.', - url='http://github.com/amazon-ion/ion-python', - author='Amazon Ion Team', - author_email='ion-team@amazon.com', - license='Apache License 2.0', - - - packages=find_packages(exclude=['tests*']), - include_package_data=True, - namespace_packages=['amazon'], - - setup_requires=[ - 'pytest-runner', - ], - - tests_require=[ - 'pytest', - ], - **kw - ) - - -run_setup() diff --git a/src-python/amazon/__init__.py b/src-python/amazon/__init__.py index 54bff0954..99a8376d2 100644 --- a/src-python/amazon/__init__.py +++ b/src-python/amazon/__init__.py @@ -12,8 +12,6 @@ # specific language governing permissions and limitations under the # License. -# XXX Setuptools Namespace Package -#__import__("pkg_resources").declare_namespace(__name__) """ Amazon Ion """ diff --git a/tests/test_vectors.py b/tests/test_vectors.py index 315d99db4..431a4f91c 100644 --- a/tests/test_vectors.py +++ b/tests/test_vectors.py @@ -33,6 +33,9 @@ PYPY = hasattr(sys, 'pypy_translation_info') +if hasattr(sys, 'set_int_max_str_digits'): + sys.set_int_max_str_digits(10000) # See: https://github.com/python/cpython/issues/95778 + # This file lives in the tests/ directory. Up one level is tests/ and up another level is the package root, which # contains the vectors/ directory. _VECTORS_ROOT = abspath(join(abspath(__file__), u'..', u'..', u'vectors', u'iontestdata')) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 02950f008..000000000 --- a/tox.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -envlist = py37,py38,py39,pypy3 - -[testenv] -requires= pip >= 21.1.2 - setuptools >= 57.0.0 -deps=-rrequirements.txt -commands={posargs:py.test} From d33a4a44510ad8a1c68cabae329a41c340317f5c Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 15 Jan 2025 14:22:47 -0800 Subject: [PATCH 11/24] Update main workflow to include test dependencies when installing --- .github/workflows/main.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f226e183a..f8aefb16c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,10 +21,10 @@ jobs: - name: Create a virtual environment run: git submodule init && git submodule update && python3 -m venv ./venv && . venv/bin/activate - - run: python -m pip install build - - run: python -m build . - - run: python -m pip install -e . - - run: py.test + - run: python -m pip install build virtualenv + - run: python -m pip install '.[test]' + # Run our tests, but do not run the benchmark tests since they pull in libraries that do not have pypy support. + - run: py.test --ignore tests/test_benchmark_cli.py --ignore tests/test_benchmark_spec.py test-windows: runs-on: windows-latest @@ -43,5 +43,6 @@ jobs: run: git submodule init && git submodule update && python3 -m venv ./venv && . venv/Scripts/activate - run: python -m pip install build - run: python -m build - - run: python -m pip install -e . - - run: py.test + - run: python -m pip install -e '.[test]' + # Run our tests, but do not run the benchmark tests since they pull in libraries that do not have pypy support. + - run: py.test --ignore tests/test_benchmark_cli.py --ignore tests/test_benchmark_spec.py From 56cf1056b2a6cb4b7d2ee7015baa6ceed695a895 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 15 Jan 2025 14:24:50 -0800 Subject: [PATCH 12/24] Update extension to link against decNumber as a static lib --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 16ff07d8d..4c37bdf45 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,13 +7,13 @@ set(IONC_BUILD_TESTS OFF CACHE BOOL "disable ionc test builds") add_subdirectory(ion-c/ EXCLUDE_FROM_ALL) Python3_add_library(_ioncmodule MODULE "ioncmodule.c") -target_link_libraries(_ioncmodule PRIVATE objlib decNumber m) +target_link_libraries(_ioncmodule PRIVATE objlib decNumber_static m) target_include_directories(_ioncmodule PUBLIC $ $ ) -install(TARGETS _ioncmodule +install(TARGETS _ioncmodule objlib EXCLUDE_FROM_ALL COMPONENT python_module DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}) From 480350eabe2384743796cf5f6515bc1d18421e56 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 15 Jan 2025 14:25:48 -0800 Subject: [PATCH 13/24] Update extension name, changed to _ioncmodule and is now at the top-level --- src-python/amazon/ion/simpleion.py | 2 +- src/ioncmodule.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src-python/amazon/ion/simpleion.py b/src-python/amazon/ion/simpleion.py index 709e072bc..f2d98c512 100644 --- a/src-python/amazon/ion/simpleion.py +++ b/src-python/amazon/ion/simpleion.py @@ -77,7 +77,7 @@ from .writer_binary import binary_writer try: - import amazon.ion.ionc as ionc + import amazon._ioncmodule as ionc __IS_C_EXTENSION_SUPPORTED = True except ModuleNotFoundError: __IS_C_EXTENSION_SUPPORTED = False diff --git a/src/ioncmodule.c b/src/ioncmodule.c index e66e0f7d1..d34d60df6 100644 --- a/src/ioncmodule.c +++ b/src/ioncmodule.c @@ -1668,7 +1668,7 @@ static PyObject* init_module(void) { } PyMODINIT_FUNC -PyInit_ionc(void) +PyInit__ioncmodule(void) { return init_module(); } From eb51696e981ecb25f3924afd62b7f469658265c5 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 15 Jan 2025 15:17:50 -0800 Subject: [PATCH 14/24] Fix version, it got lost in the rebase --- src-python/amazon/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-python/amazon/__init__.py b/src-python/amazon/__init__.py index 99a8376d2..b1f1ce00c 100644 --- a/src-python/amazon/__init__.py +++ b/src-python/amazon/__init__.py @@ -15,6 +15,6 @@ """ Amazon Ion """ -__version__ = "0.12.0" +__version__ = "0.13.0" from pkgutil import extend_path __path__ = extend_path(__path__, __name__) From 12427adc02f4ca4f890ed2dfe64fab298cc777b9 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 15 Jan 2025 15:24:02 -0800 Subject: [PATCH 15/24] Remove requirements files; add py-build-cmake cache to gitignore --- .gitignore | 2 +- requirements.txt | 20 -------------------- requirements_benchmark.txt | 5 ----- 3 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 requirements.txt delete mode 100644 requirements_benchmark.txt diff --git a/.gitignore b/.gitignore index 21f7d7ee3..5d270e9dd 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,4 @@ /amazon/ion/ion-c-build/ /temp_*.ion /temp_*.10n - +/.py-build-cmake_cache diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 596b27b59..000000000 --- a/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -appdirs==1.4.4 -attrs==21.2.0 -distlib==0.3.9 -filelock==3.16.1 -iniconfig==1.1.1 -jsonconversion==0.2.12 -packaging==20.9 -pluggy==0.13.1 -py==1.11.0 -pyparsing==2.4.7 -pytest==6.2.4 -pytest-runner==5.3.1 -toml==0.10.2 -tox==3.23.1 -virtualenv==20.28.0 -setuptools==72.0.0 -docopt==0.6.2 -tabulate==0.9.0 -simplejson~=3.18.3 -six~=1.16.0 diff --git a/requirements_benchmark.txt b/requirements_benchmark.txt deleted file mode 100644 index 504fb34d7..000000000 --- a/requirements_benchmark.txt +++ /dev/null @@ -1,5 +0,0 @@ -cbor~=1.0.0 -cbor2~=5.4.6 -python-rapidjson~=1.19 -ujson~=5.7.0 -protobuf>=4.0.0 From 60ab03a0c40f68e89a51ad1c7472f6795991bf9a Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 15 Jan 2025 16:41:51 -0800 Subject: [PATCH 16/24] Add ionc_version method to return the version of Ion C used in the extension --- src/ioncmodule.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/ioncmodule.c b/src/ioncmodule.c index d34d60df6..45155f5e1 100644 --- a/src/ioncmodule.c +++ b/src/ioncmodule.c @@ -1531,6 +1531,12 @@ PyObject* ionc_read(PyObject* self, PyObject *args, PyObject *kwds) { return exception; } +PyObject *ionc_version() { + char const *version = ion_version_full(); + PyObject *version_obj = PyUnicode_FromString(version); + return version_obj; +} + /****************************************************************************** * Initial module * ******************************************************************************/ @@ -1541,6 +1547,7 @@ static char ioncmodule_docs[] = static PyMethodDef ioncmodule_funcs[] = { {"ionc_write", (PyCFunction)ionc_write, METH_VARARGS | METH_KEYWORDS, ioncmodule_docs}, {"ionc_read", (PyCFunction)ionc_read, METH_VARARGS | METH_KEYWORDS, ioncmodule_docs}, + {"ionc_version", (PyCFunction)ionc_version, METH_NOARGS, ioncmodule_docs}, {NULL} }; From e6e16a237dcb51003c4963e3756b0250e8464ce0 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 15 Jan 2025 16:42:53 -0800 Subject: [PATCH 17/24] Quick update pass for C Extension documentation --- C_EXTENSION.md | 73 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 8 deletions(-) diff --git a/C_EXTENSION.md b/C_EXTENSION.md index 02a483ee6..39847955a 100644 --- a/C_EXTENSION.md +++ b/C_EXTENSION.md @@ -76,16 +76,73 @@ b'\xe0\x01\x00\xea\xe9\x81\x83\xd6\x87\xb4\x83abc\xd3\x8a!{' ## Development -Architecture of Ion Python C extension: +The Ion Python C extension is built as part of the PEP 517 build process using py-build-cmake, and leaning on +Ion C's existing cmake build. A revision of Ion C is included as a submodule in this repo under `src/ion-c`. +If you would like to update the version of Ion C, simply update the submodule to point to the desired revision. + +The file `src/CMakeLists.txt` acts as the build script for the C extension itself, which then includes the Ion C +codebase into the build tree. + +With the extension built, it will be exposed to python as `amazon._ioncmodule`. For example: +```python +>>> import amazon._ioncmodule as ionc +>>> ionc.ionc_version() +'v1.1.3 (rev: d61c09a)' +>>> ``` - ioncmodule.c - | - | - ↓ -Ion C -------> Ion C binaries -----> setup.py ------> C extension -------------------> Ion Python simpleion module - compile setup import ionc module + +The `amazon.ion.simpleion` module then makes use of this extension when it is available to provide more efficient +Ion parsing. Importing `amazon._ioncmodule` directly can determine if it is available, however simpleion also provides +the field `__IS_C_EXTENSION_SUPPORTED`. +```python +>>> import amazon.ion.simpleion as ion +>>> ion.__IS_C_EXTENSION_SUPPORTED +True +>>> +``` + +In order to build the extension, along with the package itself, we can use python's build module: +```python +ion-python# python -m build . +* Creating isolated environment: venv+pip... +* Installing packages in isolated environment: + - py-build-cmake~=0.1.8 +* Getting build dependencies for sdist... +* Building sdist... +* Building wheel from sdist +* Creating isolated environment: venv+pip... +* Installing packages in isolated environment: + - py-build-cmake~=0.1.8 +* Getting build dependencies for wheel... +* Building wheel... +... +Successfully built amazon_ion-0.13.0.tar.gz and amazon_ion-0.13.0-cp310-cp310-linux_x86_64.whl +``` +This will build both the source wheel, and the binary wheel for the current system. Installing the module can +be done with pip. Depending on what you're doing with the package you may want to install different dependencies. +Different sets of optional dependencies are provided, such as `test`, and `benchmarking`. More details can be +found in the `pyproject.toml`. + +To install the package and dependencies for unit tests you can run: +```python +ion-python# python -m pip install '.[test]' +Processing /ion-python + Installing build dependencies ... done + Getting requirements to build wheel ... done + Preparing metadata (pyproject.toml) ... done +Building wheels for collected packages: amazon_ion + Building wheel for amazon_ion (pyproject.toml) ... done + Created wheel for amazon_ion: filename=amazon_ion-0.13.0-cp310-cp310-linux_x86_64.whl size=573770 sha256=96ef01efea7519a1a38d9fc9e42139ab82125aafa328cfb4a4823458de802fa1 + Stored in directory: /root/.cache/pip/wheels/78/55/f9/c6d69051d6a93c725251429f62d0d5b7d16d8c982a3772d666 +Successfully built amazon_ion +Installing collected packages: amazon_ion + Attempting uninstall: amazon_ion + Found existing installation: amazon_ion 0.13.0 + Uninstalling amazon_ion-0.13.0: + Successfully uninstalled amazon_ion-0.13.0 +Successfully installed amazon_ion-0.13.0 ``` -After setup, C extension will be built and imported to simpleion module. If there are changes in `ioncmodule.c`, build the latest C extension by running `python setup.py build_ext --inplace`. +Installing with `-e` will also allow you to update the python side of the package without having to re-install. ## Technical Details From dddd696ade68db2c234d4caa61c5009fbaaf5853 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Thu, 16 Jan 2025 13:09:43 -0800 Subject: [PATCH 18/24] Remove libm from linked libraries for windows build --- src/CMakeLists.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4c37bdf45..a6cbf8bcd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,7 +7,14 @@ set(IONC_BUILD_TESTS OFF CACHE BOOL "disable ionc test builds") add_subdirectory(ion-c/ EXCLUDE_FROM_ALL) Python3_add_library(_ioncmodule MODULE "ioncmodule.c") -target_link_libraries(_ioncmodule PRIVATE objlib decNumber_static m) + +if (MSVC) + target_link_libraries(_ioncmodule PRIVATE objlib decNumber_static) +else() + # We need to link against libm explicitly for *nix. + target_link_libraries(_ioncmodule PRIVATE objlib decNumber_static m) +endif() + target_include_directories(_ioncmodule PUBLIC $ $ From 1d5923b0b67f2b3afb02f09b146af81508d5b7ae Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Thu, 16 Jan 2025 16:16:20 -0800 Subject: [PATCH 19/24] Update regression test to use pyproject for PR source (will need to be updated to do the same for baseline once merged) --- .github/workflows/performance-regression.yml | 27 ++++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/.github/workflows/performance-regression.yml b/.github/workflows/performance-regression.yml index d484c5d0e..df4a6b658 100644 --- a/.github/workflows/performance-regression.yml +++ b/.github/workflows/performance-regression.yml @@ -55,15 +55,11 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.13' cache: 'pip' - cache-dependency-path: | - **/requirements.txt - **/requirements_benchmark.txt + cache-dependency-path: pyproject.toml - run: | - pip install -r requirements.txt - [ -e "requirements_benchmark.txt" ] && pip install -r requirements_benchmark.txt # include benchmark requirements if they exist. - # TODO: See if there's a way to cache the ion-c build output if it hasn't changed + python -m pip install '.[test,benchmarking]' detect-regression: name: Check @@ -71,7 +67,7 @@ jobs: needs: [generate-test-data, prepopulate-pip-cache] strategy: matrix: - python-version: ['3.9', '3.11'] + python-version: ['3.13'] test-data: ['nestedStruct', 'nestedList', 'sexp', 'realWorldDataSchema01', 'realWorldDataSchema02', 'realWorldDataSchema03'] fail-fast: false steps: @@ -95,9 +91,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - cache-dependency-path: | - **/requirements.txt - **/requirements_benchmark.txt + cache-dependency-path: pyproject.toml - name: Download Test Data id: 'download' @@ -106,12 +100,13 @@ jobs: name: ${{env.test_data_id}} # Generates performance results for the previous commit - - name: Create a virtual environment + - name: Create a virtual environment for baseline working-directory: ./baseline run: | pip install -r requirements.txt [ -e "requirements_benchmark.txt" ] && pip install -r requirements_benchmark.txt # include benchmark requirements if they exist. pip install . + - name: Run baseline performance benchmark id: 'baseline' working-directory: ./baseline @@ -125,12 +120,10 @@ jobs: echo "report=$PWD/report.ion" >> "$GITHUB_OUTPUT" # Generates performance results for the current commit - - name: Create a virtual environment and setup the package + - name: Create a virtual environment for PR changes working-directory: ./new - run: | - pip install -r requirements.txt - [ -e "requirements_benchmark.txt" ] && pip install -r requirements_benchmark.txt # include benchmark requirements if they exist. - pip install . + run: pip install '.[test,benchmarking]' + - name: Run new performance benchmark id: 'new' working-directory: ./new From 24348359f876d5384948e11797fbf863bfe912df Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Thu, 16 Jan 2025 16:58:01 -0800 Subject: [PATCH 20/24] Update package cache to only install dependencies, not our package --- .github/workflows/performance-regression.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/performance-regression.yml b/.github/workflows/performance-regression.yml index df4a6b658..808bb6684 100644 --- a/.github/workflows/performance-regression.yml +++ b/.github/workflows/performance-regression.yml @@ -59,7 +59,8 @@ jobs: cache: 'pip' cache-dependency-path: pyproject.toml - run: | - python -m pip install '.[test,benchmarking]' + python -m pip freeze '.[test,benchmarking]' > requirements.txt # capture only dependencies. + python -m pip install -r requirements.txt detect-regression: name: Check From 49ca0ebc22b0f43ed7c009e239eb114cc3500590 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Thu, 16 Jan 2025 17:24:05 -0800 Subject: [PATCH 21/24] Update benchmark cli's path for PR content to account for updated layout --- .github/workflows/performance-regression.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/performance-regression.yml b/.github/workflows/performance-regression.yml index 808bb6684..1c705e3e2 100644 --- a/.github/workflows/performance-regression.yml +++ b/.github/workflows/performance-regression.yml @@ -15,6 +15,7 @@ env: specs: '{command:read,format:ion_text} {command:write,format:ion_text} {command:read,format:ion_binary} {command:write,format:ion_binary}' test_data_id: 'generated-test-data' run_cli: 'python amazon/ionbenchmark/ion_benchmark_cli.py' + run_cli_new: 'python src-python/amazon/ionbenchmark/ion_benchmark_cli.py' jobs: @@ -129,7 +130,7 @@ jobs: id: 'new' working-directory: ./new run: | - ${{env.run_cli}} spec '${{env.specs}}' -d '${{env.spec_defaults}}' \ + ${{env.run_cli_new}} spec '${{env.specs}}' -d '${{env.spec_defaults}}' \ -O '{input_file:"${{steps.download.outputs.download-path}}/${{ matrix.test-data }}.10n"}' \ -o "$PWD/report.ion" -r '${{env.report_statistics}}' echo "::group::Ion Report" @@ -140,4 +141,4 @@ jobs: # Compare results and identify regression - name: Detect performance regression working-directory: ./new - run: ${{env.run_cli}} compare --fail ${{steps.baseline.outputs.report}} ${{steps.new.outputs.report}} -c '${{env.compare_statistics}}' + run: ${{env.run_cli_new}} compare --fail ${{steps.baseline.outputs.report}} ${{steps.new.outputs.report}} -c '${{env.compare_statistics}}' From 070015a3178756c836a26a961b292d7edf24576c Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Fri, 17 Jan 2025 15:49:48 -0800 Subject: [PATCH 22/24] Update virtualenv version to match HEAD's dependabot update --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c8bebeb61..aae933c3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ benchmarking = [ ] dev = [ "tox==3.23.1", - "virtualenv==20.27.0" + "virtualenv==20.28.0" ] # Build System ################################################################################ From 0ce40af6cefd1a3cbbfb111e922447c9531e5ad7 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 22 Jan 2025 14:56:40 -0800 Subject: [PATCH 23/24] Remove unneeded virtualenv install; expand wording on C ext use; remove commented cmake code --- .github/workflows/main.yml | 2 +- C_EXTENSION.md | 4 ++-- src/CMakeLists.txt | 5 ----- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f8aefb16c..eb817102c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,7 @@ jobs: - name: Create a virtual environment run: git submodule init && git submodule update && python3 -m venv ./venv && . venv/bin/activate - - run: python -m pip install build virtualenv + - run: python -m pip install build - run: python -m pip install '.[test]' # Run our tests, but do not run the benchmark tests since they pull in libraries that do not have pypy support. - run: py.test --ignore tests/test_benchmark_cli.py --ignore tests/test_benchmark_spec.py diff --git a/C_EXTENSION.md b/C_EXTENSION.md index 39847955a..592a93f5f 100644 --- a/C_EXTENSION.md +++ b/C_EXTENSION.md @@ -92,8 +92,8 @@ With the extension built, it will be exposed to python as `amazon._ioncmodule`. ``` The `amazon.ion.simpleion` module then makes use of this extension when it is available to provide more efficient -Ion parsing. Importing `amazon._ioncmodule` directly can determine if it is available, however simpleion also provides -the field `__IS_C_EXTENSION_SUPPORTED`. +Ion reading and writing. Importing `amazon._ioncmodule` directly can determine if it is available, however simpleion +also provides the field `__IS_C_EXTENSION_SUPPORTED`. ```python >>> import amazon.ion.simpleion as ion >>> ion.__IS_C_EXTENSION_SUPPORTED diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a6cbf8bcd..c78354f59 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -24,8 +24,3 @@ install(TARGETS _ioncmodule objlib EXCLUDE_FROM_ALL COMPONENT python_module DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}) - -# install(FILES _ioncmodule.pyi -# EXCLUDE_FROM_ALL -# COMPONENT python_module -# DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}) From 3fdf19b5d65fc08c50bb50135d21b2a43faa9776 Mon Sep 17 00:00:00 2001 From: Richard Giliam Date: Wed, 22 Jan 2025 14:57:33 -0800 Subject: [PATCH 24/24] Address actionlint issues --- .github/workflows/performance-regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/performance-regression.yml b/.github/workflows/performance-regression.yml index 1c705e3e2..b320f814f 100644 --- a/.github/workflows/performance-regression.yml +++ b/.github/workflows/performance-regression.yml @@ -39,7 +39,7 @@ jobs: mkdir -p testData for test in nestedStruct nestedList sexp realWorldDataSchema01 realWorldDataSchema02 realWorldDataSchema03 do - java -jar $jar_file generate -S ${{env.data_size}} --input-ion-schema $schema_dir/${test}.isl testData/${test}.10n + java -jar "$jar_file" generate -S "${{env.data_size}}" --input-ion-schema "$schema_dir/${test}.isl" "testData/${test}.10n" done - name: Upload test Ion Data to artifacts uses: actions/upload-artifact@v4