diff --git a/Dockerfile b/Dockerfile index ac85f53..e6002a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,3 +9,8 @@ RUN mv go /usr/local ENV PATH=$PATH:/usr/local/go/bin RUN pip --no-cache-dir install pylint +RUN apt-get update && \ + apt-get -qy full-upgrade && \ + apt-get install -qy curl && \ + apt-get install -qy curl && \ + curl -sSL https://get.docker.com/ | sh \ No newline at end of file diff --git a/cloudbuild.yaml b/cloudbuild.yaml index f1320ec..56532ee 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -2,21 +2,36 @@ steps: - name: gcr.io/cloud-builders/docker args: ['build', '-t', 'gcr.io/$PROJECT_ID/python3', '.'] + # - name: 'gcr.io/${PROJECT_ID}/python3' + # entrypoint: 'python3' + # args: ["-m", "pylint", 'uplink_python/uplink.py'] + # - name: gcr.io/cloud-builders/gcloud + # entrypoint: 'bash' + # args: ["-c","gcloud secrets versions access latest --secret=StorjAPIKey >>secret.txt" ] + # - name: 'gcr.io/${PROJECT_ID}/python3' + # entrypoint: 'bash' + # args: ["-c", "git clone -b v1.2.2 https://github.com/storj/uplink-c"] + # - name: 'gcr.io/${PROJECT_ID}/python3' + # entrypoint: 'bash' + # args: ["-c", "cd uplink-c && go build -o libuplinkc.so -buildmode=c-shared && cp *.so ../uplink_python/"] + # - name: 'gcr.io/${PROJECT_ID}/python3' + # entrypoint: 'python3' + # args: ['-m', 'unittest', 'test/test_cases.py', '-v'] - name: 'gcr.io/${PROJECT_ID}/python3' - entrypoint: 'python3' - args: ["-m", "pylint", 'uplink_python/uplink.py'] - - name: gcr.io/cloud-builders/gcloud - entrypoint: 'bash' - args: ["-c","gcloud secrets versions access latest --secret=StorjAPIKey >>secret.txt" ] + script: | + set -o errexit + GO111MODULE=on go install storj.io/storj-up + storj-up init + docker compose up -d + docker compose ps + storj-up credentials - name: 'gcr.io/${PROJECT_ID}/python3' - entrypoint: 'bash' - args: ["-c", "git clone -b v1.2.2 https://github.com/storj/uplink-c"] - - name: 'gcr.io/${PROJECT_ID}/python3' - entrypoint: 'bash' - args: ["-c", "cd uplink-c && go build -o libuplinkc.so -buildmode=c-shared && cp *.so ../uplink_python/"] - - name: 'gcr.io/${PROJECT_ID}/python3' - entrypoint: 'python3' - args: ['-m', 'unittest', 'test/test_cases.py', '-v'] + script: | + set -o errexit + python3 -m pip install --upgrade pip + python3 -m pip install cibuildwheel==2.11.2 + cibuildwheel --platform linux --output-dir wheelhouse . + ls wheelhouse/ tags: ['cloud-builders-community'] images: ['gcr.io/$PROJECT_ID/python3'] tags: ['cloud-builders-community'] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9577c91 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,14 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel", + "Cython", +] +build-backend = "setuptools.build_meta" + +[tool.cibuildwheel] +skip = ["cp-36*", "*musllinux*"] + +#test-requires = ["unittest"] +test-command = "python3 -m unittest -s test -v" +before-all = "pip install wget && python3 scripts/install-golang.py" diff --git a/scripts/install-golang.py b/scripts/install-golang.py new file mode 100644 index 0000000..d6ce3fb --- /dev/null +++ b/scripts/install-golang.py @@ -0,0 +1,38 @@ +# usage: python3 install-golang.py platform machine +import os +import sys +import platform +import tarfile +import wget + +from subprocess import call + +cibuildwheel_to_go_platform = { + 'x86_64' : 'amd64', + '686' : '386' +} + + + + +_platform = sys.platform + + + +print("Installing go for ", _platform, "/", platform.machine()) + +''' Download golang archive and extract it ''' +GOLANG_URL = 'https://storage.googleapis.com/golang/go1.19.2.linux-amd64.tar.gz' +if platform.machine() == "i686": + GOLANG_URL = 'https://storage.googleapis.com/golang/go1.19.2.linux-386.tar.gz' + + +filename = wget.download(GOLANG_URL, 'go.tar.gz') +print("Golang archive filename = ", filename) +file = tarfile.open(filename) +file.extractall("/usr/bin/") +file.close() +cmd = ['chmod','a+x', '/usr/bin/go/bin/go'] +out = call(cmd) +if out != 0: + raise CompileError('Go build failed') diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..1ce5b57 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,22 @@ +[metadata] +name = uplink-python +version = 1.2.2.0 +author = Utropicmedia +author_email = development@utropicmedia.com +license= Apache Software License +description = Bindings for Storj network uplink +long_description= file: README.md, +long_description_content_type = text/markdown +classifiers= + Intended Audience :: Developers + Programming Language :: Python :: 3 + License :: OSI Approved :: Apache Software License + Operating System :: OS Independent + Topic :: Software Development :: Build Tools +url = https://github.com/storj-thirdparty/uplink-python + +[options] +packages = uplink_python +install_requires= wheel +include_package_data=True +python_requires = >=3.7 diff --git a/setup.py b/setup.py index 62b4653..6119f5a 100644 --- a/setup.py +++ b/setup.py @@ -3,85 +3,56 @@ import os import platform import sysconfig +import sys + +from distutils.errors import CompileError +from subprocess import call import setuptools -from setuptools.command.install import install +from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext with open("README.md", "r") as fh: long_description = fh.read() uplinkc_version = "v1.2.2" -class Install(install): - - @staticmethod - def find_module_path(): - new_path = os.path.join(sysconfig.get_paths()['purelib'], "uplink_python") - try: - os.makedirs(new_path, exist_ok=True) - os.system("echo Directory uplink_python created successfully.") - except OSError as error: - os.system("echo Error in creating uplink_python directory. Error: " + str(error)) - return new_path - - def run(self): - - try: - install_path = self.find_module_path() - os.system("echo Package installation path: " + install_path) - if platform.system() == "Windows": - os.system("icacls " + install_path + " /grant Everyone:F /t") - else: - os.system("sudo chmod -R 777 " + install_path) - os.system("echo Building libuplinkc.so") - copy_command = "copy" if platform.system() == "Windows" else "cp" - command = "git clone -b "+uplinkc_version+ "https://github.com/storj/uplink-c && cd uplink-c" \ - "&& go build -o libuplinkc.so -buildmode=c-shared" \ - "&& " + copy_command + " *.so " + install_path - build_so = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, shell=True) - output, errors = build_so.communicate() - build_so.wait() - if output is not None: - os.system("echo " + output.decode('utf-8')) - os.system("echo Building libuplinkc.so successful.") - if errors is not None: - os.system("echo " + errors.decode('utf-8')) - os.system("echo Building libuplinkc.so failed.") - if build_so.returncode != 0: - os.exit(1) - except Exception as error: - os.system("echo " + str(error)) - os.system("echo Building libuplinkc.so failed.") - - install.run(self) +class build_go_ext(build_ext): + """Custom command to build extension from Go source files""" + def build_extension(self, ext): + + print("os.name ", os.name) + print("sys.platform ", sys.platform) + print("platform.system() ", platform.system()) + print("sysconfig.get_platform() ", sysconfig.get_platform()) + print("platform.machine() ", platform.machine()) + print("platform.architecture() ", platform.architecture()) + print("platform.python ", platform.python_version()) + print("BUILDING EXT FOR ", platform.release(), " ---- ",platform.machine()) + + ext_path = self.get_ext_fullpath(ext.name) + print("ext path = ", ext_path) + cmd = ['rm', '-rf', './uplink-c'] + out = call(cmd) + if out != 0: + raise CompileError('Go build failed') + cmd = ['git', 'clone', 'https://github.com/storj/uplink-c'] + out = call(cmd) + if out != 0: + raise CompileError('Go build failed') + os.chdir('./uplink-c') + cmd = ['/usr/bin/go/bin/go', 'build', '-buildmode=c-shared', '-o', '../'+ext_path]#, "."] + out = call(cmd) + os.chdir('..') + if out != 0: + raise CompileError('Go build failed') setuptools.setup( - name="uplink-python", - version="1.2.2.0", - author="Utropicmedia", - author_email="development@utropicmedia.com", - license='Apache Software License', - description="Python-native language binding for uplink to " - "communicate with the Storj network.", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/storj-thirdparty/uplink-python", - - packages=['uplink_python'], - install_requires=['wheel'], - include_package_data=True, - classifiers=[ - "Intended Audience :: Developers", - "Programming Language :: Python :: 3", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Topic :: Software Development :: Build Tools", + ext_modules=[ + Extension('libuplinkc', []) ], - python_requires='>=3.4', cmdclass={ - 'install': Install, + 'build_ext': build_go_ext, } ) diff --git a/test/test_data/helper.py b/test/test_data/helper.py index 0f6aac6..4a212f7 100644 --- a/test/test_data/helper.py +++ b/test/test_data/helper.py @@ -16,7 +16,7 @@ def __init__(self): self.api_key = file_handle.read() file_handle.close() - self.satellite = "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us-central-1.tardigrade.io:7777" + self.satellite = "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us1.storj.io:7777" self.encryption_phrase = "test" self.uplink = Uplink() diff --git a/test/test_data/object_test.py b/test/test_data/object_test.py index 82abc4e..54a6b38 100644 --- a/test/test_data/object_test.py +++ b/test/test_data/object_test.py @@ -3,7 +3,6 @@ import string import unittest -from uplink_python.errors import StorjException, ERROR_OBJECT_NOT_FOUND from .helper import TestPy diff --git a/uplink_python/access.py b/uplink_python/access.py index 51e9f19..4b39f16 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -7,7 +7,6 @@ _AccessStruct, _ProjectResult, _StringResult, _AccessResult, _EncryptionKeyResult,\ _EncryptionKeyStruct from uplink_python.project import Project -from uplink_python.errors import _storj_exception class Access: @@ -77,12 +76,7 @@ def derive_encryption_key(self, passphrase: str, salt: str): encryption_key_result = self.uplink.m_libuplink.uplink_derive_encryption_key(passphrase_ptr, salt_ptr, length_ptr) - # - # if error occurred - if bool(encryption_key_result.error): - raise _storj_exception(encryption_key_result.error.contents.code, - encryption_key_result.error.contents.message.decode("utf-8")) - return encryption_key_result.encryption_key + return self.uplink.unwrap_encryption_key_result(encryption_key_result) def override_encryption_key(self, bucket_name: str, prefix: str, encryption_key): """ @@ -115,8 +109,7 @@ def override_encryption_key(self, bucket_name: str, prefix: str, encryption_key) # # if error occurred if bool(error_result): - raise _storj_exception(error_result.contents.code, - error_result.contents.message.decode("utf-8")) + self.uplink.free_error_and_raise_exception(error_result) def open_project(self): """ @@ -134,12 +127,10 @@ def open_project(self): # # open project by calling the exported golang function project_result = self.uplink.m_libuplink.uplink_open_project(self.access) - # - # if error occurred - if bool(project_result.error): - raise _storj_exception(project_result.error.contents.code, - project_result.error.contents.message.decode("utf-8")) - return Project(project_result.project, self.uplink) + + _unwrapped_project = self.uplink.unwrap_project_result(project_result) + + return Project(_unwrapped_project, self.uplink) def config_open_project(self, config: Config): """ @@ -159,6 +150,7 @@ def config_open_project(self, config: Config): self.uplink.m_libuplink.uplink_config_open_project.argtypes =\ [_ConfigStruct, ctypes.POINTER(_AccessStruct)] self.uplink.m_libuplink.uplink_config_open_project.restype = _ProjectResult + self.uplink.m_libuplink.uplink_free_project_result.argtypes = [_ProjectResult] # # prepare the input for the function if config is None: @@ -168,12 +160,11 @@ def config_open_project(self, config: Config): # # open project by calling the exported golang function project_result = self.uplink.m_libuplink.uplink_config_open_project(config_obj, self.access) - # - # if error occurred - if bool(project_result.error): - raise _storj_exception(project_result.error.contents.code, - project_result.error.contents.message.decode("utf-8")) - return Project(project_result.project, self.uplink) + + _unwrapped_project = self.uplink.unwrap_project_result(project_result) + + return Project(_unwrapped_project, self.uplink) + def serialize(self): """ @@ -189,15 +180,16 @@ def serialize(self): # declare types of arguments and response of the corresponding golang function self.uplink.m_libuplink.uplink_access_serialize.argtypes = [ctypes.POINTER(_AccessStruct)] self.uplink.m_libuplink.uplink_access_serialize.restype = _StringResult + self.uplink.m_libuplink.uplink_free_string_result.argtypes = [_StringResult] # # get serialized access by calling the exported golang function string_result = self.uplink.m_libuplink.uplink_access_serialize(self.access) - # - # if error occurred - if bool(string_result.error): - raise _storj_exception(string_result.error.contents.code, - string_result.error.contents.message.decode("utf-8")) - return string_result.string.decode("utf-8") + + _unwrapped_string = self.uplink.unwrap_string_result(string_result) + + serialized_access = _unwrapped_string.decode("utf-8") + self.uplink.m_libuplink.uplink_free_string_result(string_result) + return serialized_access def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = None): """ @@ -227,6 +219,7 @@ def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = No ctypes.POINTER(_SharePrefixStruct), ctypes.c_size_t] self.uplink.m_libuplink.uplink_access_share.restype = _AccessResult + self.uplink.m_libuplink.uplink_free_access_result.argtypes = [_AccessResult] # # prepare the input for the function # check and create valid _PermissionStruct parameter @@ -251,9 +244,11 @@ def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = No # get shareable access by calling the exported golang function access_result = self.uplink.m_libuplink.uplink_access_share(self.access, permission_obj, shared_prefix_obj, array_size) - # - # if error occurred - if bool(access_result.error): - raise _storj_exception(access_result.error.contents.code, - access_result.error.contents.message.decode("utf-8")) - return Access(access_result.access, self.uplink) + + _unwrapped_access = self.uplink.unwrap_access_result(access_result) + return Access(_unwrapped_access, self.uplink) + + + def __del__(self): + """Free memory associated to this Access""" + self.uplink.free_access_struct(self.access) diff --git a/uplink_python/download.py b/uplink_python/download.py index 7e12650..abb671e 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -5,7 +5,6 @@ from uplink_python.module_def import _DownloadStruct, _ReadResult, _ProjectStruct,\ _ObjectResult, _Error -from uplink_python.errors import _storj_exception _WINDOWS = os.name == 'nt' COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 @@ -53,6 +52,8 @@ def __init__(self, download, uplink, project, bucket_name, storj_path): self.storj_path = storj_path self.uplink = uplink + + def read(self, size_to_read: int): """ function downloads up to len size_to_read bytes from the object's data stream. @@ -72,6 +73,7 @@ def read(self, size_to_read: int): ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t] self.uplink.m_libuplink.uplink_download_read.restype = _ReadResult + self.uplink.m_libuplink.uplink_free_read_result.argtypes = [_ReadResult] # # prepare the inputs for the function data_size = ctypes.c_int32(size_to_read) @@ -83,20 +85,20 @@ def read(self, size_to_read: int): # read data from Storj by calling the exported golang function read_result = self.uplink.m_libuplink.uplink_download_read(self.download, data_to_write_ptr, size_to_read) - # - # if error occurred - if bool(read_result.error): - raise _storj_exception(read_result.error.contents.code, - read_result.error.contents.message.decode("utf-8")) + + bytes_read = self.uplink.unwrap_read_result(read_result) data_read = bytes() - if int(read_result.bytes_read) != 0: + if bytes_read != 0: # # -------------------------------------------- # data conversion to type python readable form # conversion of LP_c_ubyte to python readable data variable data_read = ctypes.string_at(data_to_write_ptr, int(read_result.bytes_read)) - return data_read, int(read_result.bytes_read) + + self.uplink.m_libuplink.uplink_free_read_result(read_result) + + return data_read, bytes_read def read_file(self, file_handle, buffer_size: int = 0): """ @@ -120,8 +122,7 @@ def read_file(self, file_handle, buffer_size: int = 0): if not buffer_size: buffer_size = COPY_BUFSIZE file_size = self.file_size() - if buffer_size > file_size: - buffer_size = file_size + buffer_size = min(buffer_size, file_size) while file_size: buf, bytes_read = self.read(buffer_size) if buf: @@ -141,16 +142,18 @@ def file_size(self): self.uplink.m_libuplink.uplink_stat_object.argtypes = [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p, ctypes.c_char_p] self.uplink.m_libuplink.uplink_stat_object.restype = _ObjectResult + self.uplink.m_libuplink.uplink_free_object_result.argtypes = [_ObjectResult] # # get object information by calling the exported golang function object_result = self.uplink.m_libuplink.uplink_stat_object(self.project, self.bucket_name, self.storj_path) - # if error occurred - if bool(object_result.error): - raise _storj_exception(object_result.error.contents.code, - object_result.error.contents.message.decode("utf-8")) - # find object size - return int(object_result.object.contents.system.content_length) + + _object = self.uplink.unwrap_object_result(object_result) + + file_size = int(_object.contents.system.content_length) + + self.uplink.m_libuplink.uplink_free_object_result(object_result) + return file_size def close(self): """ @@ -170,8 +173,7 @@ def close(self): # # if error occurred if bool(error): - raise _storj_exception(error.contents.code, - error.contents.message.decode("utf-8")) + self.uplink.free_error_and_raise_exception(error) def info(self): """ @@ -188,9 +190,13 @@ def info(self): # # get last download info by calling the exported golang function object_result = self.uplink.m_libuplink.uplink_download_info(self.download) - # - # if error occurred - if bool(object_result.error): - raise _storj_exception(object_result.error.contents.code, - object_result.error.contents.message.decode("utf-8")) - return self.uplink.object_from_result(object_result.object) + + _unwrapped_object = self.uplink.unwrap_object_result(object_result) + _object = self.uplink.object_from_result(_unwrapped_object) + + self.uplink.m_libuplink.uplink_free_object_result(object_result) + + return _object + + def __del__(self): + self.uplink.free_download_struct(self.download) diff --git a/uplink_python/hello_storj.py b/uplink_python/hello_storj.py index c691e5c..2a7f0ed 100644 --- a/uplink_python/hello_storj.py +++ b/uplink_python/hello_storj.py @@ -3,9 +3,9 @@ from datetime import datetime -from .errors import StorjException, BucketNotEmptyError, BucketNotFoundError -from .module_classes import ListObjectsOptions, Permission, SharePrefix -from .uplink import Uplink +from uplink_python.errors import StorjException, BucketNotEmptyError, BucketNotFoundError +from uplink_python.module_classes import ListObjectsOptions, Permission, SharePrefix +from uplink_python.uplink import Uplink if __name__ == "__main__": diff --git a/uplink_python/project.py b/uplink_python/project.py index 1e6b267..0a78743 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -76,18 +76,20 @@ def create_bucket(self, bucket_name: str): self.uplink.m_libuplink.uplink_create_bucket.argtypes = [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p] self.uplink.m_libuplink.uplink_create_bucket.restype = _BucketResult + self.uplink.m_libuplink.uplink_free_bucket.argtypes = [_BucketResult] # # prepare the input for the function bucket_name_ptr = ctypes.c_char_p(bucket_name.encode('utf-8')) # create bucket by calling the exported golang function bucket_result = self.uplink.m_libuplink.uplink_create_bucket(self.project, bucket_name_ptr) - # - # if error occurred - if bool(bucket_result.error): - raise _storj_exception(bucket_result.error.contents.code, - bucket_result.error.contents.message.decode("utf-8")) - return self.uplink.bucket_from_result(bucket_result.bucket) + + _unwrapped_bucket = self.uplink.unwrap_bucket_result(bucket_result) + bucket = self.uplink.bucket_from_result(_unwrapped_bucket) + + self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) + + return bucket def ensure_bucket(self, bucket_name: str): """ @@ -115,12 +117,8 @@ def ensure_bucket(self, bucket_name: str): # open bucket if doesn't exist by calling the exported golang function bucket_result = self.uplink.m_libuplink.uplink_ensure_bucket(self.project, bucket_name_ptr) - # - # if error occurred - if bool(bucket_result.error): - raise _storj_exception(bucket_result.error.contents.code, - bucket_result.error.contents.message.decode("utf-8")) - return self.uplink.bucket_from_result(bucket_result.bucket) + + return self.check_bucket_result(bucket_result) def stat_bucket(self, bucket_name: str): """ @@ -146,11 +144,19 @@ def stat_bucket(self, bucket_name: str): # get bucket information by calling the exported golang function bucket_result = self.uplink.m_libuplink.uplink_stat_bucket(self.project, bucket_name_ptr) - # - # if error occurred + + return self.check_bucket_result(bucket_result) + + def check_bucket_result(self, bucket_result): if bool(bucket_result.error): - raise _storj_exception(bucket_result.error.contents.code, - bucket_result.error.contents.message.decode("utf-8")) + error_code = bucket_result.error.contents.code + error_msg = bucket_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_bucket_result.argtypes = [_BucketResult] + self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) + + raise _storj_exception(error_code, error_msg) + return self.uplink.bucket_from_result(bucket_result.bucket) def list_buckets(self, list_bucket_options: ListBucketsOptions = None): @@ -172,11 +178,15 @@ def list_buckets(self, list_bucket_options: ListBucketsOptions = None): [ctypes.POINTER(_ProjectStruct), ctypes.POINTER(_ListBucketsOptionsStruct)] self.uplink.m_libuplink.uplink_list_buckets.restype =\ ctypes.POINTER(_BucketIterator) + self.uplink.m_libuplink.uplink_free_bucket_iterator.argtypes=\ + [ctypes.POINTER(_BucketIterator)] # self.uplink.m_libuplink.uplink_bucket_iterator_item.argtypes =\ [ctypes.POINTER(_BucketIterator)] self.uplink.m_libuplink.uplink_bucket_iterator_item.restype =\ ctypes.POINTER(_BucketStruct) + self.uplink.m_libuplink.uplink_free_bucket.argtypes =\ + [ctypes.POINTER(_BucketStruct)] # self.uplink.m_libuplink.uplink_bucket_iterator_err.argtypes =\ [ctypes.POINTER(_BucketIterator)] @@ -200,13 +210,12 @@ def list_buckets(self, list_bucket_options: ListBucketsOptions = None): bucket_iterator_err = self.uplink.m_libuplink.uplink_bucket_iterator_err(bucket_iterator) if bool(bucket_iterator_err): - raise _storj_exception(bucket_iterator_err.contents.code, - bucket_iterator_err.contents.message.decode("utf-8")) - + self.uplink.free_error_and_raise_exception(bucket_iterator_err) bucket_list = list() while self.uplink.m_libuplink.uplink_bucket_iterator_next(bucket_iterator): bucket = self.uplink.m_libuplink.uplink_bucket_iterator_item(bucket_iterator) bucket_list.append(self.uplink.bucket_from_result(bucket)) + self.uplink.m_libuplink.uplink_free_bucket(bucket) return bucket_list @@ -230,6 +239,7 @@ def delete_bucket(self, bucket_name: str): self.uplink.m_libuplink.uplink_delete_bucket.argtypes = [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p] self.uplink.m_libuplink.uplink_delete_bucket.restype = _BucketResult + self.uplink.m_libuplink.uplink_free_bucket_result.argtypes = [_BucketResult] # # prepare the input for the function bucket_name_ptr = ctypes.c_char_p(bucket_name.encode('utf-8')) @@ -239,9 +249,16 @@ def delete_bucket(self, bucket_name: str): # # if error occurred if bool(bucket_result.error): - raise _storj_exception(bucket_result.error.contents.code, - bucket_result.error.contents.message.decode("utf-8")) - return self.uplink.bucket_from_result(bucket_result.bucket) + error_code = bucket_result.error.contents.code + error_msg = bucket_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) + + raise _storj_exception(error_code, error_msg) + + bucket = self.uplink.bucket_from_result(bucket_result.bucket) + + return bucket def stat_object(self, bucket_name: str, storj_path: str): """ @@ -262,6 +279,7 @@ def stat_object(self, bucket_name: str, storj_path: str): self.uplink.m_libuplink.uplink_stat_object.argtypes = [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p, ctypes.c_char_p] self.uplink.m_libuplink.uplink_stat_object.restype = _ObjectResult + self.uplink.m_libuplink.uplink_free_object_result.argtypes = [_ObjectResult] # # prepare the input for the function bucket_name_ptr = ctypes.c_char_p(bucket_name.encode('utf-8')) @@ -273,9 +291,16 @@ def stat_object(self, bucket_name: str, storj_path: str): # # if error occurred if bool(object_result.error): - raise _storj_exception(object_result.error.contents.code, - object_result.error.contents.message.decode("utf-8")) - return self.uplink.object_from_result(object_result.object) + error_code = object_result.error.contents.code + error_msg = object_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_object_result(object_result) + + raise _storj_exception(error_code, error_msg) + + _object = self.uplink.object_from_result(object_result.object) + + return _object def list_objects(self, bucket_name: str, list_object_options: ListObjectsOptions = None): """ @@ -298,11 +323,15 @@ def list_objects(self, bucket_name: str, list_object_options: ListObjectsOptions ctypes.POINTER(_ListObjectsOptionsStruct)] self.uplink.m_libuplink.uplink_list_objects.restype =\ ctypes.POINTER(_ObjectIterator) + self.uplink.m_libuplink.uplink_free_object_iterator.argtypes =\ + [ctypes.POINTER(_ObjectIterator)] # self.uplink.m_libuplink.uplink_object_iterator_item.argtypes =\ [ctypes.POINTER(_ObjectIterator)] self.uplink.m_libuplink.uplink_object_iterator_item.restype =\ ctypes.POINTER(_ObjectStruct) + self.uplink.m_libuplink.uplink_free_object.argtypes =\ + [ctypes.POINTER(_ObjectStruct)] # self.uplink.m_libuplink.uplink_object_iterator_err.argtypes =\ [ctypes.POINTER(_ObjectIterator)] @@ -330,10 +359,13 @@ def list_objects(self, bucket_name: str, list_object_options: ListObjectsOptions raise _storj_exception(object_iterator_err.contents.code, object_iterator_err.contents.message.decode("utf-8")) - object_list = list() + object_list = [] while self.uplink.m_libuplink.uplink_object_iterator_next(object_iterator): object_ = self.uplink.m_libuplink.uplink_object_iterator_item(object_iterator) object_list.append(self.uplink.object_from_result(object_)) + self.uplink.m_libuplink.uplink_free_object(object_) + + self.uplink.m_libuplink.uplink_free_object_iterator(object_iterator) return object_list def delete_object(self, bucket_name: str, storj_path: str): @@ -355,6 +387,7 @@ def delete_object(self, bucket_name: str, storj_path: str): self.uplink.m_libuplink.uplink_delete_object.argtypes = [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p, ctypes.c_char_p] self.uplink.m_libuplink.uplink_delete_object.restype = _ObjectResult + self.uplink.m_libuplink.uplink_free_object_result.argtypes = [_ObjectResult] # # prepare the input for the function bucket_name_ptr = ctypes.c_char_p(bucket_name.encode('utf-8')) @@ -363,12 +396,8 @@ def delete_object(self, bucket_name: str, storj_path: str): # delete object by calling the exported golang function object_result = self.uplink.m_libuplink.uplink_delete_object(self.project, bucket_name_ptr, storj_path_ptr) - # - # if error occurred - if bool(object_result.error): - raise _storj_exception(object_result.error.contents.code, - object_result.error.contents.message.decode("utf-8")) - return self.uplink.object_from_result(object_result.object) + + return self.uplink.unwrap_object_result(object_result) def close(self): """ @@ -388,8 +417,7 @@ def close(self): # # if error occurred if bool(error): - raise _storj_exception(error.contents.code, - error.contents.message.decode("utf-8")) + self.uplink.free_error_and_raise_exception(error) def upload_object(self, bucket_name: str, storj_path: str, upload_options: UploadOptions = None): @@ -412,6 +440,7 @@ def upload_object(self, bucket_name: str, storj_path: str, [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p, ctypes.c_char_p, ctypes.POINTER(_UploadOptionsStruct)] self.uplink.m_libuplink.uplink_upload_object.restype = _UploadResult + self.uplink.m_libuplink.uplink_free_upload_result.argtypes = [_UploadResult] # # prepare the input for the function if upload_options is None: @@ -426,12 +455,11 @@ def upload_object(self, bucket_name: str, storj_path: str, upload_result = self.uplink.m_libuplink.uplink_upload_object(self.project, bucket_name_ptr, storj_path_ptr, upload_options_obj) - # - # if error occurred - if bool(upload_result.error): - raise _storj_exception(upload_result.error.contents.code, - upload_result.error.contents.message.decode("utf-8")) - return Upload(upload_result.upload, self.uplink) + + _upload_result = self.uplink.unwrap_upload_object_result(upload_result) + + # _upload_result will be freed when committing or aborting upload. + return Upload(_upload_result, self.uplink) def download_object(self, bucket_name: str, storj_path: str, download_options: DownloadOptions = None): @@ -454,6 +482,7 @@ def download_object(self, bucket_name: str, storj_path: str, [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p, ctypes.c_char_p, ctypes.POINTER(_DownloadOptionsStruct)] self.uplink.m_libuplink.uplink_download_object.restype = _DownloadResult + self.uplink.m_libuplink.uplink_free_download_result.argtypes = [_DownloadResult] # # prepare the input for the function if download_options is None: @@ -472,7 +501,15 @@ def download_object(self, bucket_name: str, storj_path: str, # # if error occurred if bool(download_result.error): - raise _storj_exception(download_result.error.contents.code, - download_result.error.contents.message.decode("utf-8")) + error_code = download_result.error.contents.code + error_msg = download_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_download_result(download_result) + + raise _storj_exception(error_code, error_msg) + return Download(download_result.download, self.uplink, self.project, bucket_name_ptr, storj_path_ptr) + + def __del__(self): + self.uplink.free_project_struct(self.project) diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index 7836090..7679c5d 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -7,7 +7,8 @@ from uplink_python.access import Access from uplink_python.errors import _storj_exception, LibUplinkSoError -from uplink_python.module_def import _AccessResult, _ConfigStruct +from uplink_python.module_def import _AccessResult, _ConfigStruct, _DownloadResult, _ProjectResult, \ + _UploadResult, _Error from uplink_python.module_classes import Config, Bucket, Object, SystemMetadata, \ CustomMetadataEntry, CustomMetadata @@ -48,6 +49,7 @@ def __init__(self): self.m_libuplink = ctypes.CDLL(so_path) else: raise LibUplinkSoError + Uplink.__instance = self else: self.m_libuplink = Uplink.__instance.m_libuplink @@ -61,7 +63,7 @@ def object_from_result(cls, object_): content_length=object_.contents.system.content_length) array_size = object_.contents.custom.count - entries = list() + entries = [] for i in range(array_size): if bool(object_.contents.custom.entries[i]): entries_obj = object_.contents.custom.entries[i] @@ -126,8 +128,14 @@ def request_access_with_passphrase(self, satellite: str, api_key: str, passphras # # if error occurred if bool(access_result.error): - raise _storj_exception(access_result.error.contents.code, - access_result.error.contents.message.decode("utf-8")) + error_code = access_result.error.contents.code + error_msg = access_result.error.contents.message.decode("utf-8") + + self.m_libuplink.uplink_free_access_result.argtypes = [_AccessResult] + self.m_libuplink.uplink_free_access_result(access_result) + + raise _storj_exception(error_code, error_msg) + return Access(access_result.access, self) def config_request_access_with_passphrase(self, config: Config, satellite: str, api_key: str, @@ -177,12 +185,10 @@ def config_request_access_with_passphrase(self, config: Config, satellite: str, satellite_ptr, api_key_ptr, phrase_ptr) - # - # if error occurred - if bool(access_result.error): - raise _storj_exception(access_result.error.contents.code, - access_result.error.contents.message.decode("utf-8")) - return Access(access_result.access, self) + + _unwrapped_access = self.m_libuplink.unwrap_access_result(access_result) + + return Access(_unwrapped_access, self) def parse_access(self, serialized_access: str): """ @@ -211,9 +217,231 @@ def parse_access(self, serialized_access: str): # get parsed access by calling the exported golang function access_result = self.m_libuplink.uplink_parse_access(serialized_access_ptr) - # - # if error occurred - if bool(access_result.error): - raise _storj_exception(access_result.error.contents.code, - access_result.error.contents.message.decode("utf-8")) - return Access(access_result.access, self) + + _unwrapped_access = self.unwrap_access_result(access_result) + + return Access(_unwrapped_access, self) + + + def free_error_and_raise_exception(self, err ): + """ free libuplinkc error and raise corresponding _storj_exception """ + error_code = err.contents.code + error_msg = err.contents.message.decode("utf-8") + + self.m_libuplink.uplink_free_error.argtypes = [ctypes.POINTER(_Error)] + self.m_libuplink.uplink_free_error(err) + + raise _storj_exception(error_code, error_msg) + + def unwrap_libuplink_result(self, result, finalizer, attribute_name): + ''' unwrap libuplink result - raise exception if error occured''' + if bool(result.error): + error_code = result.error.contents.code + error_msg = result.error.contents.message.decode("utf-8") + finalizer(result) + raise _storj_exception(error_code, error_msg) + + result = getattr(result, attribute_name) + return result + + def unwrap_access_result(self, access_result): + """ + unwrap access result + + Parameters + ---------- + access_result : _AccessResult + + Returns + ------- + ctypes.POINTER(_AccessStruct) + """ + return self.unwrap_libuplink_result( + access_result, self.m_libuplink.uplink_free_access_result, 'access') + + def unwrap_bucket_result(self, bucket_result): + """ + unwrap bucket result + + Parameters + ---------- + bucket_result : _BucketResult + + Returns + ------- + ctypes.POINTER(_BucketStruct) + """ + return self.unwrap_libuplink_result( + bucket_result, self.m_libuplink.uplink_free_bucket_result, 'bucket') + + def unwrap_encryption_key_result(self, encryption_key_result): + """ + unwrap encryption key result + + Parameters + ---------- + encryption_key_result : _EncryptionKeyResult + + Returns + ------- + ctypes.POINTER(_EncryptionKeyStruct) + """ + return self.unwrap_libuplink_result(encryption_key_result, + self.m_libuplink.uplink_free_encryption_key_result, 'encryption_key') + + def unwrap_object_result(self, object_result): + """ + unwrap object result + + Parameters + ---------- + object_result : _ObjectResult + + Returns + ------- + ctypes.POINTER(_ObjectStruct) + """ + return self.unwrap_libuplink_result( + object_result, self.m_libuplink.uplink_free_object_result, 'object') + + def unwrap_project_result(self, project_result): + """ + unwrap project result + + Parameters + ---------- + project_result : _ProjectResult + + Returns + ------- + ctypes.POINTER(_ProjectStruct) + """ + return self.unwrap_libuplink_result( + project_result, self.m_libuplink.uplink_free_project_result, 'project') + + def unwrap_read_result(self, read_result): + """ + unwrap read result + + Parameters + ---------- + read_result : _ReadResult + + Returns + ------- + ctypes.c_size_t + """ + return self.unwrap_libuplink_result( + read_result, self.m_libuplink.uplink_free_read_result, 'bytes_read') + + def unwrap_string_result(self, string_result): + """ + unwrap project result + + Parameters + ---------- + string_result : _StringResult + + Returns + ------- + ctypes.c_char_p + """ + return self.unwrap_libuplink_result( + string_result, self.m_libuplink.uplink_free_string_result, 'string') + + def unwrap_upload_object_result(self, upload_object_result): + """ + unwrap project result + + Parameters + ---------- + upload_object_result : _UploadResult + + Returns + ------- + ctypes.POINTER(_UploadStruct) + """ + return self.unwrap_libuplink_result( + upload_object_result, self.m_libuplink.uplink_free_upload_result, 'upload') + + def unwrap_upload_write_result(self, result_object): + """ + unwrap upload write result + + Parameters + ---------- + upload_write_result : _WriteResult + + Returns + ------- + ctypes.c_size_t + """ + return self.unwrap_libuplink_result( + result_object, self.m_libuplink.uplink_free_write_result, 'bytes_written') + + def free_access_struct(self, access_struct): + """ + free access result + + Parameters + ---------- + access_struct : _AccessStruct + + Returns + ------- + None + """ + _access_result = _AccessResult() + _access_result.upload = access_struct + self.m_libuplink.uplink_free_access_result(_access_result) + + def free_upload_struct(self, upload_struct): + """ + free upload struct + + Parameters + ---------- + upload_struct : _UploadStruct + + Returns + ------- + None + """ + _upload_result = _UploadResult() + _upload_result.upload = upload_struct + self.m_libuplink.uplink_free_upload_result(_upload_result) + + def free_download_struct(self, download_struct): + """ + free download struct + + Parameters + ---------- + download_struct : _DownloadStruct + + Returns + ------- + None + """ + _download_result = _DownloadResult() + _download_result.download = download_struct + self.m_libuplink.uplink_free_download_result(_download_result) + + def free_project_struct(self, project_struct): + """ + free project struct + + Parameters + ---------- + project_struct : _ProjectStruct + + Returns + ------- + None + """ + self.m_libuplink.uplink_free_project_result.argtypes = [_ProjectResult] + + _project_result = _ProjectResult() + _project_result.project = project_struct + self.m_libuplink.uplink_free_project_result(_project_result) + diff --git a/uplink_python/upload.py b/uplink_python/upload.py index 9b0e169..e0a336c 100644 --- a/uplink_python/upload.py +++ b/uplink_python/upload.py @@ -5,7 +5,6 @@ from uplink_python.module_classes import CustomMetadata from uplink_python.module_def import _UploadStruct, _WriteResult, _Error, _CustomMetadataStruct, _ObjectResult -from uplink_python.errors import _storj_exception _WINDOWS = os.name == 'nt' COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 @@ -65,6 +64,7 @@ def write(self, data_to_write: bytes, size_to_write: int): ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t] self.uplink.m_libuplink.uplink_upload_write.restype = _WriteResult + self.uplink.m_libuplink.uplink_free_write_result.argtypes = [_WriteResult] # # prepare the inputs for the function # -------------------------------------------- @@ -80,12 +80,8 @@ def write(self, data_to_write: bytes, size_to_write: int): # upload data by calling the exported golang function write_result = self.uplink.m_libuplink.uplink_upload_write(self.upload, data_to_write_ptr, size_to_write_obj) - # - # if error occurred - if bool(write_result.error): - _storj_exception(write_result.error.contents.code, - write_result.error.contents.message.decode("utf-8")) - return int(write_result.bytes_written) + + return self.uplink.unwrap_upload_write_result(write_result) def write_file(self, file_handle, buffer_size: int = 0): """ @@ -131,11 +127,11 @@ def commit(self): # upload commit by calling the exported golang function error = self.uplink.m_libuplink.uplink_upload_commit(self.upload) + # # if error occurred if bool(error): - raise _storj_exception(error.contents.code, - error.contents.message.decode("utf-8")) + self.uplink.free_error_and_raise_exception(error) def abort(self): """ @@ -155,9 +151,10 @@ def abort(self): error = self.uplink.m_libuplink.uplink_upload_abort(self.upload) # # if error occurred + self.uplink.free_upload_struct(self.upload) if bool(error): - raise _storj_exception(error.contents.code, - error.contents.message.decode("utf-8")) + self.uplink.free_error_and_raise_exception(error) + def set_custom_metadata(self, custom_metadata: CustomMetadata = None): """ @@ -185,11 +182,9 @@ def set_custom_metadata(self, custom_metadata: CustomMetadata = None): # # set custom metadata to upload by calling the exported golang function error = self.uplink.m_libuplink.uplink_upload_set_custom_metadata(self.upload, custom_metadata_obj) - # - # if error occurred + if bool(error): - raise _storj_exception(error.contents.code, - error.contents.message.decode("utf-8")) + self.uplink.free_error_and_raise_exception(error) def info(self): """ @@ -203,12 +198,15 @@ def info(self): # declare types of arguments and response of the corresponding golang function self.uplink.m_libuplink.uplink_upload_info.argtypes = [ctypes.POINTER(_UploadStruct)] self.uplink.m_libuplink.uplink_upload_info.restype = _ObjectResult + self.uplink.m_libuplink.uplink_free_object_result.argtypes = [_ObjectResult] # # get last upload info by calling the exported golang function object_result = self.uplink.m_libuplink.uplink_upload_info(self.upload) - # - # if error occurred - if bool(object_result.error): - raise _storj_exception(object_result.error.contents.code, - object_result.error.contents.message.decode("utf-8")) - return self.uplink.object_from_result(object_result.object) + + _unwrapped_object = self.uplink.unwrap_object_result(object_result) + info = self.uplink.object_from_result(_unwrapped_object) + self.uplink.m_libuplink.uplink_free_object(_unwrapped_object) + return info + + def __del__(self): + self.uplink.free_upload_struct(self.upload)