[EN-3260] add Downloader, build.native-dependencies, ubuntu-18.04, test on PR

ttldtor · ttldtor · commit 1c8eb4d27f7c · 2021-08-30T15:40:04.000+03:00
diff --git a/.github/workflows/publishpackage.yml b/.github/workflows/publishpackage.yml
@@ -11,8 +11,8 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: [3.6, 3.7, 3.8]
+        os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest]
+        python-version: [3.6, 3.7, 3.8, 3.9]
 
     runs-on: ${{ matrix.os }}
 
@@ -34,12 +34,11 @@ jobs:
           poetry install --no-root
         shell: bash
       - name: Create tarball
-        if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7
+        if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-18.04'
         run: |
           task build -f sdist
         shell: bash
       - name: Create wheel
-        if: matrix.os == 'windows-latest' || matrix.os == 'macos-latest'
         run: |
           task build -f wheel
         shell: bash
diff --git a/.github/workflows/test_on_pr.yml b/.github/workflows/test_on_pr.yml
@@ -0,0 +1,76 @@
+# This workflow will check package installation and runs tests on PR
+name: Test on PR
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+jobs:
+  installation:
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest]
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install poetry
+      run: |
+        pip install poetry
+      shell: bash
+    - name: Set poetry env
+      run: |
+        poetry config virtualenvs.create false
+        poetry install --no-root --no-dev
+        pip install taskipy cython
+        task build -f sdist
+        pip uninstall --yes taskipy cython
+      shell: bash
+    - name: Install package artifact
+      run: |
+        pip install dist/dxfeed*
+        pip uninstall --yes dxfeed
+      shell: bash
+  tests:
+    needs: installation
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest]
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install poetry
+      run: |
+        pip install poetry
+      shell: bash
+    - name: Set poetry env
+      run: |
+        poetry config virtualenvs.create false
+        poetry install --no-root
+      shell: bash
+    - name: Run tests
+      run: |
+        task test
+      shell: bash
+    - name: Generate doc
+      run: |
+        task html_docs
+      shell: bash
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.7
diff --git a/.github/workflows/testdevelop.yml b/.github/workflows/testdevelop.yml
@@ -12,7 +12,7 @@ jobs:
   installation:
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest]
         python-version: [3.6, 3.7, 3.8, 3.9]
 
     runs-on: ${{ matrix.os }}
diff --git a/.github/workflows/testpackage.yml b/.github/workflows/testpackage.yml
@@ -13,8 +13,8 @@ jobs:
   installation:
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: [3.6, 3.7, 3.8]
+        os: [ubuntu-18.04, ubuntu-latest, windows-latest, macos-latest]
+        python-version: [3.6, 3.7, 3.8, 3.9]
 
     runs-on: ${{ matrix.os }}
 
diff --git a/build.py b/build.py
@@ -1,17 +1,96 @@
 import os
-import shutil
+import platform
 import struct
-
-from setuptools import Extension, find_packages
-from setuptools.dist import Distribution
-from pathlib import Path
 from io import BytesIO
-from zipfile import ZipFile
+from pathlib import Path
 from urllib.request import urlopen
-import platform
+from zipfile import ZipFile
+import shutil
+from setuptools import Extension, find_packages
+from setuptools.dist import Distribution
+import toml
+
+
+class Downloader(object):
+    def __init__(self, version: str, path_to_extract: Path,
+                 path_to_copy_includes: Path, path_to_copy_libs: Path):
+        self.__version = version
+        self.__path_to_extract = path_to_extract
+        self.__path_to_copy_includes = path_to_copy_includes
+        self.__path_to_copy_libs = path_to_copy_libs
+
+    def download(self) -> str:
+        __c_api_extracted_root_dir = self.__path_to_extract / f'dxfeed-c-api-{self.__version}-no-tls'
+        is_x64 = 8 * struct.calcsize("P") == 64
+
+        if platform.system() == 'Windows':
+            current_os = 'windows'
+        elif platform.system() == 'Darwin':
+            current_os = 'macosx'
+        else:
+            current_os = 'centos'  # Since centos uses an earlier version of libc
+
+        if (not os.path.exists(self.__path_to_extract)) or (not os.path.exists(__c_api_extracted_root_dir)):
+            url = f'https://github.com/dxFeed/dxfeed-c-api/releases/download/{self.__version}/dxfeed-c-api-' \
+                  f'{self.__version}-{current_os}-no-tls.zip '
+            print(f'Downloading the "{url}"')
+            resp = urlopen(url)
+            zipfile = ZipFile(BytesIO(resp.read()))
+            print(f'Extracting to "{self.__path_to_extract}"')
+            zipfile.extractall(self.__path_to_extract)
+
+        if __debug__:
+            # noinspection PyUnreachableCode
+            c_api_debug_suffix = 'd'
+        else:
+            c_api_debug_suffix = ''
+
+        if is_x64:
+            c_api_x64_suffix = '_64'
+        else:
+            c_api_x64_suffix = ''
+
+        __c_api_library_name = f'DXFeed{c_api_debug_suffix}{c_api_x64_suffix}'
+
+        if platform.system() == 'Windows':
+            if is_x64:
+                c_api_extracted_library_dir = __c_api_extracted_root_dir / 'bin' / 'x64'
+                c_api_library_file_name = f'{__c_api_library_name}.dll'
+                c_api_library_file_name2 = f'{__c_api_library_name}.lib'
+            else:
+                c_api_extracted_library_dir = __c_api_extracted_root_dir / 'bin' / 'x86'
+                c_api_library_file_name = f'{__c_api_library_name}.dll'
+                c_api_library_file_name2 = f'{__c_api_library_name}.lib'
+        elif platform.system() == 'Darwin':
+            if is_x64:
+                __c_api_extracted_root_dir = __c_api_extracted_root_dir / f'DXFeedAll-{self.__version}-x64-no-tls'
+                c_api_extracted_library_dir = __c_api_extracted_root_dir / 'bin' / 'x64'
+                c_api_library_file_name = f'lib{__c_api_library_name}.dylib'
+            else:
+                raise Exception('Unsupported platform')
+        else:
+            if is_x64:
+                __c_api_extracted_root_dir = __c_api_extracted_root_dir / f'DXFeedAll-{self.__version}-x64-no-tls'
+                c_api_extracted_library_dir = __c_api_extracted_root_dir / 'bin' / 'x64'
+                c_api_library_file_name = f'lib{__c_api_library_name}.so'
+            else:
+                raise Exception('Unsupported platform')
+
+        if not os.path.exists(self.__path_to_copy_includes):
+            shutil.copytree(__c_api_extracted_root_dir / 'include', self.__path_to_copy_includes)
+        if not os.path.exists(self.__path_to_copy_libs / c_api_library_file_name):
+            if not os.path.exists(self.__path_to_copy_libs):
+                os.makedirs(self.__path_to_copy_libs)
+            shutil.copyfile(c_api_extracted_library_dir / c_api_library_file_name,
+                            self.__path_to_copy_libs / c_api_library_file_name)
+        if platform.system() == 'Windows':
+            # noinspection PyUnboundLocalVariable
+            if not os.path.exists(self.__path_to_copy_libs / c_api_library_file_name2):
+                shutil.copyfile(c_api_extracted_library_dir / c_api_library_file_name2,
+                                self.__path_to_copy_libs / c_api_library_file_name2)
+
+        return __c_api_library_name
 
-DEFAULT_C_API_VERSION = '8.3.0'
-c_api_version = os.getenv('DXFEED_C_API_VERSION', DEFAULT_C_API_VERSION)
 
 try:
     from Cython.Build import cythonize
@@ -24,77 +103,18 @@
 
 root_path = Path(__file__).resolve().parent
 
-is_x64 = 8 * struct.calcsize("P") == 64
-
-if platform.system() == 'Windows':
-    current_os = 'windows'
-elif platform.system() == 'Darwin':
-    current_os = 'macosx'
-else:
-    current_os = 'centos'  # Since centos uses an earlier version of libc
+pyproject = toml.load(root_path / 'pyproject.toml')
+c_api_version = pyproject['build']['native-dependencies']['dxfeed_c_api']
+if c_api_version == 'env':
+    c_api_version = os.getenv('DXFEED_C_API_VERSION')
 
 c_api_root_dir = root_path / 'dxfeed' / 'dxfeed-c-api'
+path_to_extract = root_path / 'dxfeed' / 'tmp'
 c_api_include_dir = c_api_root_dir / 'include'
 c_api_bin_dir = root_path / 'dxfeed' / 'core'
-path_to_extract = root_path / 'dxfeed' / 'tmp'
-c_api_extracted_root_dir = path_to_extract / f'dxfeed-c-api-{c_api_version}-no-tls'
-
-if (not os.path.exists(path_to_extract)) or (not os.path.exists(c_api_extracted_root_dir)):
-    url = f'https://github.com/dxFeed/dxfeed-c-api/releases/download/{c_api_version}/dxfeed-c-api-{c_api_version}-' \
-          f'{current_os}-no-tls.zip '
-    print(f'Downloading the "{url}"')
-    resp = urlopen(url)
-    zipfile = ZipFile(BytesIO(resp.read()))
-    print(f'Extracting to "{path_to_extract}"')
-    zipfile.extractall(path_to_extract)
-
-if __debug__:
-    # noinspection PyUnreachableCode
-    c_api_debug_suffix = 'd'
-else:
-    c_api_debug_suffix = ''
-
-if is_x64:
-    c_api_x64_suffix = '_64'
-else:
-    c_api_x64_suffix = ''
 
-c_api_library_name = f'DXFeed{c_api_debug_suffix}{c_api_x64_suffix}'
-
-if platform.system() == 'Windows':
-    if is_x64:
-        c_api_extracted_library_dir = c_api_extracted_root_dir / 'bin' / 'x64'
-        c_api_library_file_name = f'{c_api_library_name}.dll'
-        c_api_library_file_name2 = f'{c_api_library_name}.lib'
-    else:
-        c_api_extracted_library_dir = c_api_extracted_root_dir / 'bin' / 'x86'
-        c_api_library_file_name = f'{c_api_library_name}.dll'
-        c_api_library_file_name2 = f'{c_api_library_name}.lib'
-elif platform.system() == 'Darwin':
-    if is_x64:
-        c_api_extracted_root_dir = c_api_extracted_root_dir / f'DXFeedAll-{c_api_version}-x64-no-tls'
-        c_api_extracted_library_dir = c_api_extracted_root_dir / 'bin' / 'x64'
-        c_api_library_file_name = f'lib{c_api_library_name}.dylib'
-    else:
-        raise Exception('Unsupported platform')
-else:
-    if is_x64:
-        c_api_extracted_root_dir = c_api_extracted_root_dir / f'DXFeedAll-{c_api_version}-x64-no-tls'
-        c_api_extracted_library_dir = c_api_extracted_root_dir / 'bin' / 'x64'
-        c_api_library_name = 'DXFeed_64'
-        c_api_library_file_name = f'lib{c_api_library_name}.so'
-    else:
-        raise Exception('Unsupported platform')
-
-if not os.path.exists(c_api_include_dir):
-    shutil.copytree(c_api_extracted_root_dir / 'include', c_api_include_dir)
-if not os.path.exists(c_api_bin_dir / c_api_library_file_name):
-    #    os.makedirs(c_api_bin_dir)
-    shutil.copyfile(c_api_extracted_library_dir / c_api_library_file_name, c_api_bin_dir / c_api_library_file_name)
-if platform.system() == 'Windows':
-    # noinspection PyUnboundLocalVariable
-    if not os.path.exists(c_api_bin_dir / c_api_library_file_name2):
-        shutil.copyfile(c_api_extracted_library_dir / c_api_library_file_name2, c_api_bin_dir / c_api_library_file_name2)
+downloader = Downloader(c_api_version, path_to_extract, c_api_include_dir, c_api_bin_dir)
+c_api_library_name = downloader.download()
 
 if platform.system() == 'Windows':
     runtime_library_dirs = None
@@ -152,7 +172,3 @@ def build_extensions():
     build_ext_cmd.ensure_finalized()
     build_ext_cmd.inplace = 1
     build_ext_cmd.run()
-
-    print(f'Removing the "{path_to_extract}"')
-    if os.path.exists(path_to_extract):
-        shutil.rmtree(path_to_extract)
diff --git a/clear.py b/clear.py
@@ -16,8 +16,5 @@
 if os.path.exists('dxfeed/tmp'):
     shutil.rmtree('dxfeed/tmp')
 
-if os.path.exists('dxfeed/dxfeed-c-api/bin'):
-    shutil.rmtree('dxfeed/dxfeed-c-api/bin')
-
 if os.path.exists('dxfeed/dxfeed-c-api/include'):
     shutil.rmtree('dxfeed/dxfeed-c-api/include')
diff --git a/dxfeed/core/utils/helpers.pyx b/dxfeed/core/utils/helpers.pyx
@@ -90,5 +90,5 @@ def get_include():
     """
     out_dir = list()
     out_dir.append(
-        str(os.path.realpath(Path(dxfeed.__file__).resolve().parent / '..' / 'libs' / 'dxfeed-c-api' / 'include')))
+        str(os.path.realpath(Path(dxfeed.__file__).resolve().parent / 'dxfeed-c-api' / 'include')))
     return out_dir
diff --git a/pyproject.toml b/pyproject.toml
@@ -12,6 +12,10 @@ classifiers = ['Development Status :: 4 - Beta',
                'Operating System :: MacOS',
                'Programming Language :: Cython',
                'Programming Language :: Python :: 3 :: Only',
+               'Programming Language :: Python :: 3.6',
+               'Programming Language :: Python :: 3.7',
+               'Programming Language :: Python :: 3.8',
+               'Programming Language :: Python :: 3.9',
                'Topic :: Office/Business :: Financial',
                'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
                'Natural Language :: English']
@@ -42,6 +46,7 @@ pytest = '^6.2.4'
 sphinx = '^4.1.2'
 sphinx_rtd_theme = '^0.5.2'
 pygments = '^2.10'
+toml = '^0.10.2'
 
 [tool.poetry.extras]
 docs = ['jupyter', 'cython']
@@ -56,5 +61,8 @@ post_build = 'task clear'
 post_test = 'task clear'
 
 [build-system]
-requires = ['poetry_core>=1.0.0', 'setuptools>=57.4.0', 'cython']
+requires = ['poetry_core>=1.0.0', 'setuptools>=57.4.0', 'cython', 'toml>=0.10.2']
 build-backend = 'poetry.core.masonry.api'
+
+[build.native-dependencies]
+dxfeed_c_api = '8.3.0'
