diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8f349c66..7eca144b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,18 +5,18 @@ on: [push] jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: - python-version: [3.6, 3.7, 3.8, '3.10', '3.11', '3.12', pypy2, pypy3] + python-version: ['3.8', '3.9','3.10', '3.11', '3.12'] env: SOLVEBIO_API_HOST: ${{ secrets.QUARTZBIO_API_HOST }} SOLVEBIO_API_KEY: ${{ secrets.QUARTZBIO_API_KEY }} steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Display Python version @@ -38,35 +38,3 @@ jobs: python -m pytest recipes/tests/test_recipes_sync.py python -m pytest solvebio/test/test_object.py python -m flake8 solvebio - build_py27: - runs-on: ubuntu-20.04 - env: - SOLVEBIO_API_HOST: ${{ secrets.QUARTZBIO_API_HOST }} - SOLVEBIO_API_KEY: ${{ secrets.QUARTZBIO_API_KEY }} - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Setup Python 2.7 - run: | - sudo apt install python2 - sudo apt-get install -y curl python2 - sudo curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py - sudo python2.7 get-pip.py - - name: Display Python version - run: python2.7 -c "import sys; print(sys.version)" - - name: Export pythonpath - run: | - export PYTHONPATH=$PYTHONPATH:$(pwd) - - name: Install Tox and any other packages - run: | - python2.7 -m pip install -U wheel --user - python2.7 -m pip install setuptools - python2.7 -m pip install flake8 pytest - - name: Install dependencies - run: | - python2.7 -m pip install -r requirements-dev.txt - python2.7 -m pip install XlsxWriter===0.9.3 - - name: Scripts - run: | - python2.7 -m pytest recipes/tests/test_recipes_sync.py - python2.7 -m flake8 solvebio diff --git a/solvebio/cli/data.py b/solvebio/cli/data.py index b729eca5..b16d9e6b 100644 --- a/solvebio/cli/data.py +++ b/solvebio/cli/data.py @@ -52,7 +52,7 @@ def should_exclude(path, exclude_paths, dry_run=False, print_logs=True): # An exclude path may be a directory, strip trailing slash and add /* # if not already there. if not exclude_path.endswith("/*") and fnmatch( - path, exclude_path.rstrip("/") + "/*" + path, exclude_path.rstrip("/") + "/*" ): if print_logs: print( @@ -119,16 +119,16 @@ def _folder_exists(folder_full_path, remote_folders_existing, follow_shortcuts): def _upload_folder( - domain, - vault, - base_remote_path, - base_local_path, - local_start, - exclude_paths=None, - dry_run=False, - num_processes=1, - archive_folder=None, - follow_shortcuts=False + domain, + vault, + base_remote_path, + base_local_path, + local_start, + exclude_paths=None, + dry_run=False, + num_processes=1, + archive_folder=None, + follow_shortcuts=False ): all_folders = [] all_files = [] @@ -247,7 +247,7 @@ def _create_file_job(args): client = SolveClient(*client_auth) remote_parent = None - try: + try: remote_parent = Object.get_by_full_path( remote_folder_full_path, assert_type="folder", @@ -282,12 +282,13 @@ def _create_file_job(args): except Exception as e: return e + def _object_exists(remote_parent, local_path, _client): if remote_parent is None: return False full_path, path_dict = Object.validate_full_path( - os.path.join('{}:{}'.format(remote_parent.vault.full_path, remote_parent.path), - os.path.basename(local_path)), client=_client) + os.path.join('{}:{}'.format(remote_parent.vault.full_path, remote_parent.path), + os.path.basename(local_path)), client=_client) try: obj = Object.get_by_full_path(full_path, client=_client) if not obj.is_file: @@ -303,6 +304,7 @@ def _object_exists(remote_parent, local_path, _client): except NotFoundError: return False + def _create_template_from_file(template_file, dry_run=False): mode = "r" fopen = open @@ -661,15 +663,15 @@ def download(args): def _download( - full_path, - local_folder_path, - dry_run=False, - recursive=False, - excludes=[], - includes=[], - delete=False, - follow_shortcuts=False, - num_processes=None, + full_path, + local_folder_path, + dry_run=False, + recursive=False, + excludes=[], + includes=[], + delete=False, + follow_shortcuts=False, + num_processes=None, ): """ Given a folder or file, download all the files contained @@ -747,16 +749,15 @@ def _download( def _download_recursive( - full_path, - local_folder_path, - dry_run=False, - excludes=[], - includes=[], - delete=False, - follow_shortcuts=False, - num_processes=None, + full_path, + local_folder_path, + dry_run=False, + excludes=[], + includes=[], + delete=False, + follow_shortcuts=False, + num_processes=None, ): - if "**" in full_path: raise Exception( "Paths containing ** are not compatible with the --recursive flag." @@ -812,7 +813,7 @@ def _download_recursive( # Skip over files that are excluded (not recovered by include) if should_exclude( - local_path, excludes, print_logs=False + local_path, excludes, print_logs=False ) and not should_exclude(local_path, includes, print_logs=False): continue @@ -883,7 +884,7 @@ def _download_worker(file_info): if num_processes <= 0: num_processes = os.cpu_count() - print("[Warning] num-processes cannot be less than 1. Defaulting to CPU count: ({})". format(num_processes)) + print("[Warning] num-processes cannot be less than 1. Defaulting to CPU count: ({})".format(num_processes)) print("Downloading in parallel with {} processes.".format(num_processes))