Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 4 additions & 36 deletions .github/workflows/python-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,18 @@ on: [push]

jobs:
build:
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: [3.6, 3.7, 3.8, '3.10', '3.11', '3.12', pypy2, pypy3]
python-version: ['3.8', '3.9','3.10', '3.11', '3.12']
env:
SOLVEBIO_API_HOST: ${{ secrets.QUARTZBIO_API_HOST }}
SOLVEBIO_API_KEY: ${{ secrets.QUARTZBIO_API_KEY }}
steps:
- name: Checkout repository
uses: actions/checkout@v2
uses: actions/checkout@v4
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Display Python version
Expand All @@ -38,35 +38,3 @@ jobs:
python -m pytest recipes/tests/test_recipes_sync.py
python -m pytest solvebio/test/test_object.py
python -m flake8 solvebio
build_py27:
runs-on: ubuntu-20.04
env:
SOLVEBIO_API_HOST: ${{ secrets.QUARTZBIO_API_HOST }}
SOLVEBIO_API_KEY: ${{ secrets.QUARTZBIO_API_KEY }}
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Setup Python 2.7
run: |
sudo apt install python2
sudo apt-get install -y curl python2
sudo curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
sudo python2.7 get-pip.py
- name: Display Python version
run: python2.7 -c "import sys; print(sys.version)"
- name: Export pythonpath
run: |
export PYTHONPATH=$PYTHONPATH:$(pwd)
- name: Install Tox and any other packages
run: |
python2.7 -m pip install -U wheel --user
python2.7 -m pip install setuptools
python2.7 -m pip install flake8 pytest
- name: Install dependencies
run: |
python2.7 -m pip install -r requirements-dev.txt
python2.7 -m pip install XlsxWriter===0.9.3
- name: Scripts
run: |
python2.7 -m pytest recipes/tests/test_recipes_sync.py
python2.7 -m flake8 solvebio
69 changes: 35 additions & 34 deletions solvebio/cli/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def should_exclude(path, exclude_paths, dry_run=False, print_logs=True):
# An exclude path may be a directory, strip trailing slash and add /*
# if not already there.
if not exclude_path.endswith("/*") and fnmatch(
path, exclude_path.rstrip("/") + "/*"
path, exclude_path.rstrip("/") + "/*"
):
if print_logs:
print(
Expand Down Expand Up @@ -119,16 +119,16 @@ def _folder_exists(folder_full_path, remote_folders_existing, follow_shortcuts):


def _upload_folder(
domain,
vault,
base_remote_path,
base_local_path,
local_start,
exclude_paths=None,
dry_run=False,
num_processes=1,
archive_folder=None,
follow_shortcuts=False
domain,
vault,
base_remote_path,
base_local_path,
local_start,
exclude_paths=None,
dry_run=False,
num_processes=1,
archive_folder=None,
follow_shortcuts=False
):
all_folders = []
all_files = []
Expand Down Expand Up @@ -247,7 +247,7 @@ def _create_file_job(args):
client = SolveClient(*client_auth)

remote_parent = None
try:
try:
remote_parent = Object.get_by_full_path(
remote_folder_full_path,
assert_type="folder",
Expand Down Expand Up @@ -282,12 +282,13 @@ def _create_file_job(args):
except Exception as e:
return e


def _object_exists(remote_parent, local_path, _client):
if remote_parent is None:
return False
full_path, path_dict = Object.validate_full_path(
os.path.join('{}:{}'.format(remote_parent.vault.full_path, remote_parent.path),
os.path.basename(local_path)), client=_client)
os.path.join('{}:{}'.format(remote_parent.vault.full_path, remote_parent.path),
os.path.basename(local_path)), client=_client)
try:
obj = Object.get_by_full_path(full_path, client=_client)
if not obj.is_file:
Expand All @@ -303,6 +304,7 @@ def _object_exists(remote_parent, local_path, _client):
except NotFoundError:
return False


def _create_template_from_file(template_file, dry_run=False):
mode = "r"
fopen = open
Expand Down Expand Up @@ -661,15 +663,15 @@ def download(args):


def _download(
full_path,
local_folder_path,
dry_run=False,
recursive=False,
excludes=[],
includes=[],
delete=False,
follow_shortcuts=False,
num_processes=None,
full_path,
local_folder_path,
dry_run=False,
recursive=False,
excludes=[],
includes=[],
delete=False,
follow_shortcuts=False,
num_processes=None,
):
"""
Given a folder or file, download all the files contained
Expand Down Expand Up @@ -747,16 +749,15 @@ def _download(


def _download_recursive(
full_path,
local_folder_path,
dry_run=False,
excludes=[],
includes=[],
delete=False,
follow_shortcuts=False,
num_processes=None,
full_path,
local_folder_path,
dry_run=False,
excludes=[],
includes=[],
delete=False,
follow_shortcuts=False,
num_processes=None,
):

if "**" in full_path:
raise Exception(
"Paths containing ** are not compatible with the --recursive flag."
Expand Down Expand Up @@ -812,7 +813,7 @@ def _download_recursive(

# Skip over files that are excluded (not recovered by include)
if should_exclude(
local_path, excludes, print_logs=False
local_path, excludes, print_logs=False
) and not should_exclude(local_path, includes, print_logs=False):
continue

Expand Down Expand Up @@ -883,7 +884,7 @@ def _download_worker(file_info):

if num_processes <= 0:
num_processes = os.cpu_count()
print("[Warning] num-processes cannot be less than 1. Defaulting to CPU count: ({})". format(num_processes))
print("[Warning] num-processes cannot be less than 1. Defaulting to CPU count: ({})".format(num_processes))

print("Downloading in parallel with {} processes.".format(num_processes))

Expand Down