Retry mechanism for file upload

recipes/sync_recipe_utils.py

@@ -86,7 +86,7 @@ def export_recipes_to_yaml(recipes, yml_file):
         class RecipeDumper(yaml.Dumper):
             pass
 
-        class literal(unicode):
+        class literal(str):
             pass
 
         def _dict_representer(dumper, data):
@@ -100,17 +100,9 @@ def _literal_representer(dumper, data):
         RecipeDumper.add_representer(dict, _dict_representer)
         RecipeDumper.add_representer(literal, _literal_representer)
 
-        # Needed for python2,
-        # otherwise: 'item': !!python/unicode "some string" is dumped
-        if sys.version_info < (3,0):
-            def represent_unicode(dumper, data):
-                return dumper.represent_scalar(u'tag:yaml.org,2002:str', data)
-
-            RecipeDumper.add_representer(unicode, represent_unicode)
-
         yaml_recipes = []
         for r in recipes:
-            recipe_expression = literal(unicode(dict(r)['fields'][0]['expression']))
+            recipe_expression = literal(str(dict(r)['fields'][0]['expression']))
             dict(r)['fields'][0]['expression'] = recipe_expression
             recipe_details = {
                 "name": dict(r)['name'],

recipes/tests/test_recipes_sync.py

@@ -34,7 +34,6 @@ def retrieve(*args, **kwargs):
         return usr
     monkeypatch.setattr(sync_recipes.sb.User, "retrieve", retrieve)
 
-@pytest.mark.skipif(sys.version_info < (3,0), reason="requires python3")
 def test_sync_recipe(mock_dataset_template_retrieve,
                      mock_user_retrieve):
     with pytest.raises(SystemExit) as e:

requirements-dev.txt

@@ -1,15 +1,15 @@
 flake8
 mock
 requests[security]
-six
+urllib3>=1.26.0
 flask
 percy
 selenium
-dash==1.19.0
-dash_auth==1.4.1
-dash_core_components==1.15.0
-dash_html_components==1.1.2
-dash_renderer==1.9.0
+dash>=2.14.0
+dash_auth>=2.0.0
+dash_core_components>=2.0.0
+dash_html_components>=2.0.0
+dash_renderer>=1.9.1
 Werkzeug<=2.0.3
 solvebio==2.12.0
 pyyaml==5.3.1

setup.cfg

@@ -1,2 +1,2 @@
 [metadata]
-description-file = README.md
+description_file = README.md

setup.py

@@ -1,33 +1,18 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import, print_function
 from setuptools import setup, find_packages
 
 import sys
 import warnings
 
 VERSION = 'undefined'
-install_requires = ['six', 'pyprind']
+install_requires = ['pyprind', 'requests>=2.0.0', 'urllib3>=1.26.0']
 extra = {}
 
 with open('solvebio/version.py') as f:
     for row in f.readlines():
         if row.startswith('VERSION'):
             exec(row)
 
-if sys.version_info < (2, 6):
-    warnings.warn(
-        'Python 2.5 is no longer officially supported by SolveBio. '
-        'If you have any questions, please file an issue on GitHub or '
-        'contact us at support@solvebio.com.',
-        DeprecationWarning)
-    install_requires.append('requests >= 0.8.8, < 0.10.1')
-    install_requires.append('ssl')
-elif sys.version_info < (2, 7):
-    install_requires.append('ordereddict')
-else:
-    install_requires.append('requests>=2.0.0')
-
-
 # solvebio-recipes requires additional packages
 recipes_requires = [
     'pyyaml==5.3.1',
@@ -38,17 +23,6 @@
     "recipes": recipes_requires
 }
 
-# Adjustments for Python 2 vs 3
-if sys.version_info < (3, 0):
-    # Get simplejson if we don't already have json
-    try:
-        import json  # noqa
-    except ImportError:
-        install_requires.append('simplejson')
-
-    # solvebio-recipes only available in python3
-    extras_requires = {}
-
 with open('README.md') as f:
     long_description = f.read()
 

solvebio/__init__.py

@@ -7,11 +7,10 @@
 
 Have questions or comments? email us at: support@solvebio.com
 """
-from __future__ import absolute_import
-from __future__ import print_function
 __docformat__ = 'restructuredtext'
 
 import os as _os
+import errno
 import logging as _logging
 from typing import Literal
 from .help import open_help as _open_help
@@ -61,7 +60,7 @@ def _init_logging():
                 _os.makedirs(logdir)
             except OSError as err:
                 # Re-raise anything other than 'File exists'.
-                if err[1] != 'File exists':
+                if err.errno != errno.EEXIST:
                     raise err
 
         file_handler = _logging.FileHandler(logfile_path)
@@ -74,7 +73,6 @@ def _init_logging():
     try:
         base_logger.addHandler(_logging.NullHandler())
     except:
-        # supports Python < 2.7
         class NullHandler(_logging.Handler):
             def emit(self, record):
                 pass

solvebio/annotate.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
 
 from .client import client
 

solvebio/auth.py

@@ -1,9 +1,7 @@
-from __future__ import absolute_import
-
 import os
 from typing import Literal, Tuple
 
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 
 import logging
 

solvebio/cli/auth.py

@@ -1,6 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
-from __future__ import print_function
 
 import solvebio
 from ..client import client

solvebio/cli/credentials.py

@@ -1,8 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-from __future__ import absolute_import
 from collections import namedtuple
-import six
 
 import solvebio
 
@@ -55,11 +52,11 @@ def save(self, path):
         for host in self.hosts.keys():
             attrs = self.hosts[host]
             rep = (
-                rep + "machine " + host + "\n\tlogin " + six.text_type(attrs[0]) + "\n"
+                rep + "machine " + host + "\n\tlogin " + str(attrs[0]) + "\n"
             )
             if attrs[1]:
-                rep = rep + "\taccount " + six.text_type(attrs[1]) + "\n"
-            rep = rep + "\tpassword " + six.text_type(attrs[2]) + "\n"
+                rep = rep + "\taccount " + str(attrs[1]) + "\n"
+            rep = rep + "\tpassword " + str(attrs[2]) + "\n"
 
         f = open(path, "w")
         f.write(rep)

solvebio/cli/data.py

@@ -1,12 +1,8 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
-from __future__ import print_function
 
 import concurrent.futures
 from concurrent.futures import ThreadPoolExecutor
 
-from six.moves import input as raw_input
-
 import os
 import re
 import sys
@@ -128,7 +124,8 @@ def _upload_folder(
         dry_run=False,
         num_processes=1,
         archive_folder=None,
-        follow_shortcuts=False
+        follow_shortcuts=False,
+        max_retries=3,
 ):
     all_folders = []
     all_files = []
@@ -183,7 +180,7 @@ def _upload_folder(
             if should_exclude(local_file_path, exclude_paths, dry_run=dry_run):
                 continue
             all_files.append((local_file_path, remote_folder_full_path, vault.full_path,
-                              dry_run, archive_folder, client_auth, follow_shortcuts))
+                              dry_run, archive_folder, client_auth, follow_shortcuts, max_retries))
 
     if num_processes > 1:
         # Only perform optimization if parallelization is requested by the user
@@ -236,12 +233,13 @@ def _create_file_job(args):
         args[4] (archive_folder): An archive folder to move existing files into
         args[5] (client_auth): Tuple containing API host, token, and token type
         args[6] (follow_shortcuts): Boolean to follow shortcuts on the remote_folder_path
+        args[7] (max_retries): Maximum number of retries per upload part
     Returns:
         None or Exception if exception is raised.
     """
     try:
-        local_file_path, remote_folder_full_path, vault_path, dry_run, archive_folder, client_auth, follow_shortcuts \
-            = args
+        (local_file_path, remote_folder_full_path, vault_path, dry_run,
+         archive_folder, client_auth, follow_shortcuts, max_retries) = args
 
         # Provides the global host, token, token_type
         client = SolveClient(*client_auth)
@@ -274,6 +272,8 @@ def _create_file_job(args):
             remote_parent.vault.full_path,
             archive_folder=archive_folder,
             follow_shortcuts=follow_shortcuts,
+            num_processes=1,  # Default for single file uploads in parallel processing
+            max_retries=max_retries,
             client=client
         )
         return
@@ -511,15 +511,23 @@ def upload(args):
                 dry_run=args.dry_run,
                 num_processes=args.num_processes,
                 archive_folder=args.archive_folder,
-                follow_shortcuts=follow_shortcuts
+                follow_shortcuts=follow_shortcuts,
+                max_retries=args.max_retries,
             )
         else:
             if args.dry_run:
                 print(
                     "[Dry Run] Uploading {} to {}".format(local_path, path_dict["path"])
                 )
             else:
-                Object.upload_file(local_path, path_dict["path"], vault.full_path, archive_folder=args.archive_folder)
+                Object.upload_file(
+                    local_path,
+                    path_dict["path"],
+                    vault.full_path,
+                    archive_folder=args.archive_folder,
+                    num_processes=args.num_processes,
+                    max_retries=args.max_retries,
+                )
 
 
 def import_file(args):
@@ -1107,7 +1115,7 @@ def tag(args):
     if not args.no_input:
 
         print("")
-        res = raw_input(
+        res = input(
             "Are you sure you want to apply the above changes to "
             "{} object(s) in {} vault(s)? [y/N] ".format(
                 len(taggable_objects), len(seen_vaults.keys())

solvebio/cli/ipython.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 import sys
 import os
 

solvebio/cli/main.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
 import os
 import sys
 import copy
@@ -280,6 +279,13 @@ class SolveArgumentParser(argparse.ArgumentParser):
                     "help": "Resolves shortcuts when Uploading.",
                     "action": "store_true",
                 },
+                {
+                    "flags": "--max-retries",
+                    "help": "Maximum number of retries per upload part for multipart uploads. "
+                    "Defaults to 3.",
+                    "default": 3,
+                    "type": int,
+                },
                 {
                     "name": "local_path",
                     "help": "The path to the local file or directory " "to upload",

solvebio/cli/tutorial.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 import os
 from pydoc import pager
 

solvebio/client.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
 
 import json
 import time
@@ -24,7 +23,7 @@
 import ssl
 import sys
 
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 # Try using pyopenssl if available.
 # Requires: pip install pyopenssl ndg-httpsclient pyasn1

solvebio/contrib/dash/__init__.py

@@ -1,3 +1,2 @@
-from __future__ import absolute_import
 from .solvebio_auth import SolveBioAuth  # noqa: F401
 from .solvebio_dash import SolveBioDash  # noqa: F401