diff --git a/badge_server/datastore_cache.py b/badge_server/datastore_cache.py index fb2b7563..d5043844 100644 --- a/badge_server/datastore_cache.py +++ b/badge_server/datastore_cache.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """A key/value cache using Google Cloud Datastore.""" import json @@ -21,6 +20,7 @@ class DatastoreCache: + def __init__(self): self._datastore_client = datastore.Client() diff --git a/badge_server/fake_cache.py b/badge_server/fake_cache.py index e70813aa..91836b69 100644 --- a/badge_server/fake_cache.py +++ b/badge_server/fake_cache.py @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """An in-memory key/value cache.""" from typing import Any class FakeCache: + def __init__(self): self._cache = {} diff --git a/badge_server/main.py b/badge_server/main.py index 446cdf9c..30f4ecb4 100644 --- a/badge_server/main.py +++ b/badge_server/main.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """A HTTP server that generates badges for google python projects Requires Python 3.6 or later. @@ -114,8 +113,8 @@ def _get_pair_compatibility_dict(package_name: str) -> dict: unsupported_package_mapping = configs.PKG_PY_VERSION_NOT_SUPPORTED for res in compatibility_results: - version = res.python_major_version # eg. '2', '3' - pyver = badge_utils.PY_VER_MAPPING[version] # eg. 'py2', 'py3' + version = res.python_major_version # eg. '2', '3' + pyver = badge_utils.PY_VER_MAPPING[version] # eg. 'py2', 'py3' if result_dict[pyver]['details'] is None: result_dict[pyver]['details'] = {} @@ -197,10 +196,8 @@ def _get_dependency_dict(package_name: str) -> dict: return result_dict -def _get_badge_status( - self_compat_res: dict, - google_compat_res: dict, - dependency_res: dict) -> str: +def _get_badge_status(self_compat_res: dict, google_compat_res: dict, + dependency_res: dict) -> str: """Get the badge status. The badge status will determine the right hand text and the color of diff --git a/badge_server/redis_cache.py b/badge_server/redis_cache.py index aa09d7f0..50ffaae7 100644 --- a/badge_server/redis_cache.py +++ b/badge_server/redis_cache.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """A key/value cache using Redis.""" import os @@ -22,11 +21,11 @@ class RedisCache: + def __init__(self): redis_host = os.environ.get('REDISHOST', '10.0.0.3') redis_port = int(os.environ.get('REDISPORT', 6379)) - self._redis_client = redis.StrictRedis( - host=redis_host, port=redis_port) + self._redis_client = redis.StrictRedis(host=redis_host, port=redis_port) def get(self, name: str) -> Any: """Returns a Python value given a key. None if not found.""" diff --git a/badge_server/test_badge_server.py b/badge_server/test_badge_server.py index ba64d26b..f51c3fb3 100644 --- a/badge_server/test_badge_server.py +++ b/badge_server/test_badge_server.py @@ -29,19 +29,24 @@ class TestBadgeServer(unittest.TestCase): def setUp(self): self.mock_checker = mock.Mock(autospec=True) self.fake_store = fake_compatibility_store.CompatibilityStore() - self.patch_checker = mock.patch( - 'main.badge_utils.checker', self.mock_checker) - self.patch_store = mock.patch( - 'main.badge_utils.store', self.fake_store) + self.patch_checker = mock.patch('main.badge_utils.checker', + self.mock_checker) + self.patch_store = mock.patch('main.badge_utils.store', self.fake_store) def test__get_self_compatibility_dict(self): from compatibility_lib import compatibility_store from compatibility_lib import package expected = { - 'py2': {'status': 'SUCCESS', 'details': - 'The package does not support this version of python.'}, - 'py3': {'status': 'SUCCESS', 'details': 'NO DETAILS'}, + 'py2': { + 'status': 'SUCCESS', + 'details': + 'The package does not support this version of python.' + }, + 'py3': { + 'status': 'SUCCESS', + 'details': 'NO DETAILS' + }, } PACKAGE = package.Package('tensorflow') @@ -49,8 +54,8 @@ def test__get_self_compatibility_dict(self): packages=[PACKAGE], python_major_version=3, status=compatibility_store.Status.SUCCESS) - self.fake_store._packages_to_compatibility_result[ - frozenset([PACKAGE])] = [cr_py3] + self.fake_store._packages_to_compatibility_result[frozenset( + [PACKAGE])] = [cr_py3] with self.patch_checker, self.patch_store: result_dict = main._get_self_compatibility_dict('tensorflow') @@ -59,8 +64,14 @@ def test__get_self_compatibility_dict(self): def test__get_pair_compatibility_dict_success(self): expected = { - 'py2': {'status': 'SUCCESS', 'details': None}, - 'py3': {'status': 'SUCCESS', 'details': None} + 'py2': { + 'status': 'SUCCESS', + 'details': None + }, + 'py3': { + 'status': 'SUCCESS', + 'details': None + } } pkgs = ['google-api-core', 'google-api-python-client'] @@ -75,10 +86,18 @@ def test__get_pair_compatibility_dict_warning(self): from compatibility_lib import package expected = { - 'py2': {'status': 'CHECK_WARNING', - 'details': {'package2': 'NO DETAILS'} }, - 'py3': {'status': 'CHECK_WARNING', - 'details': {'package2': 'NO DETAILS'} }, + 'py2': { + 'status': 'CHECK_WARNING', + 'details': { + 'package2': 'NO DETAILS' + } + }, + 'py3': { + 'status': 'CHECK_WARNING', + 'details': { + 'package2': 'NO DETAILS' + } + }, } PACKAGE_1 = package.Package("package1") @@ -92,26 +111,30 @@ def test__get_pair_compatibility_dict_warning(self): python_major_version=3, status=compatibility_store.Status.CHECK_WARNING) pair_result = [cr_py2, cr_py3] - self.fake_store._packages_to_compatibility_result[ - frozenset([PACKAGE_1, PACKAGE_2])] = pair_result + self.fake_store._packages_to_compatibility_result[frozenset( + [PACKAGE_1, PACKAGE_2])] = pair_result mock_self_res = mock.Mock() self_res = { - 'py2': { 'status': 'SUCCESS', 'details': {} }, - 'py3': { 'status': 'SUCCESS', 'details': {} }, + 'py2': { + 'status': 'SUCCESS', + 'details': {} + }, + 'py3': { + 'status': 'SUCCESS', + 'details': {} + }, } mock_self_res.return_value = self_res - patch_self_status = mock.patch( - 'main._get_self_compatibility_dict', - mock_self_res) + patch_self_status = mock.patch('main._get_self_compatibility_dict', + mock_self_res) pkgs = ['package2'] patch_configs = mock.patch('main.configs.PKG_LIST', pkgs) with self.patch_checker, self.patch_store, patch_self_status, \ patch_configs: - result_dict = main._get_pair_compatibility_dict( - 'package1') + result_dict = main._get_pair_compatibility_dict('package1') self.assertEqual(result_dict, expected) @@ -120,8 +143,14 @@ def test__get_pair_compatibility_dict_install_error(self): from compatibility_lib import package expected = { - 'py2': { 'status': 'SUCCESS', 'details': {} }, - 'py3': { 'status': 'SUCCESS', 'details': {} }, + 'py2': { + 'status': 'SUCCESS', + 'details': {} + }, + 'py3': { + 'status': 'SUCCESS', + 'details': {} + }, } PACKAGE_1 = package.Package("package1") @@ -135,15 +164,14 @@ def test__get_pair_compatibility_dict_install_error(self): python_major_version=3, status=compatibility_store.Status.SUCCESS) pair_result = [cr_py2, cr_py3] - self.fake_store._packages_to_compatibility_result[ - frozenset([PACKAGE_1, PACKAGE_2])] = pair_result + self.fake_store._packages_to_compatibility_result[frozenset( + [PACKAGE_1, PACKAGE_2])] = pair_result pkgs = ['tensorflow'] patch_configs = mock.patch('main.configs.PKG_LIST', pkgs) with self.patch_checker, self.patch_store, patch_configs: - result_dict = main._get_pair_compatibility_dict( - 'package1') + result_dict = main._get_pair_compatibility_dict('package1') self.assertEqual(result_dict, expected) @@ -154,8 +182,14 @@ def test__get_pair_compatibility_dict_self_conflict(self): from compatibility_lib import package expected = { - 'py2': { 'status': 'SUCCESS', 'details': {} }, - 'py3': { 'status': 'SUCCESS', 'details': {} }, + 'py2': { + 'status': 'SUCCESS', + 'details': {} + }, + 'py3': { + 'status': 'SUCCESS', + 'details': {} + }, } PACKAGE_1 = package.Package("package1") @@ -169,39 +203,58 @@ def test__get_pair_compatibility_dict_self_conflict(self): python_major_version=3, status=compatibility_store.Status.CHECK_WARNING) pair_result = [cr_py2, cr_py3] - self.fake_store._packages_to_compatibility_result[ - frozenset([PACKAGE_1, PACKAGE_2])] = pair_result + self.fake_store._packages_to_compatibility_result[frozenset( + [PACKAGE_1, PACKAGE_2])] = pair_result mock_self_res = mock.Mock() self_res = { - 'py2': { 'status': 'CHECK_WARNING', 'details': {} }, - 'py3': { 'status': 'CHECK_WARNING', 'details': {} }, + 'py2': { + 'status': 'CHECK_WARNING', + 'details': {} + }, + 'py3': { + 'status': 'CHECK_WARNING', + 'details': {} + }, } mock_self_res.return_value = self_res - patch_self_status = mock.patch( - 'main._get_self_compatibility_dict', - mock_self_res) + patch_self_status = mock.patch('main._get_self_compatibility_dict', + mock_self_res) pkgs = ['tensorflow'] patch_configs = mock.patch('main.configs.PKG_LIST', pkgs) with self.patch_checker, self.patch_store, patch_self_status, \ patch_configs: - result_dict = main._get_pair_compatibility_dict( - 'package1') + result_dict = main._get_pair_compatibility_dict('package1') self.assertEqual(result_dict, expected) def test__get_check_results_success(self): expected_self_res = { - 'py2': { 'status': 'SUCCESS', 'details': {} }, - 'py3': { 'status': 'SUCCESS', 'details': {} }, + 'py2': { + 'status': 'SUCCESS', + 'details': {} + }, + 'py3': { + 'status': 'SUCCESS', + 'details': {} + }, } expected_google_res = { - 'py2': { 'status': 'SUCCESS', 'details': {} }, - 'py3': { 'status': 'SUCCESS', 'details': {} }, + 'py2': { + 'status': 'SUCCESS', + 'details': {} + }, + 'py3': { + 'status': 'SUCCESS', + 'details': {} + }, + } + expected_dep_res = { + 'status': 'UP_TO_DATE', + 'details': {}, } - expected_dep_res = { 'status': 'UP_TO_DATE', 'details': {}, } mock_self_res = mock.Mock() mock_self_res.return_value = expected_self_res @@ -212,15 +265,15 @@ def test__get_check_results_success(self): mock_dep_res = mock.Mock() mock_dep_res.return_value = expected_dep_res - patch_self_res = mock.patch( - 'main._get_self_compatibility_dict', mock_self_res) - patch_google_res = mock.patch( - 'main._get_pair_compatibility_dict', mock_google_res) - patch_dep_res = mock.patch( - 'main._get_dependency_dict', mock_dep_res) + patch_self_res = mock.patch('main._get_self_compatibility_dict', + mock_self_res) + patch_google_res = mock.patch('main._get_pair_compatibility_dict', + mock_google_res) + patch_dep_res = mock.patch('main._get_dependency_dict', mock_dep_res) with patch_self_res, patch_google_res, patch_dep_res: - self_res, google_res, dep_res = main._get_check_results('opencensus') + self_res, google_res, dep_res = main._get_check_results( + 'opencensus') status = main._get_badge_status(self_res, google_res, dep_res) self.assertEqual(self_res, expected_self_res) @@ -230,14 +283,29 @@ def test__get_check_results_success(self): def test__get_check_results_unknown(self): expected_self_res = { - 'py2': { 'status': 'UNKNOWN', 'details': {} }, - 'py3': { 'status': 'UNKNOWN', 'details': {} }, + 'py2': { + 'status': 'UNKNOWN', + 'details': {} + }, + 'py3': { + 'status': 'UNKNOWN', + 'details': {} + }, } expected_google_res = { - 'py2': { 'status': 'UNKNOWN', 'details': {} }, - 'py3': { 'status': 'UNKNOWN', 'details': {} }, + 'py2': { + 'status': 'UNKNOWN', + 'details': {} + }, + 'py3': { + 'status': 'UNKNOWN', + 'details': {} + }, + } + expected_dep_res = { + 'status': 'UNKNOWN', + 'details': {}, } - expected_dep_res = { 'status': 'UNKNOWN', 'details': {}, } mock_self_res = mock.Mock() mock_self_res.return_value = expected_self_res @@ -248,15 +316,15 @@ def test__get_check_results_unknown(self): mock_dep_res = mock.Mock() mock_dep_res.return_value = expected_dep_res - patch_self_res = mock.patch( - 'main._get_self_compatibility_dict', mock_self_res) - patch_google_res = mock.patch( - 'main._get_pair_compatibility_dict', mock_google_res) - patch_dep_res = mock.patch( - 'main._get_dependency_dict', mock_dep_res) + patch_self_res = mock.patch('main._get_self_compatibility_dict', + mock_self_res) + patch_google_res = mock.patch('main._get_pair_compatibility_dict', + mock_google_res) + patch_dep_res = mock.patch('main._get_dependency_dict', mock_dep_res) with patch_self_res, patch_google_res, patch_dep_res: - self_res, google_res, dep_res = main._get_check_results('unknown_package') + self_res, google_res, dep_res = main._get_check_results( + 'unknown_package') status = main._get_badge_status(self_res, google_res, dep_res) self.assertEqual(self_res, expected_self_res) @@ -266,14 +334,29 @@ def test__get_check_results_unknown(self): def test__get_check_results_check_warning(self): expected_self_res = { - 'py2': { 'status': 'CHECK_WARNING', 'details': {} }, - 'py3': { 'status': 'CHECK_WARNING', 'details': {} }, + 'py2': { + 'status': 'CHECK_WARNING', + 'details': {} + }, + 'py3': { + 'status': 'CHECK_WARNING', + 'details': {} + }, } expected_google_res = { - 'py2': { 'status': 'SUCCESS', 'details': {} }, - 'py3': { 'status': 'SUCCESS', 'details': {} }, + 'py2': { + 'status': 'SUCCESS', + 'details': {} + }, + 'py3': { + 'status': 'SUCCESS', + 'details': {} + }, + } + expected_dep_res = { + 'status': 'UP_TO_DATE', + 'details': {}, } - expected_dep_res = { 'status': 'UP_TO_DATE', 'details': {}, } mock_self_res = mock.Mock() mock_self_res.return_value = expected_self_res @@ -284,15 +367,15 @@ def test__get_check_results_check_warning(self): mock_dep_res = mock.Mock() mock_dep_res.return_value = expected_dep_res - patch_self_res = mock.patch( - 'main._get_self_compatibility_dict', mock_self_res) - patch_google_res = mock.patch( - 'main._get_pair_compatibility_dict', mock_google_res) - patch_dep_res = mock.patch( - 'main._get_dependency_dict', mock_dep_res) + patch_self_res = mock.patch('main._get_self_compatibility_dict', + mock_self_res) + patch_google_res = mock.patch('main._get_pair_compatibility_dict', + mock_google_res) + patch_dep_res = mock.patch('main._get_dependency_dict', mock_dep_res) with patch_self_res, patch_google_res, patch_dep_res: - self_res, google_res, dep_res = main._get_check_results('opencensus') + self_res, google_res, dep_res = main._get_check_results( + 'opencensus') status = main._get_badge_status(self_res, google_res, dep_res) self.assertEqual(self_res, expected_self_res) diff --git a/badge_server/utils.py b/badge_server/utils.py index cbbd799e..db03a40f 100644 --- a/badge_server/utils.py +++ b/badge_server/utils.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Common utils methods for badge server.""" import datetime @@ -41,8 +40,7 @@ store = compatibility_store.CompatibilityStore(mysql_unix_socket=UNIX_SOCKET) highlighter = dependency_highlighter.DependencyHighlighter( checker=checker, store=store) -finder = deprecated_dep_finder.DeprecatedDepFinder( - checker=checker, store=store) +finder = deprecated_dep_finder.DeprecatedDepFinder(checker=checker, store=store) priority_level = dependency_highlighter.PriorityLevel TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S" @@ -103,10 +101,9 @@ def initialize_cache(): return cache -def _build_default_result( - status: str, - include_pyversion: bool = True, - details: Optional = None) -> dict: +def _build_default_result(status: str, + include_pyversion: bool = True, + details: Optional = None) -> dict: """Build the default result for different conditions.""" # Dependency badge if not include_pyversion: @@ -150,9 +147,7 @@ def _get_badge(res: dict, badge_name: str) -> str: status = status.replace('_', ' ') return pybadges.badge( - left_text=badge_name, - right_text=status, - right_color=color) + left_text=badge_name, right_text=status, right_color=color) def _calculate_commit_number(package: str) -> Optional[str]: @@ -171,8 +166,8 @@ def _calculate_commit_number(package: str) -> Optional[str]: commits = json.loads(f.read()) return commits[0]['sha'] except Exception as e: - logging.warning( - 'Unable to generate caching key for "%s": %s', package, e) + logging.warning('Unable to generate caching key for "%s": %s', + package, e) return None return None @@ -184,8 +179,8 @@ def _is_github_cache_valid(cache_timestamp_str=None): if cache_timestamp_str is None: return False - cache_timestamp = datetime.datetime.strptime( - cache_timestamp_str, TIMESTAMP_FORMAT) + cache_timestamp = datetime.datetime.strptime(cache_timestamp_str, + TIMESTAMP_FORMAT) current_timestamp = datetime.datetime.now() seconds_diff = (current_timestamp - cache_timestamp).seconds diff --git a/compatibility_lib/compatibility_lib/compatibility_checker.py b/compatibility_lib/compatibility_lib/compatibility_checker.py index 03dc0f73..11f119c1 100644 --- a/compatibility_lib/compatibility_lib/compatibility_checker.py +++ b/compatibility_lib/compatibility_lib/compatibility_checker.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Send request to server to get self and pairwise compatibility data.""" import itertools @@ -44,10 +43,7 @@ def check(self, packages, python_version): ' by our checker server.' return UNKNOWN_STATUS_RESULT - data = { - 'python-version': python_version, - 'package': packages - } + data = {'python-version': python_version, 'package': packages} # Set the timeout to 299 seconds, which should be less than the # docker timeout (300 seconds). result = requests.get(SERVER_URL, params=data, timeout=299) @@ -63,21 +59,22 @@ def filter_packages(self, packages, python_version): pkg_name = configs.WHITELIST_URLS[pkg] else: pkg_name = pkg - if pkg_name not in configs.PKG_PY_VERSION_NOT_SUPPORTED[ - int(python_version)]: + if pkg_name not in configs.PKG_PY_VERSION_NOT_SUPPORTED[int( + python_version)]: filtered_packages.append(pkg) return filtered_packages - @retrying.retry(wait_random_min=1000, - wait_random_max=2000) + @retrying.retry(wait_random_min=1000, wait_random_max=2000) def retrying_check(self, args): """Retrying logic for sending requests to checker server.""" packages = args[0] python_version = args[1] return self.check(packages, python_version) - def collect_check_packages( - self, python_version=None, packages=None, pkg_sets=None): + def collect_check_packages(self, + python_version=None, + packages=None, + pkg_sets=None): # Generating single packages if packages is None: packages = configs.PKG_LIST @@ -91,8 +88,7 @@ def collect_check_packages( check_singles.append(([pkg], py_ver)) else: filtered_single = self.filter_packages(packages, python_version) - check_singles = [ - ([pkg], python_version) for pkg in filtered_single] + check_singles = [([pkg], python_version) for pkg in filtered_single] # Generating pairs if pkg_sets is None: @@ -103,8 +99,7 @@ def collect_check_packages( for py_ver in ['2', '3']: filtered_pkgs = [] for pkgs in pkg_sets: - if list(pkgs) != self.filter_packages(pkgs, - py_ver): + if list(pkgs) != self.filter_packages(pkgs, py_ver): continue filtered_pkgs.append(pkgs) for pkg_set in filtered_pkgs: @@ -112,27 +107,27 @@ def collect_check_packages( else: filtered_pkgs = [] for pkgs in pkg_sets: - if list(pkgs) != self.filter_packages(pkgs, - python_version): + if list(pkgs) != self.filter_packages(pkgs, python_version): continue filtered_pkgs.append(pkgs) - check_pairs = [(list(pkg_set), python_version) - for pkg_set in pkg_sets] + check_pairs = [ + (list(pkg_set), python_version) for pkg_set in pkg_sets + ] res = tuple(check_singles) + tuple(check_pairs) return res - def get_compatibility( - self, python_version=None, packages=None, pkg_sets=None): + def get_compatibility(self, + python_version=None, + packages=None, + pkg_sets=None): """Get the compatibility data for each package and package pairs.""" - check_packages = self.collect_check_packages( - python_version, packages, pkg_sets) + check_packages = self.collect_check_packages(python_version, packages, + pkg_sets) with concurrent.futures.ThreadPoolExecutor( max_workers=self.max_workers) as p: - pkg_set_results = p.map( - self.retrying_check, - tuple(check_packages)) + pkg_set_results = p.map(self.retrying_check, tuple(check_packages)) for result in zip(pkg_set_results): yield result diff --git a/compatibility_lib/compatibility_lib/compatibility_store.py b/compatibility_lib/compatibility_lib/compatibility_store.py index 40469502..73938266 100644 --- a/compatibility_lib/compatibility_lib/compatibility_store.py +++ b/compatibility_lib/compatibility_lib/compatibility_store.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Storage for package compatibility information.""" from contextlib import closing @@ -74,8 +73,8 @@ def __init__(self, def __repr__(self): return ('CompatibilityResult({}, {}, {}, {}, {}, {})'.format( - self.packages, self.python_major_version, self.status, - self.details, self.timestamp, self.dependency_info)) + self.packages, self.python_major_version, self.status, self.details, + self.timestamp, self.dependency_info)) def __hash__(self): return hash((tuple(self.packages), self.status, self.timestamp)) @@ -84,8 +83,7 @@ def __eq__(self, o): if isinstance(o, CompatibilityResult): return (frozenset(self.packages) == frozenset(o.packages) and self.python_major_version == o.python_major_version and - self.status == o.status and - self.details == o.details and + self.status == o.status and self.details == o.details and self.dependency_info == o.dependency_info and self.timestamp == o.timestamp) return NotImplemented @@ -190,8 +188,7 @@ def _row_to_compatibility_status(packages: Iterable[package.Package], ) @staticmethod - def _compatibility_status_to_row( - cs: CompatibilityResult) -> Tuple: + def _compatibility_status_to_row(cs: CompatibilityResult) -> Tuple: """Converts a CompatibilityResult into a tuple.""" status = cs.status.value py_version = str(cs.python_major_version) @@ -200,11 +197,11 @@ def _compatibility_status_to_row( install_name = cs.packages[0].install_name return (install_name, status, py_version, None, details) else: - names = sorted([cs.packages[0].install_name, - cs.packages[1].install_name]) + names = sorted( + [cs.packages[0].install_name, cs.packages[1].install_name]) install_name_lower, install_name_higher = names - return (install_name_lower, install_name_higher, - status, py_version, None, details) + return (install_name_lower, install_name_higher, status, py_version, + None, details) @staticmethod def _compatibility_status_to_release_time_rows( @@ -218,14 +215,11 @@ def _compatibility_status_to_release_time_rows( rows = [] for pkg, version_info in dependency_info.items(): - row = (install_name, - pkg, - version_info['installed_version'], + row = (install_name, pkg, version_info['installed_version'], version_info['installed_version_time'], version_info['latest_version'], version_info['latest_version_time'], - version_info['is_latest'], - version_info['current_time']) + version_info['is_latest'], version_info['current_time']) rows.append(row) return rows @@ -283,8 +277,8 @@ def get_self_compatibilities(self, for row in results: install_name = row[0] p = install_name_to_package[install_name] - package_to_result[p].append(self._row_to_compatibility_status( - [p], row)) + package_to_result[p].append( + self._row_to_compatibility_status([p], row)) return {p: crs for (p, crs) in package_to_result.items()} def get_pair_compatibility(self, packages: List[package.Package]) -> \ @@ -299,8 +293,8 @@ def get_pair_compatibility(self, packages: List[package.Package]) -> \ One CompatibilityResult per Python version. """ if len(packages) != 2: - raise ValueError( - 'expected 2 packages, got {}'.format(len(packages))) + raise ValueError('expected 2 packages, got {}'.format( + len(packages))) packages = sorted(packages, key=lambda p: p.install_name) query = ("SELECT * FROM pairwise_compatibility_status " @@ -310,12 +304,12 @@ def get_pair_compatibility(self, packages: List[package.Package]) -> \ with closing(self.connect()) as conn: with closing(conn.cursor()) as cursor: cursor.execute( - query, - (packages[0].install_name, packages[1].install_name)) + query, (packages[0].install_name, packages[1].install_name)) results = cursor.fetchall() - return [self._row_to_compatibility_status(packages, row) - for row in results] + return [ + self._row_to_compatibility_status(packages, row) for row in results + ] def get_compatibility_combinations(self, packages: List[package.Package]) -> \ @@ -354,8 +348,7 @@ def get_compatibility_combinations(self, p_lower = install_name_to_package[install_name_lower] p_higher = install_name_to_package[install_name_higher] packages_to_results[frozenset([p_lower, p_higher])].append( - self._row_to_compatibility_status([p_lower, p_higher], row) - ) + self._row_to_compatibility_status([p_lower, p_higher], row)) return {p: crs for (p, crs) in packages_to_results.items()} def get_pairwise_compatibility_for_package(self, package_name) -> \ @@ -392,9 +385,9 @@ def get_pairwise_compatibility_for_package(self, package_name) -> \ with closing(self.connect()) as conn: with closing(conn.cursor()) as cursor: - cursor.execute( - query, (install_names_lower, install_names_higher, - package_name, package_name)) + cursor.execute(query, + (install_names_lower, install_names_higher, + package_name, package_name)) results = cursor.fetchall() for row in results: @@ -405,23 +398,22 @@ def get_pairwise_compatibility_for_package(self, package_name) -> \ if not packages_to_results.get(key): packages_to_results[key] = [] packages_to_results[key].append( - self._row_to_compatibility_status([p_lower, p_higher], row) - ) + self._row_to_compatibility_status([p_lower, p_higher], row)) return packages_to_results def save_compatibility_statuses( - self, - compatibility_statuses: Iterable[ - CompatibilityResult]): + self, compatibility_statuses: Iterable[CompatibilityResult]): """Save the given CompatibilityStatuses""" compatibility_statuses = list(compatibility_statuses) - if any(cs for cs in compatibility_statuses - if len(cs.packages) not in [1, 2]): + if any( + cs for cs in compatibility_statuses + if len(cs.packages) not in [1, 2]): raise ValueError('CompatibilityResult must have 1 or 2 packages') - rows = [self._compatibility_status_to_row(s) for s in - compatibility_statuses] + rows = [ + self._compatibility_status_to_row(s) for s in compatibility_statuses + ] self_rows = [r for r in rows if len(r) == 5] pair_rows = [r for r in rows if len(r) == 6] @@ -483,9 +475,10 @@ def save_compatibility_statuses( if new_version > old_version: install_name_to_compatibility_result[install_name] = cs - dependency_rows = itertools.chain( - *[self._compatibility_status_to_release_time_rows(cs) - for cs in install_name_to_compatibility_result.values()]) + dependency_rows = itertools.chain(*[ + self._compatibility_status_to_release_time_rows(cs) + for cs in install_name_to_compatibility_result.values() + ]) # Insert the dependency rows in a stable order to make testing more # convenient. @@ -529,8 +522,8 @@ def _get_package_version(self, result: CompatibilityResult) -> str: for pkg, version_info in result.dependency_info.items(): if pkg == install_name_sanitized: return version_info['installed_version'] - raise ValueError('missing version information for {}'.format( - install_name_sanitized)) + raise ValueError( + 'missing version information for {}'.format(install_name_sanitized)) def get_dependency_info(self, package_name): """Returns dependency info for an indicated Google OSS package. diff --git a/compatibility_lib/compatibility_lib/configs.py b/compatibility_lib/compatibility_lib/configs.py index 64c30ef9..ce88cd5b 100644 --- a/compatibility_lib/compatibility_lib/configs.py +++ b/compatibility_lib/compatibility_lib/configs.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Common configs for compatibility_lib.""" @@ -90,90 +89,96 @@ def _format_url(repo_name, setuppy_path=''): # or github packages. WHITELIST_URLS = { _format_url('googleapis/google-cloud-python', 'asset'): - 'google-cloud-asset', + 'google-cloud-asset', _format_url('googleapis/google-cloud-python', 'automl'): - 'google-cloud-automl', + 'google-cloud-automl', _format_url('googleapis/google-cloud-python', 'dataproc'): - 'google-cloud-dataproc', + 'google-cloud-dataproc', _format_url('googleapis/google-cloud-python', 'dlp'): - 'google-cloud-dlp', + 'google-cloud-dlp', _format_url('googleapis/google-cloud-python', 'iot'): - 'google-cloud-iot', + 'google-cloud-iot', _format_url('googleapis/google-cloud-python', 'kms'): - 'google-cloud-kms', + 'google-cloud-kms', _format_url('googleapis/python-ndb', ''): - 'google-cloud-ndb', + 'google-cloud-ndb', # This is not released yet _format_url('googleapis/google-cloud-python', 'oslogin'): - 'google-cloud-os-login', + 'google-cloud-os-login', _format_url('googleapis/google-cloud-python', 'redis'): - 'google-cloud-redis', + 'google-cloud-redis', _format_url('googleapis/google-cloud-python', 'securitycenter'): - 'google-cloud-securitycenter', + 'google-cloud-securitycenter', _format_url('googleapis/google-cloud-python', 'tasks'): - 'google-cloud-tasks', + 'google-cloud-tasks', _format_url('googleapis/google-cloud-python', 'texttospeech'): - 'google-cloud-texttospeech', + 'google-cloud-texttospeech', _format_url('googleapis/google-cloud-python', 'websecurityscanner'): - 'google-cloud-websecurityscanner', + 'google-cloud-websecurityscanner', _format_url('googleapis/google-cloud-python', 'api_core'): - 'google-api-core', + 'google-api-core', _format_url('googleapis/google-cloud-python', 'bigquery'): - 'google-cloud-bigquery', + 'google-cloud-bigquery', _format_url('googleapis/google-cloud-python', 'bigquery_datatransfer'): - 'google-cloud-bigquery-datatransfer', + 'google-cloud-bigquery-datatransfer', _format_url('googleapis/google-cloud-python', 'bigtable'): - 'google-cloud-bigtable', + 'google-cloud-bigtable', _format_url('googleapis/google-cloud-python', 'container'): - 'google-cloud-container', + 'google-cloud-container', _format_url('googleapis/google-cloud-python', 'core'): - 'google-cloud-core', + 'google-cloud-core', _format_url('googleapis/google-cloud-python', 'datastore'): - 'google-cloud-datastore', - _format_url('googleapis/google-cloud-python', 'dns'): 'google-cloud-dns', + 'google-cloud-datastore', + _format_url('googleapis/google-cloud-python', 'dns'): + 'google-cloud-dns', _format_url('googleapis/google-cloud-python', 'error_reporting'): - 'google-cloud-error-reporting', + 'google-cloud-error-reporting', _format_url('googleapis/google-cloud-python', 'firestore'): - 'google-cloud-firestore', + 'google-cloud-firestore', _format_url('googleapis/google-cloud-python', 'language'): - 'google-cloud-language', + 'google-cloud-language', _format_url('googleapis/google-cloud-python', 'logging'): - 'google-cloud-logging', + 'google-cloud-logging', _format_url('googleapis/google-cloud-python', 'monitoring'): - 'google-cloud-monitoring', + 'google-cloud-monitoring', _format_url('googleapis/google-cloud-python', 'pubsub'): - 'google-cloud-pubsub', + 'google-cloud-pubsub', _format_url('googleapis/google-cloud-python', 'resource_manager'): - 'google-cloud-resource-manager', + 'google-cloud-resource-manager', _format_url('googleapis/google-cloud-python', 'runtimeconfig'): - 'google-cloud-runtimeconfig', + 'google-cloud-runtimeconfig', _format_url('googleapis/google-cloud-python', 'spanner'): - 'google-cloud-spanner', + 'google-cloud-spanner', _format_url('googleapis/google-cloud-python', 'speech'): - 'google-cloud-speech', + 'google-cloud-speech', _format_url('googleapis/google-cloud-python', 'storage'): - 'google-cloud-storage', + 'google-cloud-storage', _format_url('googleapis/google-cloud-python', 'trace'): - 'google-cloud-trace', + 'google-cloud-trace', _format_url('googleapis/google-cloud-python', 'translate'): - 'google-cloud-translate', + 'google-cloud-translate', _format_url('googleapis/google-cloud-python', 'videointelligence'): - 'google-cloud-videointelligence', + 'google-cloud-videointelligence', _format_url('googleapis/google-cloud-python', 'vision'): - 'google-cloud-vision', + 'google-cloud-vision', _format_url('googleapis/google-api-python-client'): - 'google-api-python-client', - _format_url('googleapis/google-auth-library-python'): 'google-auth', + 'google-api-python-client', + _format_url('googleapis/google-auth-library-python'): + 'google-auth', _format_url('GoogleCloudPlatform/google-resumable-media-python'): - 'google-resumable-media', - _format_url('apache/beam', 'sdks/python'): 'apache-beam[gcp]', - _format_url('google/apitools'): 'google-apitools', - _format_url('census-instrumentation/opencensus-python'): 'opencensus', - _format_url('google/protorpc'): 'protorpc', + 'google-resumable-media', + _format_url('apache/beam', 'sdks/python'): + 'apache-beam[gcp]', + _format_url('google/apitools'): + 'google-apitools', + _format_url('census-instrumentation/opencensus-python'): + 'opencensus', + _format_url('google/protorpc'): + 'protorpc', _format_url('tensorflow/tensorflow', 'tensorflow/tools/pip_package'): - 'tensorflow', - _format_url('GoogleCloudPlatform/cloud-opensource-python', - 'compatibility_lib'): 'compatibility-lib', + 'tensorflow', + _format_url('GoogleCloudPlatform/cloud-opensource-python', 'compatibility_lib'): + 'compatibility-lib', # TODO: The following projects do not use setup.py # googleapis-common-protos # grpc-google-iam-v1 @@ -192,6 +197,11 @@ def _format_url(repo_name, setuppy_path=''): ] PKG_PY_VERSION_NOT_SUPPORTED = { - 2: ['tensorflow', ], - 3: ['apache-beam[gcp]', 'gsutil', ], + 2: [ + 'tensorflow', + ], + 3: [ + 'apache-beam[gcp]', + 'gsutil', + ], } diff --git a/compatibility_lib/compatibility_lib/dependency_highlighter.py b/compatibility_lib/compatibility_lib/dependency_highlighter.py index ae595a6c..06a4c33d 100644 --- a/compatibility_lib/compatibility_lib/dependency_highlighter.py +++ b/compatibility_lib/compatibility_lib/dependency_highlighter.py @@ -22,7 +22,7 @@ from compatibility_lib import utils DEFAULT_GRACE_PERIOD_IN_DAYS = 183 # applies to any version updates -MAJOR_GRACE_PERIOD_IN_DAYS = 30 # applies to major version updates only +MAJOR_GRACE_PERIOD_IN_DAYS = 30 # applies to major version updates only ALLOWED_MINOR_DIFF = 3 @@ -73,13 +73,10 @@ def __str__(self): 'Latest Available:\t{}\n' 'Time Since Latest:\t{} days\n' '{}\n') - return msg.format( - self.name, - self.priority.level.name, - self.installed_version, - self.latest_version, - (self.current_time-self.latest_version_time).days, - self.priority.details) + return msg.format(self.name, self.priority.level.name, + self.installed_version, self.latest_version, + (self.current_time - self.latest_version_time).days, + self.priority.details) class DependencyHighlighter(object): @@ -166,16 +163,16 @@ def check_package(self, package_name): # Skip the check if release timestamp is None. if current_time is None or latest_version_time is None: logging.warning( - 'Release time for dependency {} is not available.' - .format(name)) + 'Release time for dependency {} is not available.'. + format(name)) continue try: latest = _sanitize_release_tag(info['latest_version']) except UnstableReleaseError: logging.warning( - 'The latest version of {} is not a stable release.' - .format(name)) + 'The latest version of {} is not a stable release.'. + format(name)) continue elapsed_time = current_time - latest_version_time @@ -183,8 +180,8 @@ def check_package(self, package_name): if priority.level == PriorityLevel.UP_TO_DATE: priority = self._get_update_priority( name, install, latest, elapsed_time) - dependency = OutdatedDependency( - name, package_name, priority, info) + dependency = OutdatedDependency(name, package_name, priority, + info) outdated_dependencies.append(dependency) return outdated_dependencies diff --git a/compatibility_lib/compatibility_lib/deprecated_dep_finder.py b/compatibility_lib/compatibility_lib/deprecated_dep_finder.py index b791b690..5a697d92 100644 --- a/compatibility_lib/compatibility_lib/deprecated_dep_finder.py +++ b/compatibility_lib/compatibility_lib/deprecated_dep_finder.py @@ -98,9 +98,8 @@ def get_deprecated_deps(self, packages=None): with concurrent.futures.ThreadPoolExecutor( max_workers=self.max_workers) as p: - results = p.map( - self.get_deprecated_dep, - ((pkg) for pkg in packages)) + results = p.map(self.get_deprecated_dep, + ((pkg) for pkg in packages)) for result in zip(results): yield result diff --git a/compatibility_lib/compatibility_lib/fake_compatibility_store.py b/compatibility_lib/compatibility_lib/fake_compatibility_store.py index e020d508..ac90214d 100644 --- a/compatibility_lib/compatibility_lib/fake_compatibility_store.py +++ b/compatibility_lib/compatibility_lib/fake_compatibility_store.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """In memory storage for package compatibility information.""" import itertools @@ -31,9 +30,11 @@ def __init__(self): def get_packages(self) -> Iterable[package.Package]: """Returns all packages tracked by the system.""" - return [list(p)[0] - for p in self._packages_to_compatibility_result.keys() - if len(p) == 1] + return [ + list(p)[0] + for p in self._packages_to_compatibility_result.keys() + if len(p) == 1 + ] @staticmethod def _filter_older_versions( @@ -62,8 +63,7 @@ def get_self_compatibility(self, One CompatibilityResult per Python version. """ return self._filter_older_versions( - self._packages_to_compatibility_result.get( - frozenset([p]), [])) + self._packages_to_compatibility_result.get(frozenset([p]), [])) def get_self_compatibilities(self, packages: Iterable[package.Package]) -> \ @@ -92,9 +92,7 @@ def get_pair_compatibility(self, packages: List[package.Package]) -> \ One CompatibilityResult per Python version. """ return self._filter_older_versions( - self._packages_to_compatibility_result.get( - frozenset(packages), - [])) + self._packages_to_compatibility_result.get(frozenset(packages), [])) def get_pairwise_compatibility_for_package(self, package_name: str) -> \ Mapping[FrozenSet[package.Package], @@ -115,10 +113,12 @@ def get_pairwise_compatibility_for_package(self, package_name: str) -> \ frozenset([p1, p4]): [CompatibilityResult...], }. """ - package_pairs = [[package.Package(package_name), package.Package(name)] - for name in configs.PKG_LIST] - results = {frozenset(pair): self.get_pair_compatibility(pair) - for pair in package_pairs} + package_pairs = [[package.Package(package_name), + package.Package(name)] for name in configs.PKG_LIST] + results = { + frozenset(pair): self.get_pair_compatibility(pair) + for pair in package_pairs + } return results def get_compatibility_combinations(self, @@ -140,13 +140,13 @@ def get_compatibility_combinations(self, frozenset([p2, p3]): [CompatibilityResult...], }. """ - return {frozenset([p1, p2]): self.get_pair_compatibility([p1, p2]) - for (p1, p2) in itertools.combinations(packages, r=2)} + return { + frozenset([p1, p2]): self.get_pair_compatibility([p1, p2]) + for (p1, p2) in itertools.combinations(packages, r=2) + } - def save_compatibility_statuses( - self, - compatibility_statuses: Iterable[ - compatibility_store.CompatibilityResult]): + def save_compatibility_statuses(self, compatibility_statuses: Iterable[ + compatibility_store.CompatibilityResult]): """Save the given CompatibilityStatuses""" for cr in compatibility_statuses: diff --git a/compatibility_lib/compatibility_lib/get_compatibility_data.py b/compatibility_lib/compatibility_lib/get_compatibility_data.py index 910ed9ea..b4aa526b 100644 --- a/compatibility_lib/compatibility_lib/get_compatibility_data.py +++ b/compatibility_lib/compatibility_lib/get_compatibility_data.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Get self and pairwise compatibility data and write to bigquery.""" import argparse @@ -51,8 +50,9 @@ def _result_dict_to_compatibility_result(results): res_dict = item[0] result_content, python_version = res_dict check_result = result_content.get('result') - packages_list = [package.Package(pkg) - for pkg in result_content.get('packages')] + packages_list = [ + package.Package(pkg) for pkg in result_content.get('packages') + ] details = result_content.get('description') timestamp = datetime.datetime.now().isoformat() dependency_info = result_content.get('dependency_info') @@ -63,8 +63,7 @@ def _result_dict_to_compatibility_result(results): status=compatibility_store.Status(check_result), details=details, timestamp=timestamp, - dependency_info=dependency_info - ) + dependency_info=dependency_info) res_list.append(compatibility_result) return res_list @@ -72,8 +71,8 @@ def _result_dict_to_compatibility_result(results): @contextlib.contextmanager def run_cloud_sql_proxy(cloud_sql_proxy_path): - instance_flag = '-instances={}=tcp:{}'.format( - INSTANCE_CONNECTION_NAME, PORT) + instance_flag = '-instances={}=tcp:{}'.format(INSTANCE_CONNECTION_NAME, + PORT) if cloud_sql_proxy_path is None: assert cloud_sql_proxy_path, 'Could not find cloud_sql_proxy path' process = popen_spawn.PopenSpawn([cloud_sql_proxy_path, instance_flag]) @@ -86,9 +85,8 @@ def run_cloud_sql_proxy(cloud_sql_proxy_path): ('Cloud SQL Proxy was unable to start after 5 seconds. Output ' 'of cloud_sql_proxy: \n{}').format(process.before)) except pexpect.exceptions.EOF: - raise ConnectionError( - ('Cloud SQL Proxy exited unexpectedly. Output of ' - 'cloud_sql_proxy: \n{}').format(process.before)) + raise ConnectionError(('Cloud SQL Proxy exited unexpectedly. Output of ' + 'cloud_sql_proxy: \n{}').format(process.before)) finally: process.kill(signal.SIGTERM) @@ -118,8 +116,9 @@ def get_package_pairs(check_pypi=False, check_github=False): return self_packages, pair_packages -def write_to_status_table( - check_pypi=False, check_github=False, cloud_sql_proxy_path=None): +def write_to_status_table(check_pypi=False, + check_github=False, + cloud_sql_proxy_path=None): """Get the compatibility status for PyPI versions.""" # Write self compatibility status to BigQuery self_packages, pair_packages = get_package_pairs(check_pypi, check_github) diff --git a/compatibility_lib/compatibility_lib/package.py b/compatibility_lib/compatibility_lib/package.py index 2d77caae..7bcc3dd6 100644 --- a/compatibility_lib/compatibility_lib/package.py +++ b/compatibility_lib/compatibility_lib/package.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Represents a pip-installable Python package.""" from typing import Optional diff --git a/compatibility_lib/compatibility_lib/package_crawler_static.py b/compatibility_lib/compatibility_lib/package_crawler_static.py index fe478309..28599617 100644 --- a/compatibility_lib/compatibility_lib/package_crawler_static.py +++ b/compatibility_lib/compatibility_lib/package_crawler_static.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """A set of functions that aid in crawling a package and creating a corresponding data model""" diff --git a/compatibility_lib/compatibility_lib/semver_checker.py b/compatibility_lib/compatibility_lib/semver_checker.py index 1e8e9714..e59b49bb 100644 --- a/compatibility_lib/compatibility_lib/semver_checker.py +++ b/compatibility_lib/compatibility_lib/semver_checker.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """checks two packages for semver breakage""" from compatibility_lib import package_crawler_static as crawler diff --git a/compatibility_lib/compatibility_lib/test_compatibility_checker.py b/compatibility_lib/compatibility_lib/test_compatibility_checker.py index 3f75fa06..38244004 100644 --- a/compatibility_lib/compatibility_lib/test_compatibility_checker.py +++ b/compatibility_lib/compatibility_lib/test_compatibility_checker.py @@ -38,16 +38,14 @@ def test_check(self): mock_requests.get.return_value = mock_response patch_request = mock.patch( - 'compatibility_lib.compatibility_checker.requests', - mock_requests) + 'compatibility_lib.compatibility_checker.requests', mock_requests) with patch_request: checker.check(packages, python_version) mock_requests.get.assert_called_with( compatibility_checker.SERVER_URL, params=data, timeout=299) - self.assertEqual(compatibility_checker.SERVER_URL, - expected_server_url) + self.assertEqual(compatibility_checker.SERVER_URL, expected_server_url) def _mock_retrying_check(self, *args): packages = args[0][0] @@ -59,8 +57,13 @@ def test_get_compatibility(self): pkg_list = ['pkg1', 'pkg2', 'pkg3'] pkg_py_version_not_supported = { - 2: ['tensorflow', ], - 3: ['apache-beam[gcp]', 'gsutil', ], + 2: [ + 'tensorflow', + ], + 3: [ + 'apache-beam[gcp]', + 'gsutil', + ], } mock_config = mock.Mock() @@ -73,8 +76,7 @@ def test_get_compatibility(self): 'compatibility_lib.compatibility_checker.concurrent.futures.ThreadPoolExecutor', FakeExecutor) patch_retrying_check = mock.patch.object( - compatibility_checker.CompatibilityChecker, - 'retrying_check', + compatibility_checker.CompatibilityChecker, 'retrying_check', self._mock_retrying_check) res = [] @@ -84,24 +86,24 @@ def test_get_compatibility(self): for item in result: res.append(item) - expected = sorted([ - ((['pkg1'], '2', 'SUCCESS'),), - ((['pkg2'], '2', 'SUCCESS'),), - ((['pkg3'], '2', 'SUCCESS'),), - ((['pkg1'], '3', 'SUCCESS'),), - ((['pkg2'], '3', 'SUCCESS'),), - ((['pkg3'], '3', 'SUCCESS'),), - ((['pkg1', 'pkg2'], '2', 'SUCCESS'),), - ((['pkg1', 'pkg3'], '2', 'SUCCESS'),), - ((['pkg2', 'pkg3'], '2', 'SUCCESS'),), - ((['pkg1', 'pkg2'], '3', 'SUCCESS'),), - ((['pkg1', 'pkg3'], '3', 'SUCCESS'),), - ((['pkg2', 'pkg3'], '3', 'SUCCESS'),)]) + expected = sorted([((['pkg1'], '2', 'SUCCESS'),), + ((['pkg2'], '2', 'SUCCESS'),), + ((['pkg3'], '2', 'SUCCESS'),), + ((['pkg1'], '3', 'SUCCESS'),), + ((['pkg2'], '3', 'SUCCESS'),), + ((['pkg3'], '3', 'SUCCESS'),), + ((['pkg1', 'pkg2'], '2', 'SUCCESS'),), + ((['pkg1', 'pkg3'], '2', 'SUCCESS'),), + ((['pkg2', 'pkg3'], '2', 'SUCCESS'),), + ((['pkg1', 'pkg2'], '3', 'SUCCESS'),), + ((['pkg1', 'pkg3'], '3', 'SUCCESS'),), + ((['pkg2', 'pkg3'], '3', 'SUCCESS'),)]) self.assertEqual(sorted(res), expected) class FakeExecutor(object): + def __init__(self, max_workers=10): self.max_workers = max_workers diff --git a/compatibility_lib/compatibility_lib/test_compatibility_store.py b/compatibility_lib/compatibility_lib/test_compatibility_store.py index 6457a19a..5adf969f 100644 --- a/compatibility_lib/compatibility_lib/test_compatibility_store.py +++ b/compatibility_lib/compatibility_lib/test_compatibility_store.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Tests for compatibility_store.CompatibilityStore.""" import datetime @@ -41,8 +40,8 @@ def test_constructor_default(self): status=status) self.assertEqual(compat_result.packages, packages) - self.assertEqual( - compat_result.python_major_version, python_major_version) + self.assertEqual(compat_result.python_major_version, + python_major_version) self.assertEqual(compat_result.status, status) self.assertIsNone(compat_result.details) self.assertIsNone(compat_result.dependency_info) @@ -60,7 +59,9 @@ def test_constructor_explicit(self): "latest_version": "2.1.0", "current_time": "2018-07-13T17:11:29.140608", "latest_version_time": "2018-05-12T16:26:31", - "is_latest": True}} + "is_latest": True + } + } timestamp = datetime.datetime.utcnow() compat_result = compatibility_store.CompatibilityResult( @@ -72,8 +73,8 @@ def test_constructor_explicit(self): timestamp=timestamp) self.assertEqual(compat_result.packages, packages) - self.assertEqual( - compat_result.python_major_version, python_major_version) + self.assertEqual(compat_result.python_major_version, + python_major_version) self.assertEqual(compat_result.status, status) self.assertEqual(compat_result.details, details) self.assertEqual(compat_result.dependency_info, dependency_info) @@ -89,9 +90,8 @@ def test_get_packages(self): mock_cursor = mock.Mock() mock_pymysql.connect.return_value = mock_conn mock_conn.cursor.return_value = mock_cursor - mock_cursor.fetchall.return_value = [ - [pkgs[0], 'SUCCESS'], - [pkgs[1], 'CHECK WARNING']] + mock_cursor.fetchall.return_value = [[pkgs[0], 'SUCCESS'], + [pkgs[1], 'CHECK WARNING']] patch_pymysql = mock.patch( 'compatibility_lib.compatibility_store.pymysql', mock_pymysql) @@ -117,8 +117,7 @@ def test_get_self_compatibility(self): mock_conn.cursor.return_value = mock_cursor mock_cursor.fetchall.return_value = [row] patch_pymysql = mock.patch( - 'compatibility_lib.compatibility_store.pymysql', - mock_pymysql) + 'compatibility_lib.compatibility_store.pymysql', mock_pymysql) store = compatibility_store.CompatibilityStore() with patch_pymysql: @@ -143,8 +142,7 @@ def test_get_self_compatibilities(self): mock_conn.cursor.return_value = mock_cursor mock_cursor.fetchall.return_value = rows patch_pymysql = mock.patch( - 'compatibility_lib.compatibility_store.pymysql', - mock_pymysql) + 'compatibility_lib.compatibility_store.pymysql', mock_pymysql) store = compatibility_store.CompatibilityStore() with patch_pymysql: @@ -164,8 +162,7 @@ def test_get_pair_compatibility_value_error(self): store = compatibility_store.CompatibilityStore() patch_pymysql = mock.patch( - 'compatibility_lib.compatibility_store.pymysql', - mock_pymysql) + 'compatibility_lib.compatibility_store.pymysql', mock_pymysql) packages = [PACKAGE_1] with patch_pymysql, self.assertRaises(ValueError): store.get_pair_compatibility(packages) @@ -176,12 +173,11 @@ def test_get_pair_compatibility(self): mock_cursor = mock.Mock() mock_pymysql.connect.return_value = mock_conn mock_conn.cursor.return_value = mock_cursor - row = ('pkg1', 'pkg2', 'SUCCESS', '3', - '2018-07-17 02:14:27.15768 UTC', None) + row = ('pkg1', 'pkg2', 'SUCCESS', '3', '2018-07-17 02:14:27.15768 UTC', + None) mock_cursor.fetchall.return_value = [row] patch_pymysql = mock.patch( - 'compatibility_lib.compatibility_store.pymysql', - mock_pymysql) + 'compatibility_lib.compatibility_store.pymysql', mock_pymysql) store = compatibility_store.CompatibilityStore() packages = [PACKAGE_1, PACKAGE_2] @@ -194,12 +190,12 @@ def test_get_pair_compatibility(self): isinstance(res[0], compatibility_store.CompatibilityResult)) def test_compatibility_combinations(self): - row1 = ('package1', 'package2', 'SUCCESS', - '3', '2018-07-17 02:14:27.15768 UTC', None) - row2 = ('package1', 'package3', 'SUCCESS', - '3', '2018-07-17 02:14:27.15768 UTC', None) - row3 = ('package2', 'package3', 'SUCCESS', - '3', '2018-07-17 02:14:27.15768 UTC', None) + row1 = ('package1', 'package2', 'SUCCESS', '3', + '2018-07-17 02:14:27.15768 UTC', None) + row2 = ('package1', 'package3', 'SUCCESS', '3', + '2018-07-17 02:14:27.15768 UTC', None) + row3 = ('package2', 'package3', 'SUCCESS', '3', + '2018-07-17 02:14:27.15768 UTC', None) store = compatibility_store.CompatibilityStore() mock_pymysql = mock.Mock() @@ -235,8 +231,7 @@ def test_save_compatibility_statuses_pair(self): details=None, dependency_info=None, timestamp=None) - row_pairwise = ('package1', 'package2', 'SUCCESS', - '3', None, None) + row_pairwise = ('package1', 'package2', 'SUCCESS', '3', None, None) mock_pymysql = mock.Mock() mock_conn = mock.Mock() @@ -254,8 +249,7 @@ def test_save_compatibility_statuses_pair(self): store = compatibility_store.CompatibilityStore() store.save_compatibility_statuses([comp_status]) - mock_cursor.executemany.assert_called_with( - pair_sql, [row_pairwise]) + mock_cursor.executemany.assert_called_with(pair_sql, [row_pairwise]) def test_save_compatibility_statuses_self(self): packages = [PACKAGE_1] @@ -284,8 +278,7 @@ def test_save_compatibility_statuses_self(self): store = compatibility_store.CompatibilityStore() store.save_compatibility_statuses([comp_status]) - mock_cursor.executemany.assert_called_with( - self_sql, [row_self]) + mock_cursor.executemany.assert_called_with(self_sql, [row_self]) def test_save_compatibility_statuses_release_time(self): packages = [PACKAGE_1] @@ -295,14 +288,16 @@ def test_save_compatibility_statuses_release_time(self): python_major_version='3', status=status, details=None, - dependency_info={'dep1': { - 'installed_version': '2.1.0', - 'installed_version_time': '2018-05-12T16:26:31', - 'latest_version': '2.1.0', - 'current_time': '2018-07-13T17:11:29.140608', - 'latest_version_time': '2018-05-12T16:26:31', - 'is_latest': True, - }}, + dependency_info={ + 'dep1': { + 'installed_version': '2.1.0', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.1.0', + 'current_time': '2018-07-13T17:11:29.140608', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': True, + } + }, timestamp=None) row_release_time = ('package1', 'dep1', '2.1.0', '2018-05-12T16:26:31', '2.1.0', '2018-05-12T16:26:31', True, @@ -323,8 +318,7 @@ def test_save_compatibility_statuses_release_time(self): store = compatibility_store.CompatibilityStore() store.save_compatibility_statuses([comp_status]) - mock_cursor.executemany.assert_called_with( - sql, [row_release_time]) + mock_cursor.executemany.assert_called_with(sql, [row_release_time]) def test_save_compatibility_statuses_release_time_for_latest(self): packages = [PACKAGE_4] @@ -335,28 +329,32 @@ def test_save_compatibility_statuses_release_time_for_latest(self): python_major_version='2', status=status, details=None, - dependency_info={'package4': { - 'installed_version': '12.7.0', - 'installed_version_time': '2018-05-12T16:26:31', - 'latest_version': '12.7.0', - 'current_time': '2018-07-13T17:11:29.140608', - 'latest_version_time': '2018-05-12T16:26:31', - 'is_latest': True, - }}, + dependency_info={ + 'package4': { + 'installed_version': '12.7.0', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '12.7.0', + 'current_time': '2018-07-13T17:11:29.140608', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': True, + } + }, timestamp=timestamp) comp_status_py3 = mock.Mock( packages=packages, python_major_version='3', status=status, details=None, - dependency_info={'package4': { - 'installed_version': '2.2.0', - 'installed_version_time': '2018-05-12T16:26:31', - 'latest_version': '2.7.0', - 'current_time': '2018-07-13T17:11:29.140608', - 'latest_version_time': '2018-05-12T16:26:31', - 'is_latest': False, - }}, + dependency_info={ + 'package4': { + 'installed_version': '2.2.0', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.7.0', + 'current_time': '2018-07-13T17:11:29.140608', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': False, + } + }, timestamp=timestamp) row_release_time = ('package4[gcp]', 'package4', '12.7.0', '2018-05-12T16:26:31', '12.7.0', @@ -378,8 +376,7 @@ def test_save_compatibility_statuses_release_time_for_latest(self): store.save_compatibility_statuses( [comp_status_py2, comp_status_py3]) - mock_cursor.executemany.assert_called_with( - sql, [row_release_time]) + mock_cursor.executemany.assert_called_with(sql, [row_release_time]) def test_save_compatibility_statuses_release_time_for_latest_many_packages( self): @@ -397,29 +394,32 @@ def test_save_compatibility_statuses_release_time_for_latest_many_packages( 'current_time': '2018-07-13T17:11:29.140608', 'latest_version_time': '2018-05-12T16:26:31', 'is_latest': False, - } , + }, 'apache-beam': { - 'installed_version': '2.7.0', - 'installed_version_time': '2018-05-12T16:26:31', - 'latest_version': '2.7.0', - 'current_time': '2018-07-13T17:11:29.140608', - 'latest_version_time': '2018-05-12T16:26:31', - 'is_latest': True, - }}, + 'installed_version': '2.7.0', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.7.0', + 'current_time': '2018-07-13T17:11:29.140608', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': True, + } + }, timestamp=None) apache_beam_py3 = mock.Mock( packages=[package.Package('apache-beam[gcp]')], python_major_version='3', status=status, details=None, - dependency_info={'apache-beam': { - 'installed_version': '2.2.0', - 'installed_version_time': '2018-05-12T16:26:31', - 'latest_version': '2.7.0', - 'current_time': '2018-07-13T17:11:29.140608', - 'latest_version_time': '2018-05-12T16:26:31', - 'is_latest': False, - }}, + dependency_info={ + 'apache-beam': { + 'installed_version': '2.2.0', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.7.0', + 'current_time': '2018-07-13T17:11:29.140608', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': False, + } + }, timestamp=None) google_api_core_py2 = mock.Mock( packages=[package.Package('google-api-core')], @@ -434,21 +434,24 @@ def test_save_compatibility_statuses_release_time_for_latest_many_packages( 'current_time': '2018-07-13T17:11:29.140608', 'latest_version_time': '2018-05-12T16:26:31', 'is_latest': True, - }}, + } + }, timestamp=None) google_api_core_py3 = mock.Mock( packages=[package.Package('google-api-core')], python_major_version='3', status=status, details=None, - dependency_info={'google-api-core': { - 'installed_version': '3.7.1', - 'installed_version_time': '2018-05-12T16:26:31', - 'latest_version': '2.7.0', - 'current_time': '2018-07-13T17:11:29.140608', - 'latest_version_time': '2018-05-12T16:26:31', - 'is_latest': False, - }}, + dependency_info={ + 'google-api-core': { + 'installed_version': '3.7.1', + 'installed_version_time': '2018-05-12T16:26:31', + 'latest_version': '2.7.0', + 'current_time': '2018-07-13T17:11:29.140608', + 'latest_version_time': '2018-05-12T16:26:31', + 'is_latest': False, + } + }, timestamp=None) apache_beam_row = ('apache-beam[gcp]', 'apache-beam', '2.7.0', @@ -477,15 +480,15 @@ def test_save_compatibility_statuses_release_time_for_latest_many_packages( with patch_pymysql: store = compatibility_store.CompatibilityStore() - store.save_compatibility_statuses( - [apache_beam_py2, - apache_beam_py3, - google_api_core_py2, - google_api_core_py3]) + store.save_compatibility_statuses([ + apache_beam_py2, apache_beam_py3, google_api_core_py2, + google_api_core_py3 + ]) mock_cursor.executemany.assert_called_with( sql, [apache_beam_row, six_row, google_api_core_row]) + class MockClient(object): def __init__(self, project=None): diff --git a/compatibility_lib/compatibility_lib/test_dependency_highlighter.py b/compatibility_lib/compatibility_lib/test_dependency_highlighter.py index 6fa93840..073a5d32 100644 --- a/compatibility_lib/compatibility_lib/test_dependency_highlighter.py +++ b/compatibility_lib/compatibility_lib/test_dependency_highlighter.py @@ -27,10 +27,7 @@ def _get_dep_info(datetime=True): if not datetime: return dep_info - fields = ( - 'installed_version_time', - 'latest_version_time', - 'current_time') + fields = ('installed_version_time', 'latest_version_time', 'current_time') for _, info in dep_info.items(): for field in fields: time = info[field] @@ -40,7 +37,7 @@ def _get_dep_info(datetime=True): class TestPriority(unittest.TestCase): - def test_constructor_default(self): + def test_constructor_default(self): expected_level = dependency_highlighter.PriorityLevel.UP_TO_DATE expected_details = '' @@ -49,13 +46,12 @@ def test_constructor_default(self): self.assertEqual(expected_level, priority.level) self.assertEqual(expected_details, priority.details) - def test_constructor_explicit(self): + def test_constructor_explicit(self): expected_level = dependency_highlighter.PriorityLevel.LOW_PRIORITY expected_details = 'this is a test' priority = dependency_highlighter.Priority( - level=expected_level, - details=expected_details) + level=expected_level, details=expected_details) self.assertEqual(expected_level, priority.level) self.assertEqual(expected_details, priority.details) @@ -69,18 +65,16 @@ class TestOutdatedDependency(unittest.TestCase): 'this dependency is 1 or more major versions behind the latest') expected_info = _get_dep_info()[expected_pkgname] - expected_repr = ( - "OutdatedDependency<'google-cloud-bigquery', " - "HIGH_PRIORITY>") + expected_repr = ("OutdatedDependency<'google-cloud-bigquery', " + "HIGH_PRIORITY>") - expected_str = ( - 'Dependency Name:\tgoogle-cloud-bigquery\n' - 'Priority:\t\tHIGH_PRIORITY\n' - 'Installed Version:\t0.25.0\n' - 'Latest Available:\t1.5.0\n' - 'Time Since Latest:\t14 days\n' - 'this dependency is 1 or more major versions ' - 'behind the latest\n') + expected_str = ('Dependency Name:\tgoogle-cloud-bigquery\n' + 'Priority:\t\tHIGH_PRIORITY\n' + 'Installed Version:\t0.25.0\n' + 'Latest Available:\t1.5.0\n' + 'Time Since Latest:\t14 days\n' + 'this dependency is 1 or more major versions ' + 'behind the latest\n') outdated = dependency_highlighter.OutdatedDependency( pkgname=expected_pkgname, @@ -92,21 +86,16 @@ def test_constructor(self): self.assertEqual(self.expected_pkgname, self.outdated.name) self.assertEqual(self.expected_parent, self.outdated.parent) self.assertEqual(self.expected_priority, self.outdated.priority) - self.assertEqual( - self.expected_info['installed_version'], - self.outdated.installed_version) - self.assertEqual( - self.expected_info['installed_version_time'], - self.outdated.installed_version_time) - self.assertEqual( - self.expected_info['latest_version'], - self.outdated.latest_version) - self.assertEqual( - self.expected_info['latest_version_time'], - self.outdated.latest_version_time) - self.assertEqual( - self.expected_info['current_time'], - self.outdated.current_time) + self.assertEqual(self.expected_info['installed_version'], + self.outdated.installed_version) + self.assertEqual(self.expected_info['installed_version_time'], + self.outdated.installed_version_time) + self.assertEqual(self.expected_info['latest_version'], + self.outdated.latest_version) + self.assertEqual(self.expected_info['latest_version_time'], + self.outdated.latest_version_time) + self.assertEqual(self.expected_info['current_time'], + self.outdated.current_time) def test_repr(self): self.assertEqual(self.expected_repr, self.outdated.__repr__()) @@ -167,23 +156,24 @@ def expect(level, msg): def run(install_tup, latest_tup, numdays): install, latest = dictify(install_tup), dictify(latest_tup) - res = highlighter._get_update_priority( - 'PACKAGE', install, latest, timedelta(days=numdays)) + res = highlighter._get_update_priority('PACKAGE', install, latest, + timedelta(days=numdays)) return res cases = [ - (expect(low, not_updated), run((2,5,0), (2,6,0), 5)), - (expect(high, six_months), run((2,5,0), (2,6,0), 200)), - (expect(high, three_minor), run((2,5,0), (2,8,0), 13)), - (expect(low, not_updated), run((2,5,0), (3,0,0), 29)), - (expect(high, thirty_days), run((2,5,0), (3,0,0), 50)), - (expect(high, major_version), run((2,5,0), (3,0,4), 1)), - (expect(high, major_version), run((2,5,0), (5,0,4), 1)), + (expect(low, not_updated), run((2, 5, 0), (2, 6, 0), 5)), + (expect(high, six_months), run((2, 5, 0), (2, 6, 0), 200)), + (expect(high, three_minor), run((2, 5, 0), (2, 8, 0), 13)), + (expect(low, not_updated), run((2, 5, 0), (3, 0, 0), 29)), + (expect(high, thirty_days), run((2, 5, 0), (3, 0, 0), 50)), + (expect(high, major_version), run((2, 5, 0), (3, 0, 4), 1)), + (expect(high, major_version), run((2, 5, 0), (5, 0, 4), 1)), ] return cases def test__get_update_priority(self): + def comp(p1, p2): if p1.level != p2.level: return False @@ -236,14 +226,14 @@ def test_check_packages(self): class TestUtilityFunctions(unittest.TestCase): good_tags = [ - ('1.1', (1, 1, 0)), - ('2018.5.90', (2018, 5, 90)), - ('10.1.0.1', (10, 1, 0)), + ('1.1', (1, 1, 0)), + ('2018.5.90', (2018, 5, 90)), + ('10.1.0.1', (10, 1, 0)), ] bad_tags = [ - '3', 'abc', '1.a2.0', '4..5.0', '6.30.1-dev', - '2.2.2rc', '1.0.dev', '2.1a0', '1.1rc3' + '3', 'abc', '1.a2.0', '4..5.0', '6.30.1-dev', '2.2.2rc', '1.0.dev', + '2.1a0', '1.1rc3' ] def test__sanitize_release_tag(self): diff --git a/compatibility_lib/compatibility_lib/test_deprecated_dep_finder.py b/compatibility_lib/compatibility_lib/test_deprecated_dep_finder.py index 4ac8bb26..cc835df3 100644 --- a/compatibility_lib/compatibility_lib/test_deprecated_dep_finder.py +++ b/compatibility_lib/compatibility_lib/test_deprecated_dep_finder.py @@ -58,14 +58,12 @@ def setUp(self): }, } - self.SELF_COMP_RES = (( - { - 'result': 'SUCCESS', - 'packages': ['package1'], - 'description': None, - 'dependency_info': self.DEP_INFO, - }, - ),) + self.SELF_COMP_RES = (({ + 'result': 'SUCCESS', + 'packages': ['package1'], + 'description': None, + 'dependency_info': self.DEP_INFO, + },),) self.mock_checker = mock.Mock(autospec=True) self.fake_store = fake_compatibility_store.CompatibilityStore() @@ -89,8 +87,8 @@ def test_constructor_explicit(self): py_version='2', checker=self.mock_checker, store=self.fake_store) self.assertEqual(finder.py_version, '2') - self.assertIsInstance( - finder._dependency_info_getter, utils.DependencyInfo) + self.assertIsInstance(finder._dependency_info_getter, + utils.DependencyInfo) def test__get_development_status_from_pypi_error(self): PKG_INFO = { @@ -107,7 +105,8 @@ def test__get_development_status_from_pypi_error(self): mock_call_pypi_json_api) with patch_utils: - finder = deprecated_dep_finder.DeprecatedDepFinder(checker=self.mock_checker, store=self.fake_store) + finder = deprecated_dep_finder.DeprecatedDepFinder( + checker=self.mock_checker, store=self.fake_store) development_status = finder._get_development_status_from_pypi( 'package1') diff --git a/compatibility_lib/compatibility_lib/test_fake_compatibility_store.py b/compatibility_lib/compatibility_lib/test_fake_compatibility_store.py index 5be537e8..54c39f78 100644 --- a/compatibility_lib/compatibility_lib/test_fake_compatibility_store.py +++ b/compatibility_lib/compatibility_lib/test_fake_compatibility_store.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Tests for compatibility_store.CompatibilityStore.""" import datetime @@ -95,8 +94,10 @@ def setUp(self): self._store = fake_compatibility_store.CompatibilityStore() def test_get_self_compatibility(self): - crs = [PACKAGE_1_PY2_CR, PACKAGE_1_PY2_OLD_CR, PACKAGE_1_PY3_CR, - PACKAGE_2_PY2_CR] + crs = [ + PACKAGE_1_PY2_CR, PACKAGE_1_PY2_OLD_CR, PACKAGE_1_PY3_CR, + PACKAGE_2_PY2_CR + ] self._store.save_compatibility_statuses(crs) self.assertEqual( frozenset(self._store.get_self_compatibility(PACKAGE_1)), @@ -106,44 +107,47 @@ def test_get_self_compatibility(self): frozenset([PACKAGE_2_PY2_CR])) def test_get_self_compatibility_no_result(self): - crs = [PACKAGE_1_PY2_CR, PACKAGE_1_PY2_OLD_CR, PACKAGE_1_PY3_CR, - PACKAGE_2_PY2_CR, - PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY3_CR] + crs = [ + PACKAGE_1_PY2_CR, PACKAGE_1_PY2_OLD_CR, PACKAGE_1_PY3_CR, + PACKAGE_2_PY2_CR, PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY3_CR + ] self._store.save_compatibility_statuses(crs) self.assertFalse( frozenset(self._store.get_self_compatibility(PACKAGE_3))) def test_get_self_compatibilities(self): - crs = [PACKAGE_1_PY2_CR, PACKAGE_1_PY2_OLD_CR, PACKAGE_1_PY3_CR, - PACKAGE_2_PY2_CR, - PACKAGE_3_PY2_CR, PACKAGE_3_PY3_CR, - PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY3_CR] + crs = [ + PACKAGE_1_PY2_CR, PACKAGE_1_PY2_OLD_CR, PACKAGE_1_PY3_CR, + PACKAGE_2_PY2_CR, PACKAGE_3_PY2_CR, PACKAGE_3_PY3_CR, + PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY3_CR + ] self._store.save_compatibility_statuses(crs) self.assertEqual( - self._store.get_self_compatibilities([PACKAGE_1, PACKAGE_2]), - { + self._store.get_self_compatibilities([PACKAGE_1, PACKAGE_2]), { PACKAGE_1: [PACKAGE_1_PY2_CR, PACKAGE_1_PY3_CR], PACKAGE_2: [PACKAGE_2_PY2_CR] }) def test_get_self_compatibilities_no_results(self): - crs = [PACKAGE_1_PY2_CR, PACKAGE_1_PY2_OLD_CR, PACKAGE_1_PY3_CR, - PACKAGE_2_PY2_CR, - PACKAGE_3_PY2_CR, PACKAGE_3_PY3_CR, - PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY3_CR] + crs = [ + PACKAGE_1_PY2_CR, PACKAGE_1_PY2_OLD_CR, PACKAGE_1_PY3_CR, + PACKAGE_2_PY2_CR, PACKAGE_3_PY2_CR, PACKAGE_3_PY3_CR, + PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY3_CR + ] self._store.save_compatibility_statuses(crs) self.assertEqual( self._store.get_self_compatibilities( - [PACKAGE_1, PACKAGE_2, PACKAGE_4]), - { - PACKAGE_1: [PACKAGE_1_PY2_CR, PACKAGE_1_PY3_CR], - PACKAGE_2: [PACKAGE_2_PY2_CR], - PACKAGE_4: [], - }) + [PACKAGE_1, PACKAGE_2, PACKAGE_4]), { + PACKAGE_1: [PACKAGE_1_PY2_CR, PACKAGE_1_PY3_CR], + PACKAGE_2: [PACKAGE_2_PY2_CR], + PACKAGE_4: [], + }) def test_get_pair_compatibility(self): - crs = [PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY2_OLD_CR, - PACKAGE_1_AND_2_PY3_CR] + crs = [ + PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY2_OLD_CR, + PACKAGE_1_AND_2_PY3_CR + ] self._store.save_compatibility_statuses(crs) self.assertEqual( @@ -160,28 +164,35 @@ def test_get_pair_compatibility_no_results(self): self._store.get_pair_compatibility([PACKAGE_1, PACKAGE_3]))) def test_get_compatibility_combinations(self): - crs = [PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY2_OLD_CR, - PACKAGE_1_AND_2_PY3_CR] + crs = [ + PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY2_OLD_CR, + PACKAGE_1_AND_2_PY3_CR + ] self._store.save_compatibility_statuses(crs) self.assertEqual( - frozenset(self._store.get_compatibility_combinations( - [PACKAGE_1, PACKAGE_2])), - frozenset({frozenset([PACKAGE_1, PACKAGE_2]): [ - PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY3_CR]}) - ) + frozenset( + self._store.get_compatibility_combinations( + [PACKAGE_1, PACKAGE_2])), + frozenset({ + frozenset([PACKAGE_1, PACKAGE_2]): + [PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY3_CR] + })) def test_get_compatibility_combinations_no_results(self): - crs = [PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY2_OLD_CR, - PACKAGE_1_AND_2_PY3_CR] + crs = [ + PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY2_OLD_CR, + PACKAGE_1_AND_2_PY3_CR + ] self._store.save_compatibility_statuses(crs) self.assertEqual( - frozenset(self._store.get_compatibility_combinations( - [PACKAGE_1, PACKAGE_2, PACKAGE_3])), + frozenset( + self._store.get_compatibility_combinations( + [PACKAGE_1, PACKAGE_2, PACKAGE_3])), frozenset({ - frozenset([PACKAGE_1, PACKAGE_2]): [PACKAGE_1_AND_2_PY2_CR, - PACKAGE_1_AND_2_PY3_CR], + frozenset([PACKAGE_1, PACKAGE_2]): + [PACKAGE_1_AND_2_PY2_CR, PACKAGE_1_AND_2_PY3_CR], frozenset([PACKAGE_1, PACKAGE_3]): [], frozenset([PACKAGE_2, PACKAGE_3]): [], })) diff --git a/compatibility_lib/compatibility_lib/test_get_compatibility_data.py b/compatibility_lib/compatibility_lib/test_get_compatibility_data.py index 7e858293..4d5bd7ce 100644 --- a/compatibility_lib/compatibility_lib/test_get_compatibility_data.py +++ b/compatibility_lib/compatibility_lib/test_get_compatibility_data.py @@ -37,17 +37,15 @@ class TestGetCompatibilityData(unittest.TestCase): 'is_latest': True } } - results = ( - (( - { - 'result': 'SUCCESS', - 'packages': ['google-api-core'], - 'description': None, - 'dependency_info': dependency_info, - }, - '3', - ),), - ) + results = ((( + { + 'result': 'SUCCESS', + 'packages': ['google-api-core'], + 'description': None, + 'dependency_info': dependency_info, + }, + '3', + ),),) packages = [package.Package('google-api-core')] status = compatibility_store.Status.SUCCESS @@ -69,8 +67,7 @@ def mock_init(): 'compatibility_lib.get_compatibility_data.checker', self.mock_checker) self.patch_store = mock.patch( - 'compatibility_lib.get_compatibility_data.store', - self.fake_store) + 'compatibility_lib.get_compatibility_data.store', self.fake_store) def test_get_package_pairs_pypi(self): mock_config = mock.Mock() @@ -83,8 +80,7 @@ def test_get_package_pairs_pypi(self): } mock_config.WHITELIST_URLS = WHITELIST_URLS patch_config = mock.patch( - 'compatibility_lib.get_compatibility_data.configs', - mock_config) + 'compatibility_lib.get_compatibility_data.configs', mock_config) with patch_config, self.patch_constructor, self.patch_checker, self.patch_store: from compatibility_lib import get_compatibility_data @@ -95,13 +91,10 @@ def test_get_package_pairs_pypi(self): expected_self_packages = sorted(['package1', 'package2', 'package3']) self.assertEqual(sorted(self_packages), expected_self_packages) - expected_pair_packages = sorted( - [('package1', 'package2'), - ('package1', 'package3'), - ('package2', 'package3')]) - self.assertEqual( - sorted(pair_packages), - expected_pair_packages) + expected_pair_packages = sorted([('package1', 'package2'), + ('package1', 'package3'), + ('package2', 'package3')]) + self.assertEqual(sorted(pair_packages), expected_pair_packages) def test_get_package_pairs_github(self): mock_config = mock.Mock() @@ -114,8 +107,7 @@ def test_get_package_pairs_github(self): } mock_config.WHITELIST_URLS = WHITELIST_URLS patch_config = mock.patch( - 'compatibility_lib.get_compatibility_data.configs', - mock_config) + 'compatibility_lib.get_compatibility_data.configs', mock_config) with patch_config, self.patch_constructor, self.patch_checker, self.patch_store: from compatibility_lib import get_compatibility_data @@ -123,22 +115,18 @@ def test_get_package_pairs_github(self): self_packages, pair_packages = get_compatibility_data.get_package_pairs( check_pypi=False, check_github=True) - expected_self_packages = sorted( - ['github.com/pkg1.git', - 'github.com/pkg2.git', - 'github.com/pkg3.git']) - self.assertEqual( - sorted(self_packages), expected_self_packages) - - expected_pair_packages = sorted( - [('github.com/pkg1.git', 'package2'), - ('github.com/pkg1.git', 'package3'), - ('github.com/pkg2.git', 'package1'), - ('github.com/pkg2.git', 'package3'), - ('github.com/pkg3.git', 'package1'), - ('github.com/pkg3.git', 'package2')]) - self.assertEqual( - sorted(pair_packages), expected_pair_packages) + expected_self_packages = sorted([ + 'github.com/pkg1.git', 'github.com/pkg2.git', 'github.com/pkg3.git' + ]) + self.assertEqual(sorted(self_packages), expected_self_packages) + + expected_pair_packages = sorted([('github.com/pkg1.git', 'package2'), + ('github.com/pkg1.git', 'package3'), + ('github.com/pkg2.git', 'package1'), + ('github.com/pkg2.git', 'package3'), + ('github.com/pkg3.git', 'package1'), + ('github.com/pkg3.git', 'package2')]) + self.assertEqual(sorted(pair_packages), expected_pair_packages) def test__result_dict_to_compatibility_result(self): with self.patch_constructor, self.patch_checker, self.patch_store: @@ -148,8 +136,8 @@ def test__result_dict_to_compatibility_result(self): res_list = get_compatibility_data._result_dict_to_compatibility_result( self.results) - self.assertTrue(isinstance( - res_list[0], compatibility_store.CompatibilityResult)) + self.assertTrue( + isinstance(res_list[0], compatibility_store.CompatibilityResult)) self.assertEqual(res_list[0].dependency_info, self.dependency_info) self.assertEqual(res_list[0].packages, self.packages) self.assertEqual(res_list[0].status, self.status) @@ -174,6 +162,7 @@ def test_write_to_status_table(self): self.assertEqual(saved_item.dependency_info, self.dependency_info) self.assertEqual(saved_item.status, self.status) + class MockProxy(object): def __enter__(self): diff --git a/compatibility_lib/compatibility_lib/test_semver_checker.py b/compatibility_lib/compatibility_lib/test_semver_checker.py index 141dbee6..b41997e0 100644 --- a/compatibility_lib/compatibility_lib/test_semver_checker.py +++ b/compatibility_lib/compatibility_lib/test_semver_checker.py @@ -19,7 +19,6 @@ from compatibility_lib.semver_checker import check from compatibility_lib.package_crawler_static import get_package_info, get_module_info - CWD = os.path.dirname(os.path.realpath(__file__)) TEST_DIR = os.path.join(CWD, 'testpkgs') @@ -64,7 +63,9 @@ def test_semver_check_on_added_args(self): new_dir = os.path.join(TEST_DIR, 'added_args/0.2.0') res = check(old_dir, new_dir) - expected = ['args do not match; expecting: "self, x", got: "self, x, y"'] + expected = [ + 'args do not match; expecting: "self, x", got: "self, x, y"' + ] self.assertEqual(expected, res) def test_semver_check_on_removed_args(self): diff --git a/compatibility_lib/compatibility_lib/test_utils.py b/compatibility_lib/compatibility_lib/test_utils.py index 9dde0d6d..5211aaa7 100644 --- a/compatibility_lib/compatibility_lib/test_utils.py +++ b/compatibility_lib/compatibility_lib/test_utils.py @@ -43,6 +43,7 @@ class MockChecker(object): + def check(self, packages, python_version): if not utils._is_package_in_whitelist(packages): UNKNOWN_STATUS_RESULT['packages'] = packages @@ -58,8 +59,7 @@ def check(self, packages, python_version): } def get_compatibility(self, python_version, packages=None): - return [[self.check( - packages=packages, python_version=python_version)]] + return [[self.check(packages=packages, python_version=python_version)]] class TestDependencyInfo(unittest.TestCase): @@ -118,12 +118,12 @@ def test_get_dependency_info_compatibility_store(self): class Test__parse_datetime(unittest.TestCase): - def test__parse_datetime(self): - date_string = '2018-08-16T15:42:04.351677' - expected = '2018-08-16 00:00:00' - res = utils._parse_datetime(date_string) - self.assertEqual(str(res), expected) + def test__parse_datetime(self): + date_string = '2018-08-16T15:42:04.351677' + expected = '2018-08-16 00:00:00' + res = utils._parse_datetime(date_string) + self.assertEqual(str(res), expected) - def test__parse_datetime_empty(self): - res = utils._parse_datetime(None) - self.assertIsNone(res) + def test__parse_datetime_empty(self): + res = utils._parse_datetime(None) + self.assertIsNone(res) diff --git a/compatibility_lib/compatibility_lib/testdata/mock_depinfo_data.py b/compatibility_lib/compatibility_lib/testdata/mock_depinfo_data.py index 951b00a9..4717957e 100644 --- a/compatibility_lib/compatibility_lib/testdata/mock_depinfo_data.py +++ b/compatibility_lib/compatibility_lib/testdata/mock_depinfo_data.py @@ -11,12 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """mock data for testing""" # data taken from google-cloud-dataflow -DEP_INFO = ( - """{ +DEP_INFO = ("""{ "PyVCF": { "installed_version": "0.6.8", @@ -467,5 +465,4 @@ "is_latest": true, "current_time": "2018-08-16 01:09:02.966368+00:00" } - }""" -) + }""") diff --git a/compatibility_lib/compatibility_lib/utils.py b/compatibility_lib/compatibility_lib/utils.py index 627dc499..918d083c 100644 --- a/compatibility_lib/compatibility_lib/utils.py +++ b/compatibility_lib/compatibility_lib/utils.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Common utils for compatibility_lib.""" from datetime import datetime @@ -39,8 +38,7 @@ def __init__(self, package_name): def call_pypi_json_api(package_name, pkg_version=None): if pkg_version is not None: - pypi_pkg_url = PYPI_URL + '{}/{}/json'.format( - package_name, pkg_version) + pypi_pkg_url = PYPI_URL + '{}/{}/json'.format(package_name, pkg_version) else: pypi_pkg_url = PYPI_URL + '{}/json'.format(package_name) @@ -50,8 +48,8 @@ def call_pypi_json_api(package_name, pkg_version=None): with urllib.request.urlopen(r) as f: result = json.loads(f.read().decode('utf-8')) except urllib.error.HTTPError: - logging.error('Package {} with version {} not found in Pypi'. - format(package_name, pkg_version)) + logging.error('Package {} with version {} not found in Pypi'.format( + package_name, pkg_version)) return None return result @@ -123,12 +121,12 @@ def _get_from_endpoint(self, package_name): # the package is not supported by checker server (not in whitelist). if depinfo is None: logging.warning( - "Could not get the dependency info of package {} from server." - .format(package_name)) + "Could not get the dependency info of package {} from server.". + format(package_name)) raise PackageNotSupportedError(package_name) - fields = ('installed_version_time', - 'current_time', 'latest_version_time') + fields = ('installed_version_time', 'current_time', + 'latest_version_time') for pkgname in depinfo.keys(): for field in fields: depinfo[pkgname][field] = _parse_datetime( diff --git a/compatibility_lib/setup.py b/compatibility_lib/setup.py index 89629b0a..c77f855f 100644 --- a/compatibility_lib/setup.py +++ b/compatibility_lib/setup.py @@ -17,21 +17,19 @@ with open("README.rst", "r") as fh: long_description = fh.read() - namespaces = ['compatibility_lib'] - setuptools.setup( name="compatibility_lib", version="0.1.4", author="Cloud Python", description="A library to get and store the dependency compatibility " - "status data to BigQuery.", + "status data to BigQuery.", long_description=long_description, license="Apache-2.0", include_package_data=True, url="https://github.com/GoogleCloudPlatform/cloud-opensource-python/tree/" - "master/compatibility_lib", + "master/compatibility_lib", packages=setuptools.find_packages(), namespace_packages=namespaces, classifiers=( diff --git a/compatibility_server/compatibility_checker_server.py b/compatibility_server/compatibility_checker_server.py index aa015a1c..88972a2f 100644 --- a/compatibility_server/compatibility_checker_server.py +++ b/compatibility_server/compatibility_checker_server.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """A HTTP server that wraps pip_checker. Requires Python 3.6 or later. @@ -118,7 +117,7 @@ def _enable_exporter(): """ project_id = _get_project_id() exporter = stackdriver_exporter.new_stats_exporter( - stackdriver_exporter.Options(project_id=project_id)) + stackdriver_exporter.Options(project_id=project_id)) STATS.view_manager.register_exporter(exporter) @@ -154,8 +153,7 @@ def check(): if unsupported_packages: return flask.make_response( 'Request contains unrecognized packages: {}'.format( - ', '.join(unsupported_packages)), - 400) + ', '.join(unsupported_packages)), 400) python_version = flask.request.args.get('python-version') if not python_version: @@ -164,13 +162,12 @@ def check(): if python_version not in PYTHON_VERSION_TO_COMMAND: return flask.make_response( 'Invalid Python version specified. Must be one of: {}'.format( - ', '.join(PYTHON_VERSION_TO_COMMAND), 400)) + ', '.join(PYTHON_VERSION_TO_COMMAND), 400)) python_command = PYTHON_VERSION_TO_COMMAND[python_version] try: - pip_result = pip_checker.check( - python_command, packages, STATS) + pip_result = pip_checker.check(python_command, packages, STATS) except pip_checker.PipCheckerError as pip_error: return flask.make_response(pip_error.error_msg, 500) @@ -184,11 +181,12 @@ def check(): def main(): class Handler(wsgiref.simple_server.WSGIRequestHandler): + def log_message(self, format, *args): # Override the default log_message method to avoid logging # remote addresses. - sys.stderr.write("[%s] %s\n" % (self.log_date_time_string(), - format % args)) + sys.stderr.write( + "[%s] %s\n" % (self.log_date_time_string(), format % args)) parser = argparse.ArgumentParser(description='Process some integers.') parser.add_argument( @@ -219,13 +217,10 @@ def log_message(self, format, *args): logging.basicConfig( level=logging.INFO, format='%(levelname)-8s %(asctime)s ' + - '%(filename)s:%(lineno)s] %(message)s') + '%(filename)s:%(lineno)s] %(message)s') with wsgiref.simple_server.make_server( - args.host, - args.port, - app, - handler_class=Handler) as httpd: + args.host, args.port, app, handler_class=Handler) as httpd: httpd.serve_forever() diff --git a/compatibility_server/configs.py b/compatibility_server/configs.py index 64c30ef9..ce88cd5b 100644 --- a/compatibility_server/configs.py +++ b/compatibility_server/configs.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Common configs for compatibility_lib.""" @@ -90,90 +89,96 @@ def _format_url(repo_name, setuppy_path=''): # or github packages. WHITELIST_URLS = { _format_url('googleapis/google-cloud-python', 'asset'): - 'google-cloud-asset', + 'google-cloud-asset', _format_url('googleapis/google-cloud-python', 'automl'): - 'google-cloud-automl', + 'google-cloud-automl', _format_url('googleapis/google-cloud-python', 'dataproc'): - 'google-cloud-dataproc', + 'google-cloud-dataproc', _format_url('googleapis/google-cloud-python', 'dlp'): - 'google-cloud-dlp', + 'google-cloud-dlp', _format_url('googleapis/google-cloud-python', 'iot'): - 'google-cloud-iot', + 'google-cloud-iot', _format_url('googleapis/google-cloud-python', 'kms'): - 'google-cloud-kms', + 'google-cloud-kms', _format_url('googleapis/python-ndb', ''): - 'google-cloud-ndb', + 'google-cloud-ndb', # This is not released yet _format_url('googleapis/google-cloud-python', 'oslogin'): - 'google-cloud-os-login', + 'google-cloud-os-login', _format_url('googleapis/google-cloud-python', 'redis'): - 'google-cloud-redis', + 'google-cloud-redis', _format_url('googleapis/google-cloud-python', 'securitycenter'): - 'google-cloud-securitycenter', + 'google-cloud-securitycenter', _format_url('googleapis/google-cloud-python', 'tasks'): - 'google-cloud-tasks', + 'google-cloud-tasks', _format_url('googleapis/google-cloud-python', 'texttospeech'): - 'google-cloud-texttospeech', + 'google-cloud-texttospeech', _format_url('googleapis/google-cloud-python', 'websecurityscanner'): - 'google-cloud-websecurityscanner', + 'google-cloud-websecurityscanner', _format_url('googleapis/google-cloud-python', 'api_core'): - 'google-api-core', + 'google-api-core', _format_url('googleapis/google-cloud-python', 'bigquery'): - 'google-cloud-bigquery', + 'google-cloud-bigquery', _format_url('googleapis/google-cloud-python', 'bigquery_datatransfer'): - 'google-cloud-bigquery-datatransfer', + 'google-cloud-bigquery-datatransfer', _format_url('googleapis/google-cloud-python', 'bigtable'): - 'google-cloud-bigtable', + 'google-cloud-bigtable', _format_url('googleapis/google-cloud-python', 'container'): - 'google-cloud-container', + 'google-cloud-container', _format_url('googleapis/google-cloud-python', 'core'): - 'google-cloud-core', + 'google-cloud-core', _format_url('googleapis/google-cloud-python', 'datastore'): - 'google-cloud-datastore', - _format_url('googleapis/google-cloud-python', 'dns'): 'google-cloud-dns', + 'google-cloud-datastore', + _format_url('googleapis/google-cloud-python', 'dns'): + 'google-cloud-dns', _format_url('googleapis/google-cloud-python', 'error_reporting'): - 'google-cloud-error-reporting', + 'google-cloud-error-reporting', _format_url('googleapis/google-cloud-python', 'firestore'): - 'google-cloud-firestore', + 'google-cloud-firestore', _format_url('googleapis/google-cloud-python', 'language'): - 'google-cloud-language', + 'google-cloud-language', _format_url('googleapis/google-cloud-python', 'logging'): - 'google-cloud-logging', + 'google-cloud-logging', _format_url('googleapis/google-cloud-python', 'monitoring'): - 'google-cloud-monitoring', + 'google-cloud-monitoring', _format_url('googleapis/google-cloud-python', 'pubsub'): - 'google-cloud-pubsub', + 'google-cloud-pubsub', _format_url('googleapis/google-cloud-python', 'resource_manager'): - 'google-cloud-resource-manager', + 'google-cloud-resource-manager', _format_url('googleapis/google-cloud-python', 'runtimeconfig'): - 'google-cloud-runtimeconfig', + 'google-cloud-runtimeconfig', _format_url('googleapis/google-cloud-python', 'spanner'): - 'google-cloud-spanner', + 'google-cloud-spanner', _format_url('googleapis/google-cloud-python', 'speech'): - 'google-cloud-speech', + 'google-cloud-speech', _format_url('googleapis/google-cloud-python', 'storage'): - 'google-cloud-storage', + 'google-cloud-storage', _format_url('googleapis/google-cloud-python', 'trace'): - 'google-cloud-trace', + 'google-cloud-trace', _format_url('googleapis/google-cloud-python', 'translate'): - 'google-cloud-translate', + 'google-cloud-translate', _format_url('googleapis/google-cloud-python', 'videointelligence'): - 'google-cloud-videointelligence', + 'google-cloud-videointelligence', _format_url('googleapis/google-cloud-python', 'vision'): - 'google-cloud-vision', + 'google-cloud-vision', _format_url('googleapis/google-api-python-client'): - 'google-api-python-client', - _format_url('googleapis/google-auth-library-python'): 'google-auth', + 'google-api-python-client', + _format_url('googleapis/google-auth-library-python'): + 'google-auth', _format_url('GoogleCloudPlatform/google-resumable-media-python'): - 'google-resumable-media', - _format_url('apache/beam', 'sdks/python'): 'apache-beam[gcp]', - _format_url('google/apitools'): 'google-apitools', - _format_url('census-instrumentation/opencensus-python'): 'opencensus', - _format_url('google/protorpc'): 'protorpc', + 'google-resumable-media', + _format_url('apache/beam', 'sdks/python'): + 'apache-beam[gcp]', + _format_url('google/apitools'): + 'google-apitools', + _format_url('census-instrumentation/opencensus-python'): + 'opencensus', + _format_url('google/protorpc'): + 'protorpc', _format_url('tensorflow/tensorflow', 'tensorflow/tools/pip_package'): - 'tensorflow', - _format_url('GoogleCloudPlatform/cloud-opensource-python', - 'compatibility_lib'): 'compatibility-lib', + 'tensorflow', + _format_url('GoogleCloudPlatform/cloud-opensource-python', 'compatibility_lib'): + 'compatibility-lib', # TODO: The following projects do not use setup.py # googleapis-common-protos # grpc-google-iam-v1 @@ -192,6 +197,11 @@ def _format_url(repo_name, setuppy_path=''): ] PKG_PY_VERSION_NOT_SUPPORTED = { - 2: ['tensorflow', ], - 3: ['apache-beam[gcp]', 'gsutil', ], + 2: [ + 'tensorflow', + ], + 3: [ + 'apache-beam[gcp]', + 'gsutil', + ], } diff --git a/compatibility_server/fake_pip.py b/compatibility_server/fake_pip.py index 45332cef..5a8fac85 100755 --- a/compatibility_server/fake_pip.py +++ b/compatibility_server/fake_pip.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """A fake implementation of pip whose behavior can be modified using flags.""" import argparse @@ -102,8 +101,7 @@ def main(): elif command == 'uninstall': sys.exit(known.uninstall_returncode) else: - print('unexpected command: {}'.format(command), - end='', file=sys.stderr) + print('unexpected command: {}'.format(command), end='', file=sys.stderr) sys.exit(1) diff --git a/compatibility_server/loadtest/locustfile.py b/compatibility_server/loadtest/locustfile.py index 0033d4d1..5ec4e818 100644 --- a/compatibility_server/loadtest/locustfile.py +++ b/compatibility_server/loadtest/locustfile.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Perform a load test on the compatibility server. Usage: $ pip install locustio @@ -23,7 +22,6 @@ import locust - PYTHON2_PACKAGES = [ 'apache-beam[gcp]', 'google-cloud-bigtable', @@ -43,18 +41,25 @@ class CompatibilityCheck(locust.TaskSet): + @locust.task def single_python2(self): - query = urllib.parse.urlencode( - {'python-version': '2', - 'package': random.choice(PYTHON2_PACKAGES)}) + query = urllib.parse.urlencode({ + 'python-version': + '2', + 'package': + random.choice(PYTHON2_PACKAGES) + }) self.client.get('/?%s' % query) @locust.task def single_python3(self): - query = urllib.parse.urlencode( - {'python-version': '3', - 'package': random.choice(PYTHON3_PACKAGES)}) + query = urllib.parse.urlencode({ + 'python-version': + '3', + 'package': + random.choice(PYTHON3_PACKAGES) + }) self.client.get('/?%s' % query) @locust.task diff --git a/compatibility_server/pip_checker.py b/compatibility_server/pip_checker.py index 67dafa8b..2d21b026 100644 --- a/compatibility_server/pip_checker.py +++ b/compatibility_server/pip_checker.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Installs packages using pip and tests them for version compatibility. Conceptually, it runs: @@ -71,15 +70,13 @@ def __init__(self, error_msg: str): class PipError(PipCheckerError): """A pip command failed in an unexpected way.""" - def __init__(self, - error_msg: str, - command: List[str], - returncode: int): + def __init__(self, error_msg: str, command: List[str], returncode: int): command_string = ' '.join(shlex.quote(c) for c in command) super(PipError, self).__init__( 'Pip command ({command_string}) failed with error [{returncode}]: ' '{error_msg}'.format( - command_string=command_string, returncode=returncode, + command_string=command_string, + returncode=returncode, error_msg=error_msg)) self.command = command @@ -165,17 +162,16 @@ def __eq__(self, other): def __repr__(self): return ('PipCheckResult(packages={!r}, result_type={!r}, ' + 'result_text={!r}, dependency_info={!r})').format( - self.packages, - self.result_type, - self.result_text, - self.dependency_info) + self.packages, self.result_type, self.result_text, + self.dependency_info) def with_extra_attrs(self, dependency_info: Optional[str] = None): """Return a new PipCheckResult with extra attributes.""" - return PipCheckResult(self.packages, - self.result_type, - self.result_text, - dependency_info=dependency_info) + return PipCheckResult( + self.packages, + self.result_type, + self.result_text, + dependency_info=dependency_info) @property def packages(self) -> List[str]: @@ -206,9 +202,7 @@ class _OneshotPipCheck(): See https://pip.pypa.io/en/stable/user_guide/. """ - def __init__(self, - pip_command: List[str], - packages: List[str], + def __init__(self, pip_command: List[str], packages: List[str], stats: stats_module.Stats): """Initializes _OneshotPipCheck with the arguments needed to run pip. @@ -260,7 +254,7 @@ def _run_container(self, self._mmap.record(self._tmap) raise PipCheckerError( error_msg="An error occurred while starting a docker " - "container. Error message: {}".format(e.explanation)) + "container. Error message: {}".format(e.explanation)) except IOError as e: # TODO: Log the exception and monitor it after trying to decode # this into a requests.exception.* e.g. ReadTimeout. See: @@ -269,12 +263,11 @@ def _run_container(self, self._mmap.record(self._tmap) raise PipCheckerError( error_msg="An error occurred while starting a docker " - "container. Error message: {}".format(e)) + "container. Error message: {}".format(e)) return container - def _cleanup_container(self, - container: docker.models.containers.Container): + def _cleanup_container(self, container: docker.models.containers.Container): """Stop the container and remove it's associated storage.""" try: container.stop(timeout=0) @@ -283,7 +276,7 @@ def _cleanup_container(self, self._mmap.record(self._tmap) raise PipCheckerError( error_msg="Error occurs when cleaning up docker container." - "Container does not exist.") + "Container does not exist.") except IOError as e: # TODO: Log the exception and monitor it after trying to decode # this into a requests.exception.* e.g. ReadTimeout. See: @@ -292,7 +285,7 @@ def _cleanup_container(self, self._mmap.record(self._tmap) raise PipCheckerError( error_msg="An error occurred while stopping a docker" - "container. Error message: {}".format(e)) + "container. Error message: {}".format(e)) def _run_command( self, @@ -325,9 +318,9 @@ def _run_command( self._mmap.measure_int_put(views.DOCKER_ERROR_MEASURE, 1) self._mmap.record(self._tmap) raise PipCheckerError(error_msg="Error occurs when executing " - "commands in container." - "Error message: " - "{}".format(e.explanation)) + "commands in container." + "Error message: " + "{}".format(e.explanation)) except IOError as e: # TODO: Log the exception and monitor it after trying to decode # this into a requests.exception.* e.g. ReadTimeout. See: @@ -336,7 +329,7 @@ def _run_command( self._mmap.record(self._tmap) raise PipCheckerError( error_msg="An error occurred while running the command {} in" - "container. Error message: {}".format(command, e)) + "container. Error message: {}".format(command, e)) # Checking for cases where the command was killed by a signal. # If a process was killed by a signal, then it's exit code will be @@ -348,13 +341,11 @@ def _run_command( self._mmap.record(self._tmap) raise PipCheckerError( error_msg="The command {} was killed by signal {}. " - "This likely means that the Docker container timed " - "out. Error msg: {}".format( - command, returncode - 128, output)) + "This likely means that the Docker container timed " + "out. Error msg: {}".format(command, returncode - 128, output)) elif returncode and raise_on_failure: - raise PipError(error_msg=output, - command=command, - returncode=returncode) + raise PipError( + error_msg=output, command=command, returncode=returncode) return returncode, output @@ -369,8 +360,8 @@ def _call_pypi_json_api(pkg_name: str, pkg_version: str): with urllib.request.urlopen(r) as f: result = json.loads(f.read().decode('utf-8')) except urllib.error.HTTPError: - logging.error('Package {} with version {} not found in Pypi'. - format(pkg_name, pkg_version)) + logging.error('Package {} with version {} not found in Pypi'.format( + pkg_name, pkg_version)) return None return result @@ -378,22 +369,20 @@ def _build_command(self, subcommands: List[str]): """Build pip commands.""" return self._pip_command + subcommands - def _clone_repo( - self, - container: docker.models.containers.Container, - github_url: str): + def _clone_repo(self, container: docker.models.containers.Container, + github_url: str): """Shallow clone the google-cloud-python repository.""" # Create a temp directory _, directory = self._run_command( - container, - ['echo', '$mktemp -d'], + container, ['echo', '$mktemp -d'], stderr=False, stdout=True, raise_on_failure=True) directory = directory.strip() - shallow_clone_command = ['git', 'clone', '--depth', '1', - github_url, directory] + shallow_clone_command = [ + 'git', 'clone', '--depth', '1', github_url, directory + ] self._run_command( container, @@ -424,8 +413,8 @@ def _install(self, container: docker.models.containers.Container): # real 0m2.242s # user 0m0.472s # sys 0m0.357s - client_repo_directory = self._clone_repo( - container, CLIENTLIBS_GITHUB_URL) + client_repo_directory = self._clone_repo(container, + CLIENTLIBS_GITHUB_URL) install_names = [] for pkg in self._packages: @@ -447,15 +436,14 @@ def _install(self, container: docker.models.containers.Container): if returncode: # Checking for environment error - environment_error = PIP_ENVIRONMENT_ERROR_PATTERN.search( - output) + environment_error = PIP_ENVIRONMENT_ERROR_PATTERN.search(output) if environment_error: - raise PipError(error_msg=environment_error.group('error'), - command=command, - returncode=returncode) + raise PipError( + error_msg=environment_error.group('error'), + command=command, + returncode=returncode) return PipCheckResult(self._packages, - PipCheckResultType.INSTALL_ERROR, - output) + PipCheckResultType.INSTALL_ERROR, output) return PipCheckResult(self._packages, PipCheckResultType.SUCCESS) @@ -477,11 +465,10 @@ def _check(self, container: docker.models.containers.Container): if not has_version_conflicts: raise PipCheckerError( error_msg="The docker container timed out before executing" - "pip command. Error msg: {}".format(output)) + "pip command. Error msg: {}".format(output)) else: return PipCheckResult(self._packages, - PipCheckResultType.CHECK_WARNING, - output) + PipCheckResultType.CHECK_WARNING, output) return PipCheckResult(self._packages, PipCheckResultType.SUCCESS) def _list(self, container: docker.models.containers.Container): @@ -532,7 +519,6 @@ def _list(self, container: docker.models.containers.Container): } } """ - """Use pypi json api to get the release date of the latest version.""" pkg_version_date = {} @@ -587,8 +573,7 @@ def _list(self, container: docker.models.containers.Container): latest_version_time = None if result is not None: if 'releases' in result: - latest_release = result.get('releases').get( - latest_version) + latest_release = result.get('releases').get(latest_version) installed_release = result.get('releases').get( installed_version) if latest_release: @@ -634,8 +619,7 @@ def run(self) -> PipCheckResult: self._cleanup_container(container) -def check(pip_command: List[str], - packages: List[str], +def check(pip_command: List[str], packages: List[str], stats: stats_module.Stats) -> PipCheckResult: """Runs a version compatibility check using the given packages. diff --git a/compatibility_server/test_configs.py b/compatibility_server/test_configs.py index 5c868903..b6290600 100644 --- a/compatibility_server/test_configs.py +++ b/compatibility_server/test_configs.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Tests for configs.""" import os diff --git a/compatibility_server/test_pip_checker.py b/compatibility_server/test_pip_checker.py index 1d5c8398..80caab83 100644 --- a/compatibility_server/test_pip_checker.py +++ b/compatibility_server/test_pip_checker.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Tests for pip_checker. Uses a script "fake_pip.py" to simulate the behavior of the pip @@ -76,8 +75,8 @@ def run(self, auto_remove=False): from datetime import datetime - self.start_time = timestamp_to_seconds( - datetime.utcnow().isoformat() + 'Z') + self.start_time = timestamp_to_seconds(datetime.utcnow().isoformat() + + 'Z') return self def exec_run(self, cmd, stdout=True, stderr=True): @@ -85,14 +84,12 @@ def exec_run(self, cmd, stdout=True, stderr=True): _stdout = subprocess.PIPE if stdout else None _stderr = subprocess.PIPE if stderr else None - result = subprocess.run( - cmd, stderr=_stderr, stdout=_stdout) + result = subprocess.run(cmd, stderr=_stderr, stdout=_stdout) output = result.stdout if stdout else b'' output += result.stderr if stderr else b'' - current_time = timestamp_to_seconds( - datetime.utcnow().isoformat() + 'Z') + current_time = timestamp_to_seconds(datetime.utcnow().isoformat() + 'Z') duration = current_time - self.start_time if duration > pip_checker.TIME_OUT: @@ -105,20 +102,20 @@ def exec_run(self, cmd, stdout=True, stderr=True): class TestPipChecker(unittest.TestCase): def setUp(self): - self._fake_pip_path = os.path.join(os.path.dirname(__file__), - 'fake_pip.py') + self._fake_pip_path = os.path.join( + os.path.dirname(__file__), 'fake_pip.py') self._stats = stats_module.Stats() for view in views.ALL_VIEWS: self._stats.view_manager.register_view(view) def test__run_command_success(self): - checker = pip_checker._OneshotPipCheck( - ['python3', '-m', 'pip'], packages=['six'], stats=self._stats) + checker = pip_checker._OneshotPipCheck(['python3', '-m', 'pip'], + packages=['six'], + stats=self._stats) container = checker._run_container(MockDockerClient()) returncode, output = checker._run_command( - container, - ["echo", "testing"], + container, ["echo", "testing"], stdout=True, stderr=True, raise_on_failure=False) @@ -126,30 +123,28 @@ def test__run_command_success(self): self.assertEqual(output, 'testing\n') docker_view_name = views.DOCKER_ERROR_VIEW.name - docker_view_data = self._stats.view_manager.get_view( - docker_view_name) + docker_view_data = self._stats.view_manager.get_view(docker_view_name) self.assertEqual(docker_view_data._tag_value_aggregation_data_map, {}) def test__run_command_timeout(self): - checker = pip_checker._OneshotPipCheck( - ['python3', '-m', 'pip'], packages=['six'], stats=self._stats) + checker = pip_checker._OneshotPipCheck(['python3', '-m', 'pip'], + packages=['six'], + stats=self._stats) TIME_OUT = 0.1 patch_timeout = mock.patch('pip_checker.TIME_OUT', TIME_OUT) - with patch_timeout, self.assertRaisesRegex( - pip_checker.PipCheckerError, 'killed by signal 9'): + with patch_timeout, self.assertRaisesRegex(pip_checker.PipCheckerError, + 'killed by signal 9'): container = checker._run_container(MockDockerClient()) checker._run_command( - container, - ["sleep", "1"], + container, ["sleep", "1"], stdout=True, stderr=True, raise_on_failure=False) docker_view_name = views.DOCKER_ERROR_VIEW.name - docker_view_data = self._stats.view_manager.get_view( - docker_view_name) + docker_view_data = self._stats.view_manager.get_view(docker_view_name) docker_data_map = docker_view_data._tag_value_aggregation_data_map self.assertEqual(len(docker_data_map), 1) @@ -187,11 +182,9 @@ def test_success(self, mock_docker, mock__call_pypi_json_api): 'upload_time': '2018-06-13T18:29:51', }, ], - '1.2.3': [ - { - 'upload_time': '2018-05-10T15:00:00', - } - ], + '1.2.3': [{ + 'upload_time': '2018-05-10T15:00:00', + }], }, } @@ -205,15 +198,12 @@ def test_success(self, mock_docker, mock__call_pypi_json_api): check_result = pip_checker.check( pip_command=[ self._fake_pip_path, '--expected-install-args=-U,six', - '--freeze-output=six==1.2.3\n', - '--list-output={}'.format( + '--freeze-output=six==1.2.3\n', '--list-output={}'.format( json.dumps(expected_list_output)) ], packages=['six'], stats=self._stats) - self.assertEqual( - check_result, - expected_check_result) + self.assertEqual(check_result, expected_check_result) @mock.patch('pip_checker.docker.from_env') def test_install_failure(self, mock_docker): @@ -267,32 +257,26 @@ def test_check_warning(self, mock_docker, mock__call_pypi_json_api): 'upload_time': '2018-06-13T18:29:51', }, ], - '1.2.3': [ - { - 'upload_time': '2018-05-10T15:00:00', - } - ], + '1.2.3': [{ + 'upload_time': '2018-05-10T15:00:00', + }], }, } with patch: check_result = pip_checker.check( pip_command=[ - self._fake_pip_path, - '--check-returncode=1', + self._fake_pip_path, '--check-returncode=1', '--check-output=package has requirement A, but you have B', - '--freeze-output=six==1.2.3\n', - '--list-output={}'.format( + '--freeze-output=six==1.2.3\n', '--list-output={}'.format( json.dumps(expected_list_output)) ], packages=['six'], stats=self._stats) expected_check_result = pip_checker.PipCheckResult( - packages=['six'], - result_type=pip_checker.PipCheckResultType.CHECK_WARNING, - result_text='package has requirement A, but you have B', - dependency_info=expected_dependency_info) + packages=['six'], + result_type=pip_checker.PipCheckResultType.CHECK_WARNING, + result_text='package has requirement A, but you have B', + dependency_info=expected_dependency_info) - self.assertEqual( - check_result, - expected_check_result) + self.assertEqual(check_result, expected_check_result) diff --git a/compatibility_server/test_sanitize_packages.py b/compatibility_server/test_sanitize_packages.py index 677cf965..82bbb772 100644 --- a/compatibility_server/test_sanitize_packages.py +++ b/compatibility_server/test_sanitize_packages.py @@ -18,6 +18,7 @@ import pip_checker import unittest + class Test__sanitize_packages(unittest.TestCase): def test__sanitize_packages(self): @@ -49,8 +50,9 @@ def test_all_whitelists_pip_install(self): ('python3', py3_pkgs), ] for command, packages in args: - pip_result = pip_checker.check( - [command, '-m', 'pip'], packages, clean=True) + pip_result = pip_checker.check([command, '-m', 'pip'], + packages, + clean=True) self.assertIsNotNone(pip_result) results = dict( @@ -61,7 +63,7 @@ def test_all_whitelists_pip_install(self): if results['result'] == 'INSTALL_ERROR': logging.warning(command) logging.warning(results['description']) - self.assertFalse(results['result']=='INSTALL_ERROR') + self.assertFalse(results['result'] == 'INSTALL_ERROR') def test_nonwhitelisted_packages(self): packages = [ diff --git a/compatibility_server/views.py b/compatibility_server/views.py index a1162cc7..b550d018 100644 --- a/compatibility_server/views.py +++ b/compatibility_server/views.py @@ -19,12 +19,10 @@ # docker error DOCKER_ERROR_MEASURE = measure_module.MeasureInt( 'docker_error', 'The number of docker errors.', 'Errors') -DOCKER_ERROR_VIEW = view_module.View( - "docker_error_count", - "The number of the docker errors", - [], - DOCKER_ERROR_MEASURE, - aggregation_module.CountAggregation()) +DOCKER_ERROR_VIEW = view_module.View("docker_error_count", + "The number of the docker errors", [], + DOCKER_ERROR_MEASURE, + aggregation_module.CountAggregation()) ALL_VIEWS = [ DOCKER_ERROR_VIEW, diff --git a/dashboard/dashboard_builder.py b/dashboard/dashboard_builder.py index 9ddabf5c..5ca6bc58 100644 --- a/dashboard/dashboard_builder.py +++ b/dashboard/dashboard_builder.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Creates a web page that shows the compatibility between packages as a grid. For example: @@ -65,18 +64,15 @@ class DashboardBuilderError(Exception): class _ResultHolder(object): - def __init__( - self, - package_to_results: - Mapping[package.Package, - List[compatibility_store.CompatibilityResult]], - pairwise_to_results: - Mapping[FrozenSet[package.Package], - List[compatibility_store.CompatibilityResult]], - package_with_dependency_info=None, - checker=None, - store=None - ): + + def __init__(self, + package_to_results: Mapping[package.Package, List[ + compatibility_store.CompatibilityResult]], + pairwise_to_results: Mapping[FrozenSet[package.Package], List[ + compatibility_store.CompatibilityResult]], + package_with_dependency_info=None, + checker=None, + store=None): self._package_to_results = package_to_results self._pairwise_to_results = pairwise_to_results self._package_with_dependency_info = package_with_dependency_info @@ -227,8 +223,7 @@ def get_package_details(self, p: package.Package): return result - def get_result(self, - package_1: package.Package, + def get_result(self, package_1: package.Package, package_2: package.Package) -> Mapping[str, Any]: """Returns the installation result of two packages. @@ -255,68 +250,61 @@ def get_result(self, if (not self._package_to_results[package_1] or not self._package_to_results[package_2]): - self_result.append( - { - 'status': compatibility_store.Status.UNKNOWN.name, - 'self': True, - } - ) + self_result.append({ + 'status': compatibility_store.Status.UNKNOWN.name, + 'self': True, + }) status_type = 'self-unknown' - package_results = ( - self._package_to_results[package_1] + - self._package_to_results[package_2]) + package_results = (self._package_to_results[package_1] + + self._package_to_results[package_2]) for pr in package_results: if not self._is_py_version_incompatible(pr) and \ pr.status != compatibility_store.Status.SUCCESS: - self_result.append( - { - 'status': pr.status.value, - 'self': True, - 'details': pr.details - } - ) + self_result.append({ + 'status': pr.status.value, + 'self': True, + 'details': pr.details + }) status_type = 'self-' + pr.status.value.lower() if package_1 == package_2: if not self_result: - self_result.append( - { - 'status': compatibility_store.Status.SUCCESS.name, - 'self': True, - } - ) + self_result.append({ + 'status': + compatibility_store.Status.SUCCESS.name, + 'self': + True, + }) else: - pairwise_results = self._pairwise_to_results[ - frozenset([package_1, package_2])] + pairwise_results = self._pairwise_to_results[frozenset( + [package_1, package_2])] if not pairwise_results: - pair_result.append( - { - 'status': compatibility_store.Status.UNKNOWN.name, - 'self': False, - } - ) + pair_result.append({ + 'status': + compatibility_store.Status.UNKNOWN.name, + 'self': + False, + }) status_type = 'pairwise-unknown' for pr in pairwise_results: if not self._is_py_version_incompatible(pr) and \ pr.status != compatibility_store.Status.SUCCESS: - pair_result.append( - { - 'status': pr.status.value, - 'self': False, - 'details': pr.details - } - ) + pair_result.append({ + 'status': pr.status.value, + 'self': False, + 'details': pr.details + }) status_type = 'pairwise-' + pr.status.value.lower() if not pair_result: - pair_result.append( - { - 'status': compatibility_store.Status.SUCCESS.name, - 'self': False, - } - ) + pair_result.append({ + 'status': + compatibility_store.Status.SUCCESS.name, + 'self': + False, + }) if status_type == 'self-success': status_type = 'pairwise-success' @@ -352,11 +340,12 @@ def build_dashboard(self, template_name) -> str: def main(): parser = argparse.ArgumentParser( description='Display a grid show the dependency compatibility ' + - 'between Python packages') - parser.add_argument('--packages', nargs='+', - default=_DEFAULT_INSTALL_NAMES, - help='the packages to display compatibility ' + - 'information for') + 'between Python packages') + parser.add_argument( + '--packages', + nargs='+', + default=_DEFAULT_INSTALL_NAMES, + help='the packages to display compatibility ' + 'information for') parser.add_argument( '--browser', action='store_true', @@ -368,8 +357,8 @@ def main(): checker = compatibility_checker.CompatibilityChecker() store = compatibility_store.CompatibilityStore() - instance_flag = '-instances={}=tcp:{}'.format( - INSTANCE_CONNECTION_NAME, PORT) + instance_flag = '-instances={}=tcp:{}'.format(INSTANCE_CONNECTION_NAME, + PORT) cloud_sql_proxy_path = './cloud_sql_proxy' try: @@ -378,7 +367,8 @@ def main(): process.expect('Ready for new connection', timeout=5) packages = [ - package.Package(install_name) for install_name in args.packages] + package.Package(install_name) for install_name in args.packages + ] logging.info('Getting self compatibility results...') package_to_results = store.get_self_compatibilities(packages) logging.info('Getting pairwise compatibility results...') @@ -389,12 +379,8 @@ def main(): dep_info = store.get_dependency_info(pkg) package_with_dependency_info[pkg] = dep_info - results = _ResultHolder( - package_to_results, - pairwise_to_results, - package_with_dependency_info, - checker, - store) + results = _ResultHolder(package_to_results, pairwise_to_results, + package_with_dependency_info, checker, store) dashboard_builder = DashboardBuilder(packages, results) diff --git a/dashboard/test_dashboard_builder.py b/dashboard/test_dashboard_builder.py index ce3a3aad..3f1d68bd 100644 --- a/dashboard/test_dashboard_builder.py +++ b/dashboard/test_dashboard_builder.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Tests for dashboard_builder.""" import mock @@ -36,13 +35,13 @@ def __init__(self, py_version='3', checker=None, store=None): self.store = store def get_deprecated_deps(self, packages=None): - deprecated_deps = [ - (('gsutil', ['gcs-oauth2-boto-plugin', 'oauth2client']),), - (('opencensus', []),), - (('package1', []),), - (('package2', []),), - (('package3', ['deprecated_dep1', 'deprecated_dep2']),), - (('gcloud', ['oauth2client']),)] + deprecated_deps = [(('gsutil', + ['gcs-oauth2-boto-plugin', 'oauth2client']),), + (('opencensus', []),), (('package1', []),), + (('package2', []),), + (('package3', ['deprecated_dep1', + 'deprecated_dep2']),), + (('gcloud', ['oauth2client']),)] return deprecated_deps @@ -58,23 +57,30 @@ def check_packages(self, packages, max_workers=20): results = { 'opencensus': [], 'google-gax': [mock.Mock(name='ply', priority=2)], - 'oauth2client': []} + 'oauth2client': [] + } return results class TestResultHolderGetResult(unittest.TestCase): """Tests for dashboard_builder._ResultHolder.get_result().""" - patch_finder = mock.patch('dashboard_builder.deprecated_dep_finder.DeprecatedDepFinder', _DeprecatedDepFinder) - patch_highlighter = mock.patch('dashboard_builder.dependency_highlighter.DependencyHighlighter', _DependencyHighlighter) + patch_finder = mock.patch( + 'dashboard_builder.deprecated_dep_finder.DeprecatedDepFinder', + _DeprecatedDepFinder) + patch_highlighter = mock.patch( + 'dashboard_builder.dependency_highlighter.DependencyHighlighter', + _DependencyHighlighter) def test_self_compatibility_success(self): package_to_results = { - PACKAGE_1: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_1], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )] + PACKAGE_1: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_1], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ] } with self.patch_finder, self.patch_highlighter: @@ -83,44 +89,47 @@ def test_self_compatibility_success(self): expected = { 'status_type': 'self-success', - 'self_compatibility_check': [ - {'status': 'SUCCESS', 'self': True}], + 'self_compatibility_check': [{ + 'status': 'SUCCESS', + 'self': True + }], 'pairwise_compatibility_check': [] } - self.assertEqual( - rh.get_result(PACKAGE_1, PACKAGE_1), - expected) + self.assertEqual(rh.get_result(PACKAGE_1, PACKAGE_1), expected) def test_self_compatibility_error(self): package_to_results = { - PACKAGE_1: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_1], - python_major_version=3, - status=compatibility_store.Status.INSTALL_ERROR, - details='Installation failure', - )] + PACKAGE_1: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_1], + python_major_version=3, + status=compatibility_store.Status.INSTALL_ERROR, + details='Installation failure', + ) + ] } with self.patch_finder, self.patch_highlighter: rh = dashboard_builder._ResultHolder( package_to_results=package_to_results, pairwise_to_results={}) expected = { - 'status_type': 'self-install_error', + 'status_type': + 'self-install_error', 'self_compatibility_check': [ - {'status': 'INSTALL_ERROR', - 'self': True, - 'details': 'Installation failure' - }, - {'status': 'INSTALL_ERROR', - 'self': True, - 'details': 'Installation failure' - }, + { + 'status': 'INSTALL_ERROR', + 'self': True, + 'details': 'Installation failure' + }, + { + 'status': 'INSTALL_ERROR', + 'self': True, + 'details': 'Installation failure' + }, ], 'pairwise_compatibility_check': [] } - self.assertEqual( - rh.get_result(PACKAGE_1, PACKAGE_1), - expected) + self.assertEqual(rh.get_result(PACKAGE_1, PACKAGE_1), expected) def test_self_compatibility_no_entry(self): package_to_results = {PACKAGE_1: []} @@ -132,26 +141,31 @@ def test_self_compatibility_no_entry(self): expected = { 'status_type': 'self-unknown', 'self_compatibility_check': [ - {'status': 'UNKNOWN', 'self': True}, + { + 'status': 'UNKNOWN', + 'self': True + }, ], 'pairwise_compatibility_check': [] } - self.assertEqual( - rh.get_result(PACKAGE_1, PACKAGE_1), - expected) + self.assertEqual(rh.get_result(PACKAGE_1, PACKAGE_1), expected) def test_pairwise_success(self): package_to_results = { - PACKAGE_1: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_1], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )], - PACKAGE_2: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_2], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )] + PACKAGE_1: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_1], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ], + PACKAGE_2: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_2], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ] } pairwise_to_results = { frozenset([PACKAGE_1, PACKAGE_2]): [ @@ -159,7 +173,8 @@ def test_pairwise_success(self): packages=[PACKAGE_1, PACKAGE_2], python_major_version=3, status=compatibility_store.Status.SUCCESS, - )] + ) + ] } with self.patch_finder, self.patch_highlighter: @@ -170,26 +185,29 @@ def test_pairwise_success(self): expected = { 'status_type': 'pairwise-success', 'self_compatibility_check': [], - 'pairwise_compatibility_check': [ - {'status': 'SUCCESS', 'self': False} - ] + 'pairwise_compatibility_check': [{ + 'status': 'SUCCESS', + 'self': False + }] } - self.assertEqual( - rh.get_result(PACKAGE_1, PACKAGE_2), - expected) + self.assertEqual(rh.get_result(PACKAGE_1, PACKAGE_2), expected) def test_pairwise_error(self): package_to_results = { - PACKAGE_1: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_1], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )], - PACKAGE_2: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_2], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )] + PACKAGE_1: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_1], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ], + PACKAGE_2: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_2], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ] } pairwise_to_results = { frozenset([PACKAGE_1, PACKAGE_2]): [ @@ -198,7 +216,8 @@ def test_pairwise_error(self): python_major_version=3, status=compatibility_store.Status.INSTALL_ERROR, details='Installation failure', - )] + ) + ] } with self.patch_finder, self.patch_highlighter: @@ -206,35 +225,35 @@ def test_pairwise_error(self): package_to_results=package_to_results, pairwise_to_results=pairwise_to_results) expected = { - 'status_type': 'pairwise-install_error', + 'status_type': + 'pairwise-install_error', 'self_compatibility_check': [], - 'pairwise_compatibility_check': [ - {'status': 'INSTALL_ERROR', - 'self': False, - 'details': 'Installation failure' - } - ] + 'pairwise_compatibility_check': [{ + 'status': 'INSTALL_ERROR', + 'self': False, + 'details': 'Installation failure' + }] } - self.assertEqual( - rh.get_result(PACKAGE_1, PACKAGE_2), - expected) + self.assertEqual(rh.get_result(PACKAGE_1, PACKAGE_2), expected) def test_pairwise_no_entry(self): package_to_results = { - PACKAGE_1: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_1], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )], - PACKAGE_2: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_2], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )] - } - pairwise_to_results = { - frozenset([PACKAGE_1, PACKAGE_2]): [] + PACKAGE_1: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_1], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ], + PACKAGE_2: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_2], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ] } + pairwise_to_results = {frozenset([PACKAGE_1, PACKAGE_2]): []} with self.patch_finder, self.patch_highlighter: rh = dashboard_builder._ResultHolder( @@ -243,13 +262,12 @@ def test_pairwise_no_entry(self): expected = { 'status_type': 'pairwise-unknown', 'self_compatibility_check': [], - 'pairwise_compatibility_check': [ - {'status': 'UNKNOWN', 'self': False} - ] + 'pairwise_compatibility_check': [{ + 'status': 'UNKNOWN', + 'self': False + }] } - self.assertEqual( - rh.get_result(PACKAGE_1, PACKAGE_2), - expected) + self.assertEqual(rh.get_result(PACKAGE_1, PACKAGE_2), expected) def test_get_package_details(self): package_with_dependency_info = { @@ -263,35 +281,38 @@ def test_get_package_details(self): } } package_to_results = { - PACKAGE_1: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_1], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )], - PACKAGE_2: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_2], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )] - } - pairwise_to_results = { - frozenset([PACKAGE_1, PACKAGE_2]): [] + PACKAGE_1: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_1], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ], + PACKAGE_2: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_2], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ] } + pairwise_to_results = {frozenset([PACKAGE_1, PACKAGE_2]): []} - patch_pkg_list = mock.patch( - 'dashboard_builder.configs.PKG_LIST', ['package1', 'package2']) + patch_pkg_list = mock.patch('dashboard_builder.configs.PKG_LIST', + ['package1', 'package2']) with self.patch_finder, self.patch_highlighter, patch_pkg_list: rh = dashboard_builder._ResultHolder( package_to_results=package_to_results, pairwise_to_results=pairwise_to_results, - package_with_dependency_info = package_with_dependency_info) + package_with_dependency_info=package_with_dependency_info) result = rh.get_package_details(PACKAGE_1) expected = { 'self_conflict': False, 'pairwise_conflict': ['package2'], - 'latest_version': '1.2.0'} + 'latest_version': '1.2.0' + } self.assertEqual(result, expected) @@ -306,16 +327,20 @@ class TestResultHolderHasIssues(unittest.TestCase): def test_no_issues(self): package_to_results = { - PACKAGE_1: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_1], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )], - PACKAGE_2: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_2], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )] + PACKAGE_1: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_1], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ], + PACKAGE_2: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_2], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ] } pairwise_to_results = { frozenset([PACKAGE_1, PACKAGE_2]): [ @@ -323,7 +348,8 @@ def test_no_issues(self): packages=[PACKAGE_1, PACKAGE_2], python_major_version=3, status=compatibility_store.Status.SUCCESS, - )] + ) + ] } with self.patch_finder, self.patch_highlighter: @@ -335,17 +361,21 @@ def test_no_issues(self): def test_self_issues(self): package_to_results = { - PACKAGE_1: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_1], - python_major_version=3, - status=compatibility_store.Status.CHECK_WARNING, - details='Self Conflict', - )], - PACKAGE_2: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_2], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )], + PACKAGE_1: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_1], + python_major_version=3, + status=compatibility_store.Status.CHECK_WARNING, + details='Self Conflict', + ) + ], + PACKAGE_2: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_2], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ], } pairwise_to_results = { frozenset([PACKAGE_1, PACKAGE_2]): [ @@ -354,7 +384,8 @@ def test_self_issues(self): python_major_version=3, status=compatibility_store.Status.CHECK_WARNING, details='Conflict', - )], + ) + ], } with self.patch_finder, self.patch_highlighter: @@ -366,16 +397,20 @@ def test_self_issues(self): def test_pairwise_issues(self): package_to_results = { - PACKAGE_1: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_1], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )], - PACKAGE_2: [compatibility_store.CompatibilityResult( - packages=[PACKAGE_2], - python_major_version=3, - status=compatibility_store.Status.SUCCESS, - )] + PACKAGE_1: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_1], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ], + PACKAGE_2: [ + compatibility_store.CompatibilityResult( + packages=[PACKAGE_2], + python_major_version=3, + status=compatibility_store.Status.SUCCESS, + ) + ] } pairwise_to_results = { frozenset([PACKAGE_1, PACKAGE_2]): [ @@ -384,7 +419,8 @@ def test_pairwise_issues(self): python_major_version=3, status=compatibility_store.Status.INSTALL_ERROR, details='Installation failure', - )] + ) + ] } with self.patch_finder, self.patch_highlighter: @@ -412,23 +448,19 @@ def test_success(self): compatibility_store.CompatibilityResult( packages=[PACKAGE_1], python_major_version=3, - status=compatibility_store.Status.SUCCESS - ), + status=compatibility_store.Status.SUCCESS), compatibility_store.CompatibilityResult( packages=[PACKAGE_2], python_major_version=3, - status=compatibility_store.Status.SUCCESS - ), + status=compatibility_store.Status.SUCCESS), compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=3, - status=compatibility_store.Status.SUCCESS - ), + status=compatibility_store.Status.SUCCESS), ]) with self.patch_finder, self.patch_highlighter: package_to_results = store.get_self_compatibilities(packages) - pairwise_to_results = store.get_compatibility_combinations( - packages) + pairwise_to_results = store.get_compatibility_combinations(packages) results = dashboard_builder._ResultHolder(package_to_results, pairwise_to_results) builder = dashboard_builder.DashboardBuilder(packages, results) @@ -443,19 +475,16 @@ def test_self_failure(self): packages=[PACKAGE_1], python_major_version=3, status=compatibility_store.Status.INSTALL_ERROR, - details="Installation failure" - ), + details="Installation failure"), compatibility_store.CompatibilityResult( packages=[PACKAGE_2], python_major_version=3, - status=compatibility_store.Status.SUCCESS - ), + status=compatibility_store.Status.SUCCESS), ]) with self.patch_finder, self.patch_highlighter: package_to_results = store.get_self_compatibilities(packages) - pairwise_to_results = store.get_compatibility_combinations( - packages) + pairwise_to_results = store.get_compatibility_combinations(packages) results = dashboard_builder._ResultHolder(package_to_results, pairwise_to_results) builder = dashboard_builder.DashboardBuilder(packages, results) @@ -470,19 +499,16 @@ def test_missing_pairwise(self): compatibility_store.CompatibilityResult( packages=[PACKAGE_1], python_major_version=3, - status=compatibility_store.Status.SUCCESS - ), + status=compatibility_store.Status.SUCCESS), compatibility_store.CompatibilityResult( packages=[PACKAGE_2], python_major_version=3, - status=compatibility_store.Status.SUCCESS - ), + status=compatibility_store.Status.SUCCESS), ]) with self.patch_finder, self.patch_highlighter: package_to_results = store.get_self_compatibilities(packages) - pairwise_to_results = store.get_compatibility_combinations( - packages) + pairwise_to_results = store.get_compatibility_combinations(packages) results = dashboard_builder._ResultHolder(package_to_results, pairwise_to_results) builder = dashboard_builder.DashboardBuilder(packages, results) @@ -496,14 +522,12 @@ def test_missing_self(self): compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=3, - status=compatibility_store.Status.SUCCESS - ), + status=compatibility_store.Status.SUCCESS), ]) with self.patch_finder, self.patch_highlighter: package_to_results = store.get_self_compatibilities(packages) - pairwise_to_results = store.get_compatibility_combinations( - packages) + pairwise_to_results = store.get_compatibility_combinations(packages) results = dashboard_builder._ResultHolder(package_to_results, pairwise_to_results) builder = dashboard_builder.DashboardBuilder(packages, results) @@ -517,25 +541,21 @@ def test_pairwise_failure(self): compatibility_store.CompatibilityResult( packages=[PACKAGE_1], python_major_version=3, - status=compatibility_store.Status.SUCCESS - ), + status=compatibility_store.Status.SUCCESS), compatibility_store.CompatibilityResult( packages=[PACKAGE_2], python_major_version=3, - status=compatibility_store.Status.SUCCESS - ), + status=compatibility_store.Status.SUCCESS), compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_2], python_major_version=3, status=compatibility_store.Status.INSTALL_ERROR, - details="Installation failure" - ), + details="Installation failure"), ]) with self.patch_finder, self.patch_highlighter: package_to_results = store.get_self_compatibilities(packages) - pairwise_to_results = store.get_compatibility_combinations( - packages) + pairwise_to_results = store.get_compatibility_combinations(packages) results = dashboard_builder._ResultHolder(package_to_results, pairwise_to_results) builder = dashboard_builder.DashboardBuilder(packages, results) @@ -552,30 +572,26 @@ def test_not_show_py_ver_incompatible_results(self): compatibility_store.CompatibilityResult( packages=[PACKAGE_1], python_major_version=3, - status=compatibility_store.Status.SUCCESS - ), + status=compatibility_store.Status.SUCCESS), compatibility_store.CompatibilityResult( packages=[PACKAGE_3], python_major_version=3, - status=compatibility_store.Status.INSTALL_ERROR - ), + status=compatibility_store.Status.INSTALL_ERROR), compatibility_store.CompatibilityResult( packages=[PACKAGE_1, PACKAGE_3], python_major_version=3, status=compatibility_store.Status.INSTALL_ERROR, - details="Installation failure" - ), + details="Installation failure"), ]) patch = mock.patch( 'compatibility_lib.configs.PKG_PY_VERSION_NOT_SUPPORTED', { - 2: ['package4'], - 3: ['package3'], - }) + 2: ['package4'], + 3: ['package3'], + }) with patch, self.patch_finder, self.patch_highlighter: package_to_results = store.get_self_compatibilities(packages) - pairwise_to_results = store.get_compatibility_combinations( - packages) + pairwise_to_results = store.get_compatibility_combinations(packages) results = dashboard_builder._ResultHolder(package_to_results, pairwise_to_results) diff --git a/nox.py b/nox.py index f9c5a312..276f37e7 100644 --- a/nox.py +++ b/nox.py @@ -25,16 +25,10 @@ @nox.session def lint(session): - """Run flake8. - - Returns a failure if flake8 finds linting errors or sufficiently - serious code quality issues. - """ + """Run yapf and return an error if yapf formatting is not used.""" session.interpreter = 'python3.6' - session.install('flake8') - session.run('flake8', - '--exclude=--exclude=__pycache__,dist,.git,' - 'build,.tox,.nox,.idea,mock_*,test_*,*_test') + session.install('yapf') + session.run('yapf', '--diff', '-r', 'badge_server', 'compatibility_lib', 'compatibility_server', 'dashboard', 'scripts', 'system_test') @nox.session diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..494f106c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[yapf] +based_on_style = google diff --git a/system_test/test_badge_server.py b/system_test/test_badge_server.py index 0390bc01..9f271386 100644 --- a/system_test/test_badge_server.py +++ b/system_test/test_badge_server.py @@ -26,8 +26,8 @@ BASE_URL = 'http://0.0.0.0:8080/' PACKAGE_FOR_TEST = 'opencensus' -RETRY_WAIT_PERIOD = 8000 # Wait 8 seconds between each retry -RETRY_MAX_ATTEMPT = 10 # Retry 10 times +RETRY_WAIT_PERIOD = 8000 # Wait 8 seconds between each retry +RETRY_MAX_ATTEMPT = 10 # Retry 10 times def wait_app_to_start(): @@ -40,10 +40,7 @@ def run_application(): """Start running the compatibility checker server.""" cmd = 'python badge_server/main.py ' process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - shell=True, - preexec_fn=os.setsid) + cmd, stdout=subprocess.PIPE, shell=True, preexec_fn=os.setsid) return process @@ -60,8 +57,8 @@ def tearDown(self): # Kill the application process os.killpg(os.getpgid(self.process.pid), signal.SIGTERM) - @retry(wait_fixed=RETRY_WAIT_PERIOD, - stop_max_attempt_number=RETRY_MAX_ATTEMPT) + @retry( + wait_fixed=RETRY_WAIT_PERIOD, stop_max_attempt_number=RETRY_MAX_ATTEMPT) def test_one_badge(self): response = requests.get('{}/'.format(BASE_URL)) status_code = response.status_code diff --git a/system_test/test_compatibility_checker_server.py b/system_test/test_compatibility_checker_server.py index ec3b6c81..0b26dd37 100644 --- a/system_test/test_compatibility_checker_server.py +++ b/system_test/test_compatibility_checker_server.py @@ -26,14 +26,14 @@ BASE_URL = 'http://0.0.0.0:8888/' PACKAGE_FOR_TEST = 'opencensus' -RETRY_WAIT_PERIOD = 8000 # Wait 8 seconds between each retry -RETRY_MAX_ATTEMPT = 10 # Retry 10 times +RETRY_WAIT_PERIOD = 8000 # Wait 8 seconds between each retry +RETRY_MAX_ATTEMPT = 10 # Retry 10 times def wait_app_to_start(): """Wait the application to start running.""" - url = '{}?package={}&python-version={}'.format( - BASE_URL, PACKAGE_FOR_TEST, 3) + url = '{}?package={}&python-version={}'.format(BASE_URL, PACKAGE_FOR_TEST, + 3) cmd = 'wget --retry-connrefused --tries=5 \'{}\''.format(url) subprocess.check_call(cmd, shell=True) @@ -43,10 +43,7 @@ def run_application(): cmd = 'python compatibility_server/compatibility_checker_server.py ' \ '--host=\'0.0.0.0\' --port=8888' process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - shell=True, - preexec_fn=os.setsid) + cmd, stdout=subprocess.PIPE, shell=True, preexec_fn=os.setsid) return process @@ -63,17 +60,18 @@ def tearDown(self): # Kill the flask application process os.killpg(os.getpgid(self.process.pid), signal.SIGTERM) - @retry(wait_fixed=RETRY_WAIT_PERIOD, - stop_max_attempt_number=RETRY_MAX_ATTEMPT) + @retry( + wait_fixed=RETRY_WAIT_PERIOD, stop_max_attempt_number=RETRY_MAX_ATTEMPT) def test_check_compatibility(self): response = requests.get('{}?package={}&python-version={}'.format( BASE_URL, PACKAGE_FOR_TEST, 3)) status_code = response.status_code content = response.content.decode('utf-8') - content_dict = ast.literal_eval(content.replace( - 'true', '"true"').replace( - 'false', '"false"').replace('null', '"null"')) + content_dict = ast.literal_eval( + content.replace('true', '"true"').replace('false', + '"false"').replace( + 'null', '"null"')) self.assertEqual(status_code, 200) self.assertEqual(content_dict['packages'], [PACKAGE_FOR_TEST])