Merge pull request #202 from datafold/aug11

[Tests] now using connect() instead of connect_to_uri(); refactor

Author: erezsh

data_diff/diff_tables.py

@@ -91,7 +91,9 @@ def __post_init__(self):
             raise ValueError(f"Error: min_key expected to be smaller than max_key! ({self.min_key} >= {self.max_key})")
 
         if self.min_update is not None and self.max_update is not None and self.min_update >= self.max_update:
-            raise ValueError(f"Error: min_update expected to be smaller than max_update! ({self.min_update} >= {self.max_update})")
+            raise ValueError(
+                f"Error: min_update expected to be smaller than max_update! ({self.min_update} >= {self.max_update})"
+            )
 
     @property
     def _update_column(self):

data_diff/utils.py

@@ -5,6 +5,7 @@
 from abc import ABC, abstractmethod
 from urllib.parse import urlparse
 from uuid import UUID
+import operator
 import string
 
 alphanums = string.digits + string.ascii_lowercase
@@ -221,3 +222,19 @@ def match_like(pattern: str, strs: Sequence[str]) -> Iterable[str]:
     for s in strs:
         if reo.match(s):
             yield s
+
+
+def accumulate(iterable, func=operator.add, *, initial=None):
+    'Return running totals'
+    # Taken from https://docs.python.org/3/library/itertools.html#itertools.accumulate, to backport 'initial' to 3.7
+    it = iter(iterable)
+    total = initial
+    if initial is None:
+        try:
+            total = next(it)
+        except StopIteration:
+            return
+    yield total
+    for element in it:
+        total = func(total, element)
+        yield total

tests/test_database_types.py

@@ -8,11 +8,13 @@
 from datetime import datetime, timedelta, timezone
 import logging
 from decimal import Decimal
+from itertools import islice, repeat, chain
+
 from parameterized import parameterized
 
 from data_diff import databases as db
 from data_diff.databases import postgresql, oracle
-from data_diff.utils import number_to_human
+from data_diff.utils import number_to_human, accumulate
 from data_diff.diff_tables import TableDiffer, TableSegment, DEFAULT_BISECTION_THRESHOLD
 from .common import (
     CONN_STRINGS,
@@ -25,7 +27,7 @@
 )
 
 
-CONNS = {k: db.connect_to_uri(v, N_THREADS) for k, v in CONN_STRINGS.items()}
+CONNS = {k: db.connect.connect(v, N_THREADS) for k, v in CONN_STRINGS.items()}
 
 CONNS[db.MySQL].query("SET @@session.time_zone='+00:00'", None)
 oracle.SESSION_TIME_ZONE = postgresql.SESSION_TIME_ZONE = "UTC"
@@ -290,54 +292,28 @@ def __init__(self, max):
         self.max = max
 
     def __iter__(self):
-        iter = DateTimeFaker(self.max)
-        iter.prev = datetime(2000, 1, 1, 0, 0, 0, 0)
-        iter.i = 0
-        return iter
+        initial = datetime(2000, 1, 1, 0, 0, 0, 0)
+        step = timedelta(seconds=3, microseconds=571)
+        return islice(chain(self.MANUAL_FAKES, accumulate(repeat(step), initial=initial)), self.max)
 
     def __len__(self):
         return self.max
 
-    def __next__(self) -> datetime:
-        if self.i < len(self.MANUAL_FAKES):
-            fake = self.MANUAL_FAKES[self.i]
-            self.i += 1
-            return fake
-        elif self.i < self.max:
-            self.prev = self.prev + timedelta(seconds=3, microseconds=571)
-            self.i += 1
-            return self.prev
-        else:
-            raise StopIteration
-
 
 class IntFaker:
-    MANUAL_FAKES = [127, -3, -9, 37, 15, 127]
+    MANUAL_FAKES = [127, -3, -9, 37, 15, 0]
 
     def __init__(self, max):
         self.max = max
 
     def __iter__(self):
-        iter = IntFaker(self.max)
-        iter.prev = -128
-        iter.i = 0
-        return iter
+        initial = -128
+        step = 1
+        return islice(chain(self.MANUAL_FAKES, accumulate(repeat(step), initial=initial)), self.max)
 
     def __len__(self):
         return self.max
 
-    def __next__(self) -> int:
-        if self.i < len(self.MANUAL_FAKES):
-            fake = self.MANUAL_FAKES[self.i]
-            self.i += 1
-            return fake
-        elif self.i < self.max:
-            self.prev += 1
-            self.i += 1
-            return self.prev
-        else:
-            raise StopIteration
-
 
 class FloatFaker:
     MANUAL_FAKES = [
@@ -363,26 +339,13 @@ def __init__(self, max):
         self.max = max
 
     def __iter__(self):
-        iter = FloatFaker(self.max)
-        iter.prev = -10.0001
-        iter.i = 0
-        return iter
+        initial = -10.0001
+        step = 0.00571
+        return islice(chain(self.MANUAL_FAKES, accumulate(repeat(step), initial=initial)), self.max)
 
     def __len__(self):
         return self.max
 
-    def __next__(self) -> float:
-        if self.i < len(self.MANUAL_FAKES):
-            fake = self.MANUAL_FAKES[self.i]
-            self.i += 1
-            return fake
-        elif self.i < self.max:
-            self.prev += 0.00571
-            self.i += 1
-            return self.prev
-        else:
-            raise StopIteration
-
 
 class UUID_Faker:
     def __init__(self, max):

tests/test_diff_tables.py

@@ -6,7 +6,7 @@
 import preql
 import arrow  # comes with preql
 
-from data_diff.databases import connect_to_uri
+from data_diff.databases.connect import connect
 from data_diff.diff_tables import TableDiffer, TableSegment, split_space
 from data_diff import databases as db
 from data_diff.utils import ArithAlphanumeric
@@ -21,7 +21,7 @@
 )
 
 DATABASE_URIS = {k.__name__: v for k, v in CONN_STRINGS.items()}
-DATABASE_INSTANCES = {k.__name__: connect_to_uri(v, N_THREADS) for k, v in CONN_STRINGS.items()}
+DATABASE_INSTANCES = {k.__name__: connect(v, N_THREADS) for k, v in CONN_STRINGS.items()}
 
 TEST_DATABASES = {x.__name__ for x in (db.MySQL, db.PostgreSQL, db.Oracle, db.Redshift, db.Snowflake, db.BigQuery)}
 

tests/test_postgresql.py

@@ -1,13 +1,13 @@
 import unittest
 
-from data_diff.databases.connect import connect_to_uri
+from data_diff.databases.connect import connect
 from data_diff import TableSegment, TableDiffer
 from .common import TEST_POSTGRESQL_CONN_STRING, random_table_suffix
 
 
 class TestWithConnection(unittest.TestCase):
     def setUp(self) -> None:
-        self.connection = connect_to_uri(TEST_POSTGRESQL_CONN_STRING)
+        self.connection = connect(TEST_POSTGRESQL_CONN_STRING)
 
         self.connection.query('CREATE EXTENSION IF NOT EXISTS "uuid-ossp";', None)