Merge branch 'master' into string_key_column

Author: erezsh

README.md

@@ -110,22 +110,22 @@ $ data-diff \
 
 ## Supported Databases
 
-| Database      | Connection string                                                                       | Status |
-|---------------|-----------------------------------------------------------------------------------------|--------|
-| PostgreSQL    | `postgresql://user:password@hostname:5432/database`                                     |  💚    |
-| MySQL         | `mysql://user:password@hostname:5432/database`                                          |  💚    |
-| Snowflake     | `snowflake://user:password@account/database/SCHEMA?warehouse=WAREHOUSE&role=role`       |  💚    |
-| Oracle        | `oracle://username:password@hostname/database`                                          |  💛    |
-| BigQuery      | `bigquery://project/dataset`                                                            |  💛    |
-| Redshift      | `redshift://username:password@hostname:5439/database`                                   |  💛    |
-| Presto        | `presto://username:password@hostname:8080/database`                                     |  💛    |
-| ElasticSearch |                                                                                         |  📝    |
-| Databricks    |                                                                                         |  📝    |
-| Planetscale   |                                                                                         |  📝    |
-| Clickhouse    |                                                                                         |  📝    |
-| Pinot         |                                                                                         |  📝    |
-| Druid         |                                                                                         |  📝    |
-| Kafka         |                                                                                         |  📝    |
+| Database      | Connection string                                                                                | Status |
+|---------------|--------------------------------------------------------------------------------------------------|--------|
+| PostgreSQL    | `postgresql://<user>:<password>@<hostname>:5432/<database>`                                      |  💚    |
+| MySQL         | `mysql://<user>:<password>@<hostname>:5432/<database>`                                           |  💚    |
+| Snowflake     | `"snowflake://<user>:<password>@<account>/<database>/<SCHEMA>?warehouse=<WAREHOUSE>&role=<role>"`|  💚    |
+| Oracle        | `oracle://<username>:<password>@<hostname>/database`                                             |  💛    |
+| BigQuery      | `bigquery://<project>/<dataset>`                                                                 |  💛    |
+| Redshift      | `redshift://<username>:<password>@<hostname>:5439/<database>`                                    |  💛    |
+| Presto        | `presto://<username>:<password>@<hostname>:8080/<database>`                                      |  💛    |
+| ElasticSearch |                                                                                                  |  📝    |
+| Databricks    |                                                                                                  |  📝    |
+| Planetscale   |                                                                                                  |  📝    |
+| Clickhouse    |                                                                                                  |  📝    |
+| Pinot         |                                                                                                  |  📝    |
+| Druid         |                                                                                                  |  📝    |
+| Kafka         |                                                                                                  |  📝    |
 
 * 💚: Implemented and thoroughly tested.
 * 💛: Implemented, but not thoroughly tested yet.
@@ -171,7 +171,10 @@ Users can also install several drivers at once:
 Usage: `data-diff DB1_URI TABLE1_NAME DB2_URI TABLE2_NAME [OPTIONS]`
 
 See the [example command](#example-command-and-output) and the [sample
-connection strings](#supported-databases).
+connection strings](#supported-databases). 
+
+Note that for some databases, the arguments that you enter in the command line 
+may be case-sensitive. This is the case for the Snowflake schema and table names.
 
 Options:
 

data_diff/databases/mysql.py

@@ -20,6 +20,7 @@ class MySQL(ThreadedDatabase):
         "float": Float,
         "decimal": Decimal,
         "int": Integer,
+        "bigint": Integer,
         # Text
         "varchar": Text,
         "char": Text,

data_diff/databases/oracle.py

@@ -27,6 +27,9 @@ class Oracle(ThreadedDatabase):
     def __init__(self, host, port, user, password, *, database, thread_count, **kw):
         assert not port
         self.kwargs = dict(user=user, password=password, dsn="%s/%s" % (host, database), **kw)
+
+        self.default_schema = user
+
         super().__init__(thread_count=thread_count)
 
     def create_connection(self):
@@ -54,13 +57,11 @@ def to_string(self, s: str):
         return f"cast({s} as varchar(1024))"
 
     def select_table_schema(self, path: DbPath) -> str:
-        if len(path) > 1:
-            raise ValueError("Unexpected table path for oracle")
-        (table,) = path
+        schema, table = self._normalize_table_path(path)
 
         return (
             f"SELECT column_name, data_type, 6 as datetime_precision, data_precision as numeric_precision, data_scale as numeric_scale"
-            f" FROM USER_TAB_COLUMNS WHERE table_name = '{table.upper()}'"
+            f" FROM ALL_TAB_COLUMNS WHERE table_name = '{table.upper()}' AND owner = '{schema.upper()}'"
         )
 
     def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:

data_diff/sql.py

@@ -120,8 +120,12 @@ class Checksum(Sql):
     exprs: Sequence[SqlOrStr]
 
     def compile(self, c: Compiler):
-        compiled_exprs = ", ".join(map(c.compile, self.exprs))
-        expr = f"concat({compiled_exprs})"
+        if len(self.exprs) > 1:
+            compiled_exprs = ", ".join(map(c.compile, self.exprs))
+            expr = f"concat({compiled_exprs})"
+        else:
+            expr ,= self.exprs
+            expr = c.compile(expr)
         md5 = c.database.md5_to_int(expr)
         return f"sum({md5})"
 

pyproject.toml

@@ -2,7 +2,7 @@
 name = "data-diff"
 version = "0.2.1"
 description = "Command-line tool and Python library to efficiently diff rows across two different databases."
-authors = ["Erez Shinnan <erezshin@gmail.com>", "Simon Eskildsen <simon@sirupsen.com>"]
+authors = ["Datafold <data-diff@datafold.com>"]
 license = "MIT"
 readme = "README.md"
 repository = "https://github.com/datafold/data-diff"

tests/test_database_types.py

@@ -183,7 +183,7 @@ def __iter__(self):
         "int": [
             # "smallint",  # 2 bytes
             "int",  # 4 bytes
-            # "bigint", # 8 bytes
+            "bigint", # 8 bytes
         ],
         # https://www.postgresql.org/docs/current/datatype-datetime.html
         "datetime_no_timezone": [
@@ -211,7 +211,7 @@ def __iter__(self):
             # "smallint", # 2 bytes
             # "mediumint", # 3 bytes
             "int",  # 4 bytes
-            # "bigint", # 8 bytes
+            "bigint", # 8 bytes
         ],
         # https://dev.mysql.com/doc/refman/8.0/en/datetime.html
         "datetime_no_timezone": [
@@ -253,8 +253,8 @@ def __iter__(self):
         "int": [
             # all 38 digits with 0 precision, don't need to test all
             "int",
-            # "integer",
-            # "bigint",
+            "integer",
+            "bigint",
             # "smallint",
             # "tinyint",
             # "byteint"
@@ -323,7 +323,7 @@ def __iter__(self):
             # "smallint", # 2 bytes
             # "mediumint", # 3 bytes
             "int",  # 4 bytes
-            # "bigint", # 8 bytes
+            "bigint", # 8 bytes
         ],
         "datetime_no_timezone": [
             "timestamp",