Merge pull request #123 from oracle/dev/1.7.0

v1.7.0

Author: aosingh

.github/workflows/oracle-xe-adapter-tests.yml

@@ -48,7 +48,7 @@ jobs:
       - name: Install dbt-oracle with core dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest 'dbt-tests-adapter~=1.6,<1.7'
+          pip install pytest 'dbt-tests-adapter~=1.7,<1.8'
           pip install -r requirements.txt
           pip install -e .
 

.gitignore

@@ -147,4 +147,6 @@ doc/build.gitbak
 .venv1.4/
 .venv1.5/
 .venv1.6/
+.venv1.7/
+.venv*/
 dbt_adbs_py_test_project

Makefile

@@ -1,5 +1,5 @@
 # Configuration variables
-VERSION=1.6.1
+VERSION=1.7.0
 PROJ_DIR?=$(shell pwd)
 VENV_DIR?=${PROJ_DIR}/.bldenv
 BUILD_DIR=${PROJ_DIR}/build
@@ -36,6 +36,7 @@ adbs_local_env_test: wheel clean_venv
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt snapshot --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt snapshot --profiles-dir ./
+	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt show --limit 4 --select people --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt docs generate --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run-operation drop_schema --args 'relation: ${DBT_ORACLE_SCHEMA}' --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt clean --profiles-dir ./
@@ -57,6 +58,7 @@ adbs_pypi_test: clean_venv
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt snapshot --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt snapshot --profiles-dir ./
+	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt show --limit 4 --select people --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt docs generate --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run-operation drop_schema --args 'relation: ${DBT_ORACLE_SCHEMA}' --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt clean --profiles-dir ./

dbt/adapters/oracle/__version__.py

@@ -14,4 +14,4 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 """
-version = "1.6.7"
+version = "1.7.3"

dbt/adapters/oracle/connection_helper.py

@@ -100,13 +100,13 @@ class OracleDriverType(str, enum.Enum):
 SQLNET_ORA_CONFIG = None
 
 # Set the environment variable ORA_PYTHON_DRIVER_TYPE to one of “cx”, “thin”, or “thick”:
-ORA_PYTHON_DRIVER_TYPE = os.getenv('ORA_PYTHON_DRIVER_TYPE', 'cx').upper()
+ORA_PYTHON_DRIVER_TYPE = os.getenv('ORA_PYTHON_DRIVER_TYPE', 'thin').upper()
 if ORA_PYTHON_DRIVER_TYPE == OracleDriverType.CX_ORACLE:
     logger.info("Running in cx mode")
     description = (
         f"cx_oracle is no longer maintained, use python-oracledb"
         f"\n\nTo switch to python-oracledb set the environment variable ORA_PYTHON_DRIVER_TYPE=thin "
-        f"\n\nDefault value of ORA_PYTHON_DRIVER_TYPE will be switched to thin started with v1.7.0 "
+        f"\n\nStarting with dbt-oracle version 1.7, default value of ORA_PYTHON_DRIVER_TYPE is thin"
         f"\n\nRead the guideline here: "
         f"https://docs.getdbt.com/reference/warehouse-setups/oracle-setup#configure-the-python-driver-mode"
         f"\n\nDocumentation for python-oracledb can be found here: "

dbt/adapters/oracle/connections.py

@@ -82,10 +82,12 @@ class OracleAdapterCredentials(Credentials):
     of parameters profiled in the profile.
     """
     # Mandatory required arguments.
-    user: str
-    password: str
+    user: Optional[str] = None
+    password: Optional[str] = None
+
     # Specifying database is optional
-    database: Optional[str]
+    database: Optional[str] = None
+    schema: Optional[str] = None
 
     # OracleConnectionMethod.TNS
     tns_name: Optional[str] = None
@@ -212,7 +214,7 @@ def open(cls, connection):
         }
 
         if oracledb.__name__ == "oracledb":
-            conn_config['connection_id_prefix'] = 'dbt-oracle-'
+            conn_config['connection_id_prefix'] = f'dbt-oracle-{dbt_version}-'
 
         if credentials.shardingkey:
             conn_config['shardingkey'] = credentials.shardingkey
@@ -239,7 +241,7 @@ def open(cls, connection):
             handle = oracledb.connect(**conn_config)
             # client_identifier and module are saved in corresponding columns in v$session
             session_info = cls.get_session_info(credentials=credentials)
-            logger.info(f"Session info :{json.dumps(session_info)}")
+            logger.debug(f"Session info :{json.dumps(session_info)}")
             for k, v in session_info.items():
                 try:
                     setattr(handle, k, v)

dbt/adapters/oracle/python_submissions.py

@@ -17,6 +17,8 @@
 import http
 import json
 from typing import Dict
+import uuid
+import platform
 
 import requests
 import time
@@ -25,6 +27,7 @@
 from dbt.adapters.oracle import OracleAdapterCredentials
 from dbt.events import AdapterLogger
 from dbt.ui import red, green
+from dbt.version import __version__ as dbt_version
 
 # ADB-S OML Rest API minimum timeout is 1800 seconds
 DEFAULT_TIMEOUT_IN_SECONDS = 1800
@@ -140,6 +143,20 @@ def __init__(self,
         self.oml4py_client = OracleOML4PYClient(oml_cloud_service_url=credential.oml_cloud_service_url,
                                                 username=credential.user,
                                                 password=credential.password)
+        self.session_info = self.get_session_info(credentials=credential)
+
+    @staticmethod
+    def get_session_info(credentials):
+        default_action = "DBT RUN OML4PY"
+        default_client_identifier = f'dbt-oracle-client-{uuid.uuid4()}'
+        default_client_info = "_".join([platform.node(), platform.machine()])
+        default_module = f'dbt-{dbt_version}'
+        return {
+            "action": credentials.session_info.get("action", default_action),
+            "client_identifier": credentials.session_info.get("client_identifier", default_client_identifier),
+            "clientinfo": credentials.session_info.get("client_info", default_client_info),
+            "module": credentials.session_info.get("module", default_module)
+        }
 
     def schedule_async_job_and_wait_for_completion(self, data):
         logger.info(f"Running Python aysnc job using {data}")
@@ -196,7 +213,8 @@ def schedule_async_job_and_wait_for_completion(self, data):
 
     def __call__(self, *args, **kwargs):
         data = {
-            "service": self.service
+            "service": self.service,
+            "parameters": json.dumps(self.session_info)
         }
         if self.async_flag:
             data["asyncFlag"] = self.async_flag

dbt/include/oracle/macros/adapters.sql

@@ -300,29 +300,7 @@
 
 {% macro oracle__drop_relation(relation) -%}
   {% call statement('drop_relation', auto_begin=False) -%}
-   DECLARE
-     dne_942    EXCEPTION;
-     PRAGMA EXCEPTION_INIT(dne_942, -942);
-     attempted_ddl_on_in_use_GTT EXCEPTION;
-     pragma EXCEPTION_INIT(attempted_ddl_on_in_use_GTT, -14452);
-     mv_dne_12003 EXCEPTION;
-     PRAGMA EXCEPTION_INIT(mv_dne_12003, -12003);
-  BEGIN
-     SAVEPOINT start_transaction;
-     {%- if relation.is_materialized_view -%}
-     EXECUTE IMMEDIATE '{{ oracle__drop_materialized_view(relation) }}';
-     {%- else -%}
-     EXECUTE IMMEDIATE 'DROP {{ relation.type }} {{ relation }} cascade constraint';
-     {%- endif -%}
-     COMMIT;
-  EXCEPTION
-     WHEN attempted_ddl_on_in_use_GTT THEN
-        NULL; -- if it its a global temporary table, leave it alone.
-     WHEN dne_942 THEN
-        NULL;
-     WHEN mv_dne_12003 THEN
-        NULL;
-  END;
+   {{ oracle__get_drop_sql(relation) }}
   {%- endcall %}
 {% endmacro %}
 

dbt/include/oracle/macros/freshness.sql

@@ -0,0 +1,31 @@
+{#
+ Copyright (c) 2023, Oracle and/or its affiliates.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+#}
+{% macro oracle__collect_freshness(source, loaded_at_field, filter) %}
+  {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}
+    select
+      {% if loaded_at_field | upper == 'ORA_ROWSCN' %}
+      SCN_TO_TIMESTAMP(max(ORA_ROWSCN)) as max_loaded_at,
+      {% else %}
+      max({{ loaded_at_field }}) as max_loaded_at,
+      {% endif %}
+      {{ current_timestamp() }} as snapshotted_at
+    from {{ source }}
+    {% if filter %}
+    where {{ filter }}
+    {% endif %}
+  {% endcall %}
+  {{ return(load_result('collect_freshness')) }}
+{% endmacro %}

dbt/include/oracle/macros/materializations/materialized_view/materialized_view.sql

@@ -8,10 +8,6 @@
   DROP MATERIALIZED VIEW {{ relation }}
 {%- endmacro %}
 
-{% macro oracle__get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) %}
-    {{ oracle__drop_relation(existing_relation) }}
-    {{ oracle__get_create_materialized_view_as_sql(relation, sql) }}
-{% endmacro %}
 
 {% macro oracle__get_create_materialized_view_as_sql(relation, sql) %}
     {%- set materialized_view = relation.from_runtime_config(config) -%}
@@ -30,13 +26,10 @@
     backup_relation,
     intermediate_relation
 ) %}
-
-    -- apply a full refresh immediately if needed
     {% if configuration_changes.requires_full_refresh %}
-
-        {{ get_replace_materialized_view_as_sql(relation, sql, existing_relation, backup_relation, intermediate_relation) }}
-
-    -- otherwise apply individual changes as needed
+    {{- log('Applying REPLACE to: ' ~ existing_relation ) -}}
+    {{ oracle__drop_relation(existing_relation) }}
+    {{ oracle__get_create_materialized_view_as_sql(relation, sql) }}
     {% else %}
 
         {%- set refresh_method = configuration_changes.refresh_method -%}
@@ -80,3 +73,43 @@
     {% do return({'materialized_view': _materialized_view}) %}
 {%- endmacro %}
 
+
+{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}
+
+    {% set full_refresh_mode = should_full_refresh() %}
+
+    -- determine the scenario we're in: create, full_refresh, alter, refresh data
+    {% if existing_relation is none %}
+        {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}
+    {% elif full_refresh_mode or not existing_relation.is_materialized_view %}
+        {{- log('Applying REPLACE to: ' ~ existing_relation ) -}}
+        {{ oracle__drop_relation(existing_relation) }}
+        {% set build_sql = oracle__get_create_materialized_view_as_sql(target_relation, sql) %}
+    {% else %}
+
+        -- get config options
+        {% set on_configuration_change = config.get('on_configuration_change') %}
+        {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}
+
+        {% if configuration_changes is none %}
+            {% set build_sql = refresh_materialized_view(target_relation) %}
+
+        {% elif on_configuration_change == 'apply' %}
+            {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}
+        {% elif on_configuration_change == 'continue' %}
+            {% set build_sql = '' %}
+            {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation ~ "`") }}
+        {% elif on_configuration_change == 'fail' %}
+            {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation ~ "`") }}
+
+        {% else %}
+            -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'
+            {{ exceptions.raise_compiler_error("Unexpected configuration scenario") }}
+
+        {% endif %}
+
+    {% endif %}
+
+    {% do return(build_sql) %}
+
+{% endmacro %}