Merge branch 'main' into interchange

Author: tswast

CHANGELOG.md

@@ -4,6 +4,21 @@
 
 [1]: https://pypi.org/project/bigframes/#history
 
+## [2.19.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.18.0...v2.19.0) (2025-09-09)
+
+
+### Features
+
+* Add str.join method ([#2054](https://github.com/googleapis/python-bigquery-dataframes/issues/2054)) ([8804ada](https://github.com/googleapis/python-bigquery-dataframes/commit/8804adaf8ba23fdcad6e42a7bf034bd0a11c890f))
+* Support display.max_colwidth option ([#2053](https://github.com/googleapis/python-bigquery-dataframes/issues/2053)) ([5229e07](https://github.com/googleapis/python-bigquery-dataframes/commit/5229e07b4535c01b0cdbd731455ff225a373b5c8))
+* Support VPC egress setting in remote function ([#2059](https://github.com/googleapis/python-bigquery-dataframes/issues/2059)) ([5df779d](https://github.com/googleapis/python-bigquery-dataframes/commit/5df779d4f421d3ba777cfd928d99ca2e8a3f79ad))
+
+
+### Bug Fixes
+
+* Fix issue mishandling chunked array while loading data ([#2051](https://github.com/googleapis/python-bigquery-dataframes/issues/2051)) ([873d0ee](https://github.com/googleapis/python-bigquery-dataframes/commit/873d0eee474ed34f1d5164c37383f2737dbec4db))
+* Remove warning for slot_millis_sum ([#2047](https://github.com/googleapis/python-bigquery-dataframes/issues/2047)) ([425a691](https://github.com/googleapis/python-bigquery-dataframes/commit/425a6917d5442eeb4df486c6eed1fd136bbcedfb))
+
 ## [2.18.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v2.17.0...v2.18.0) (2025-09-03)
 
 

bigframes/_config/auth.py

@@ -0,0 +1,57 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import threading
+from typing import Optional
+
+import google.auth.credentials
+import google.auth.transport.requests
+import pydata_google_auth
+
+_SCOPES = ["https://www.googleapis.com/auth/cloud-platform"]
+
+# Put the lock here rather than in BigQueryOptions so that BigQueryOptions
+# remains deepcopy-able.
+_AUTH_LOCK = threading.Lock()
+_cached_credentials: Optional[google.auth.credentials.Credentials] = None
+_cached_project_default: Optional[str] = None
+
+
+def get_default_credentials_with_project() -> tuple[
+    google.auth.credentials.Credentials, Optional[str]
+]:
+    global _AUTH_LOCK, _cached_credentials, _cached_project_default
+
+    with _AUTH_LOCK:
+        if _cached_credentials is not None:
+            return _cached_credentials, _cached_project_default
+
+        _cached_credentials, _cached_project_default = pydata_google_auth.default(
+            scopes=_SCOPES, use_local_webserver=False
+        )
+
+        # Ensure an access token is available.
+        _cached_credentials.refresh(google.auth.transport.requests.Request())
+
+    return _cached_credentials, _cached_project_default
+
+
+def reset_default_credentials_and_project():
+    global _AUTH_LOCK, _cached_credentials, _cached_project_default
+
+    with _AUTH_LOCK:
+        _cached_credentials = None
+        _cached_project_default = None

bigframes/_config/bigquery_options.py

@@ -22,6 +22,7 @@
 import google.auth.credentials
 import requests.adapters
 
+import bigframes._config.auth
 import bigframes._importing
 import bigframes.enums
 import bigframes.exceptions as bfe
@@ -37,6 +38,7 @@
 
 def _get_validated_location(value: Optional[str]) -> Optional[str]:
     import bigframes._tools.strings
+    import bigframes.constants
 
     if value is None or value in bigframes.constants.ALL_BIGQUERY_LOCATIONS:
         return value
@@ -141,20 +143,52 @@ def application_name(self, value: Optional[str]):
             )
         self._application_name = value
 
+    def _try_set_default_credentials_and_project(
+        self,
+    ) -> tuple[google.auth.credentials.Credentials, Optional[str]]:
+        # Don't fetch credentials or project if credentials is already set.
+        # If it's set, we've already authenticated, so if the user wants to
+        # re-auth, they should explicitly reset the credentials.
+        if self._credentials is not None:
+            return self._credentials, self._project
+
+        (
+            credentials,
+            credentials_project,
+        ) = bigframes._config.auth.get_default_credentials_with_project()
+        self._credentials = credentials
+
+        # Avoid overriding an explicitly set project with a default value.
+        if self._project is None:
+            self._project = credentials_project
+
+        return credentials, self._project
+
     @property
-    def credentials(self) -> Optional[google.auth.credentials.Credentials]:
+    def credentials(self) -> google.auth.credentials.Credentials:
         """The OAuth2 credentials to use for this client.
 
+        Set to None to force re-authentication.
+
         Returns:
             None or google.auth.credentials.Credentials:
                 google.auth.credentials.Credentials if exists; otherwise None.
         """
-        return self._credentials
+        if self._credentials:
+            return self._credentials
+
+        credentials, _ = self._try_set_default_credentials_and_project()
+        return credentials
 
     @credentials.setter
     def credentials(self, value: Optional[google.auth.credentials.Credentials]):
         if self._session_started and self._credentials is not value:
             raise ValueError(SESSION_STARTED_MESSAGE.format(attribute="credentials"))
+
+        if value is None:
+            # The user has _explicitly_ asked that we re-authenticate.
+            bigframes._config.auth.reset_default_credentials_and_project()
+
         self._credentials = value
 
     @property
@@ -183,7 +217,11 @@ def project(self) -> Optional[str]:
             None or str:
                 Google Cloud project ID as a string; otherwise None.
         """
-        return self._project
+        if self._project:
+            return self._project
+
+        _, project = self._try_set_default_credentials_and_project()
+        return project
 
     @project.setter
     def project(self, value: Optional[str]):

bigframes/bigquery/__init__.py

@@ -50,6 +50,7 @@
     json_value,
     json_value_array,
     parse_json,
+    to_json_string,
 )
 from bigframes.bigquery._operations.search import create_vector_index, vector_search
 from bigframes.bigquery._operations.sql import sql_scalar
@@ -87,6 +88,7 @@
     json_value,
     json_value_array,
     parse_json,
+    to_json_string,
     # search ops
     create_vector_index,
     vector_search,

bigframes/bigquery/_operations/json.py

@@ -430,6 +430,40 @@ def json_value_array(
     return input._apply_unary_op(ops.JSONValueArray(json_path=json_path))
 
 
+def to_json_string(
+    input: series.Series,
+) -> series.Series:
+    """Converts a series to a JSON-formatted STRING value.
+
+    **Examples:**
+
+        >>> import bigframes.pandas as bpd
+        >>> import bigframes.bigquery as bbq
+        >>> bpd.options.display.progress_bar = None
+
+        >>> s = bpd.Series([1, 2, 3])
+        >>> bbq.to_json_string(s)
+        0    1
+        1    2
+        2    3
+        dtype: string
+
+        >>> s = bpd.Series([{"int": 1, "str": "pandas"}, {"int": 2, "str": "numpy"}])
+        >>> bbq.to_json_string(s)
+        0    {"int":1,"str":"pandas"}
+        1     {"int":2,"str":"numpy"}
+        dtype: string
+
+    Args:
+        input (bigframes.series.Series):
+            The Series to be converted.
+
+    Returns:
+        bigframes.series.Series: A new Series with the JSON-formatted STRING value.
+    """
+    return input._apply_unary_op(ops.ToJSONString())
+
+
 @utils.preview(name="The JSON-related API `parse_json`")
 def parse_json(
     input: series.Series,

bigframes/core/blocks.py

@@ -27,7 +27,6 @@
 import functools
 import itertools
 import random
-import textwrap
 import typing
 from typing import (
     Iterable,
@@ -55,16 +54,13 @@
 from bigframes.core import agg_expressions, local_data
 import bigframes.core as core
 import bigframes.core.agg_expressions as ex_types
-import bigframes.core.compile.googlesql as googlesql
 import bigframes.core.expression as ex
 import bigframes.core.expression as scalars
 import bigframes.core.guid as guid
 import bigframes.core.identifiers
 import bigframes.core.join_def as join_defs
 import bigframes.core.ordering as ordering
 import bigframes.core.pyarrow_utils as pyarrow_utils
-import bigframes.core.schema as bf_schema
-import bigframes.core.sql as sql
 import bigframes.core.utils as utils
 import bigframes.core.window_spec as windows
 import bigframes.dtypes
@@ -2779,14 +2775,6 @@ def _throw_if_null_index(self, opname: str):
             )
 
     def _get_rows_as_json_values(self) -> Block:
-        # We want to preserve any ordering currently present before turning to
-        # direct SQL manipulation. We will restore the ordering when we rebuild
-        # expression.
-        # TODO(shobs): Replace direct SQL manipulation by structured expression
-        # manipulation
-        expr, ordering_column_name = self.expr.promote_offsets()
-        expr_sql = self.session._executor.to_sql(expr)
-
         # Names of the columns to serialize for the row.
         # We will use the repr-eval pattern to serialize a value here and
         # deserialize in the cloud function. Let's make sure that would work.
@@ -2802,93 +2790,44 @@ def _get_rows_as_json_values(self) -> Block:
                 )
 
             column_names.append(serialized_column_name)
-        column_names_csv = sql.csv(map(sql.simple_literal, column_names))
-
-        # index columns count
-        index_columns_count = len(self.index_columns)
 
         # column references to form the array of values for the row
         column_types = list(self.index.dtypes) + list(self.dtypes)
         column_references = []
         for type_, col in zip(column_types, self.expr.column_ids):
-            if isinstance(type_, pd.ArrowDtype) and pa.types.is_binary(
-                type_.pyarrow_dtype
-            ):
-                column_references.append(sql.to_json_string(col))
+            if type_ == bigframes.dtypes.BYTES_DTYPE:
+                column_references.append(ops.ToJSONString().as_expr(col))
+            elif type_ == bigframes.dtypes.BOOL_DTYPE:
+                # cast operator produces True/False, but function template expects lower case
+                column_references.append(
+                    ops.lower_op.as_expr(
+                        ops.AsTypeOp(bigframes.dtypes.STRING_DTYPE).as_expr(col)
+                    )
+                )
             else:
-                column_references.append(sql.cast_as_string(col))
-
-        column_references_csv = sql.csv(column_references)
-
-        # types of the columns to serialize for the row
-        column_types_csv = sql.csv(
-            [sql.simple_literal(str(typ)) for typ in column_types]
-        )
+                column_references.append(
+                    ops.AsTypeOp(bigframes.dtypes.STRING_DTYPE).as_expr(col)
+                )
 
         # row dtype to use for deserializing the row as pandas series
         pandas_row_dtype = bigframes.dtypes.lcd_type(*column_types)
         if pandas_row_dtype is None:
             pandas_row_dtype = "object"
-        pandas_row_dtype = sql.simple_literal(str(pandas_row_dtype))
-
-        # create a json column representing row through SQL manipulation
-        row_json_column_name = guid.generate_guid()
-        select_columns = (
-            [ordering_column_name] + list(self.index_columns) + [row_json_column_name]
-        )
-        select_columns_csv = sql.csv(
-            [googlesql.identifier(col) for col in select_columns]
-        )
-        json_sql = f"""\
-With T0 AS (
-{textwrap.indent(expr_sql, "    ")}
-),
-T1 AS (
-    SELECT *,
-           TO_JSON_STRING(JSON_OBJECT(
-               "names", [{column_names_csv}],
-               "types", [{column_types_csv}],
-               "values", [{column_references_csv}],
-               "indexlength", {index_columns_count},
-               "dtype", {pandas_row_dtype}
-           )) AS {googlesql.identifier(row_json_column_name)} FROM T0
-)
-SELECT {select_columns_csv} FROM T1
-"""
-        # The only ways this code is used is through df.apply(axis=1) cope path
-        destination, query_job = self.session._loader._query_to_destination(
-            json_sql, cluster_candidates=[ordering_column_name]
-        )
-        if not destination:
-            raise ValueError(f"Query job {query_job} did not produce result table")
-
-        new_schema = (
-            self.expr.schema.select([*self.index_columns])
-            .append(
-                bf_schema.SchemaItem(
-                    row_json_column_name, bigframes.dtypes.STRING_DTYPE
-                )
-            )
-            .append(
-                bf_schema.SchemaItem(ordering_column_name, bigframes.dtypes.INT_DTYPE)
-            )
-        )
+        pandas_row_dtype = str(pandas_row_dtype)
 
-        dest_table = self.session.bqclient.get_table(destination)
-        expr = core.ArrayValue.from_table(
-            dest_table,
-            schema=new_schema,
-            session=self.session,
-            offsets_col=ordering_column_name,
-            n_rows=dest_table.num_rows,
-        ).drop_columns([ordering_column_name])
-        block = Block(
-            expr,
-            index_columns=self.index_columns,
-            column_labels=[row_json_column_name],
-            index_labels=self._index_labels,
+        struct_op = ops.StructOp(
+            column_names=("names", "types", "values", "indexlength", "dtype")
         )
-        return block
+        names_val = ex.const(tuple(column_names))
+        types_val = ex.const(tuple(map(str, column_types)))
+        values_val = ops.ToArrayOp().as_expr(*column_references)
+        indexlength_val = ex.const(len(self.index_columns))
+        dtype_val = ex.const(str(pandas_row_dtype))
+        struct_expr = struct_op.as_expr(
+            names_val, types_val, values_val, indexlength_val, dtype_val
+        )
+        block, col_id = self.project_expr(ops.ToJSONString().as_expr(struct_expr))
+        return block.select_column(col_id)
 
 
 class BlockIndexProperties: