Skip to content

Commit 393a2f9

Browse files
committed
revert the code refactor in loader.py, I will use a seperate pr for this refactor
1 parent 12e2a63 commit 393a2f9

File tree

1 file changed

+20
-3
lines changed

1 file changed

+20
-3
lines changed

bigframes/session/loader.py

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import google.cloud.bigquery.table
4646
from google.cloud.bigquery_storage_v1 import types as bq_storage_types
4747
import pandas
48+
import pyarrow as pa
4849

4950
import bigframes._tools
5051
import bigframes._tools.strings
@@ -1306,6 +1307,22 @@ def _transform_read_gbq_configuration(configuration: Optional[dict]) -> dict:
13061307
return configuration
13071308

13081309

1310+
def _has_json_arrow_type(arrow_type: pa.DataType) -> bool:
1311+
"""
1312+
Searches recursively for JSON array type within a PyArrow DataType.
1313+
"""
1314+
if arrow_type == bigframes.dtypes.JSON_ARROW_TYPE:
1315+
return True
1316+
if pa.types.is_list(arrow_type):
1317+
return _has_json_arrow_type(arrow_type.value_type)
1318+
if pa.types.is_struct(arrow_type):
1319+
for i in range(arrow_type.num_fields):
1320+
if _has_json_arrow_type(arrow_type.field(i).type):
1321+
return True
1322+
return False
1323+
return False
1324+
1325+
13091326
def _validate_dtype_can_load(name: str, column_type: bigframes.dtypes.Dtype):
13101327
"""
13111328
Determines whether a datatype is supported by bq load jobs.
@@ -1322,9 +1339,9 @@ def _validate_dtype_can_load(name: str, column_type: bigframes.dtypes.Dtype):
13221339
if column_type == bigframes.dtypes.JSON_DTYPE:
13231340
return
13241341

1325-
if isinstance(
1326-
column_type, pandas.ArrowDtype
1327-
) and bigframes.dtypes.contains_db_dtypes_json_dtype(column_type):
1342+
if isinstance(column_type, pandas.ArrowDtype) and _has_json_arrow_type(
1343+
column_type.pyarrow_dtype
1344+
):
13281345
raise NotImplementedError(
13291346
f"Nested JSON types, found in column `{name}`: `{column_type}`', "
13301347
f"are currently unsupported for upload. {constants.FEEDBACK_LINK}"

0 commit comments

Comments
 (0)