From 8f75b30061cbf983fc95b1cfb9837bd2579b880e Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Wed, 3 Jun 2026 14:12:45 +0000 Subject: [PATCH 1/4] feat: add `EnableProjectDiscovery` connection property for metadata methods --- .../bigquery/jdbc/BigQueryConnection.java | 46 +++++++ .../jdbc/BigQueryDatabaseMetaData.java | 72 +++++----- .../bigquery/jdbc/BigQueryJdbcUrlUtility.java | 9 ++ .../cloud/bigquery/jdbc/DataSource.java | 22 ++++ .../jdbc/BigQueryDatabaseMetaDataTest.java | 124 ++++++++++++++++++ .../jdbc/BigQueryJdbcUrlUtilityTest.java | 19 +++ 6 files changed, 262 insertions(+), 30 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index 07c95236c020..b0797253d4b8 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -23,6 +23,9 @@ import com.google.api.gax.rpc.FixedHeaderProvider; import com.google.api.gax.rpc.HeaderProvider; import com.google.api.gax.rpc.TransportChannelProvider; +import com.google.api.services.bigquery.Bigquery; +import com.google.api.services.bigquery.model.ProjectList; +import com.google.api.services.bigquery.model.ProjectList.Projects; import com.google.auth.Credentials; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; @@ -36,14 +39,17 @@ import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.exception.BigQueryJdbcRuntimeException; import com.google.cloud.bigquery.exception.BigQueryJdbcSqlFeatureNotSupportedException; +import com.google.cloud.bigquery.spi.v2.BigQueryRpc; import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; import com.google.cloud.bigquery.storage.v1.BigQueryReadSettings; import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; import com.google.cloud.bigquery.storage.v1.BigQueryWriteSettings; import com.google.cloud.http.HttpTransportOptions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSortedSet; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Field; import java.sql.CallableStatement; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -120,6 +126,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { BigQueryJdbcUrlUtility.SWA_APPEND_ROW_COUNT_PROPERTY_NAME, BigQueryJdbcUrlUtility.SWA_ACTIVATION_ROW_COUNT_PROPERTY_NAME, BigQueryJdbcUrlUtility.FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME, + BigQueryJdbcUrlUtility.ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME, BigQueryJdbcUrlUtility.REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME, BigQueryJdbcUrlUtility.SSL_TRUST_STORE_PROPERTY_NAME, BigQueryJdbcUrlUtility.MAX_BYTES_BILLED_PROPERTY_NAME, @@ -169,6 +176,8 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { int highThroughputMinTableSize; int highThroughputActivationRatio; boolean enableSession; + boolean enableProjectDiscovery; + private List discoveredProjectsCache; boolean unsupportedHTAPIFallback; boolean useQueryCache; String queryDialect; @@ -335,6 +344,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { this.additionalProjects = ds.getAdditionalProjects(); this.filterTablesOnDefaultDataset = ds.getFilterTablesOnDefaultDataset(); + this.enableProjectDiscovery = ds.getEnableProjectDiscovery(); this.requestGoogleDriveScope = ds.getRequestGoogleDriveScope(); this.metadataFetchThreadCount = ds.getMetadataFetchThreadCount(); this.requestReason = ds.getRequestReason(); @@ -1221,6 +1231,42 @@ private boolean checkIsReadOnlyTokenUsed(Map authProps) { return false; } + public boolean isEnableProjectDiscovery() { + return this.enableProjectDiscovery; + } + + public synchronized List getDiscoveredProjects() { + if (this.discoveredProjectsCache != null) { + return this.discoveredProjectsCache; + } + + try { + BigQueryOptions options = (BigQueryOptions) getBigQuery().getOptions(); + BigQueryRpc rpc = (BigQueryRpc) options.getRpc(); + Field bqField = rpc.getClass().getDeclaredField("bigquery"); + bqField.setAccessible(true); + Bigquery lowLevelBq = (Bigquery) bqField.get(rpc); + + List projects = new ArrayList<>(); + String pageToken = null; + do { + ProjectList projectList = lowLevelBq.projects().list().setPageToken(pageToken).execute(); + if (projectList.getProjects() != null) { + for (Projects p : projectList.getProjects()) { + projects.add(p.getProjectReference().getProjectId()); + } + } + pageToken = projectList.getNextPageToken(); + } while (pageToken != null); + + this.discoveredProjectsCache = ImmutableList.copyOf(projects); + } catch (Exception e) { + LOG.warning(e, "Failed to list all accessible projects, falling back to connection default."); + this.discoveredProjectsCache = ImmutableList.of(); + } + return this.discoveredProjectsCache; + } + @Override public T unwrap(Class iface) throws SQLException { if (iface.isInstance(this)) { diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index 32ed62d91fd6..ce83e57385a5 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -3652,44 +3652,51 @@ public ResultSet getSchemas(String catalog, String schemaPattern) { return; } + ExecutorService apiExecutor = null; try { + apiExecutor = Executors.newFixedThreadPool(API_EXECUTOR_POOL_SIZE); + List>> apiFutures = new ArrayList<>(); for (String currentProjectToScan : projectsToScanList) { if (Thread.currentThread().isInterrupted()) { - LOG.warning( - "Schema fetcher interrupted during project iteration for project: " - + currentProjectToScan); break; } - LOG.info("Fetching schemas for project: " + currentProjectToScan); - List datasetsInProject = - findMatchingBigQueryObjects( - "Dataset", - () -> - bigquery.listDatasets( - currentProjectToScan, - BigQuery.DatasetListOption.pageSize(DEFAULT_PAGE_SIZE)), - (name) -> bigquery.getDataset(DatasetId.of(currentProjectToScan, name)), - (ds) -> ds.getDatasetId().getDataset(), - schemaPattern, - schemaRegex, - LOG); + Callable> apiCallable = + () -> + findMatchingBigQueryObjects( + "Dataset", + () -> + bigquery.listDatasets( + currentProjectToScan, + BigQuery.DatasetListOption.pageSize(DEFAULT_PAGE_SIZE)), + (name) -> bigquery.getDataset(DatasetId.of(currentProjectToScan, name)), + (ds) -> ds.getDatasetId().getDataset(), + schemaPattern, + schemaRegex, + LOG); + apiFutures.add(apiExecutor.submit(apiCallable)); + } + apiExecutor.shutdown(); - if (datasetsInProject.isEmpty() || Thread.currentThread().isInterrupted()) { - LOG.info( - "Fetcher thread found no matching datasets in project: " - + currentProjectToScan); - continue; + for (Future> apiFuture : apiFutures) { + if (Thread.currentThread().isInterrupted()) { + break; } - - LOG.fine("Processing found datasets for project: " + currentProjectToScan); - for (Dataset dataset : datasetsInProject) { - if (Thread.currentThread().isInterrupted()) { - LOG.warning( - "Schema fetcher interrupted during dataset iteration for project: " - + currentProjectToScan); - break; + try { + List datasetsInProject = apiFuture.get(); + if (datasetsInProject != null) { + for (Dataset dataset : datasetsInProject) { + if (Thread.currentThread().isInterrupted()) break; + processSchemaInfo(dataset, collectedResults, localResultSchemaFields); + } } - processSchemaInfo(dataset, collectedResults, localResultSchemaFields); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warning("Fetcher thread interrupted while waiting for API future result."); + break; + } catch (ExecutionException e) { + LOG.warning("Error executing findMatchingDatasets task: " + e.getMessage()); + } catch (CancellationException e) { + LOG.warning("A findMatchingDatasets task was cancelled."); } } @@ -3706,6 +3713,7 @@ public ResultSet getSchemas(String catalog, String schemaPattern) { } catch (Throwable t) { LOG.severe("Unexpected error in schema fetcher runnable: " + t.getMessage()); } finally { + shutdownExecutor(apiExecutor); signalEndOfData(queue, localResultSchemaFields); LOG.info("Schema fetcher thread finished."); } @@ -5197,6 +5205,10 @@ private List getAccessibleCatalogNames() { } } + if (this.connection.isEnableProjectDiscovery()) { + accessibleCatalogs.addAll(this.connection.getDiscoveredProjects()); + } + List sortedCatalogs = new ArrayList<>(accessibleCatalogs); Collections.sort(sortedCatalogs); return sortedCatalogs; diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java index 0a19bed7a2c8..44841f7d16c5 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java @@ -167,6 +167,8 @@ protected boolean removeEldestEntry(Map.Entry> eldes static final String FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME = "FilterTablesOnDefaultDataset"; static final boolean DEFAULT_FILTER_TABLES_ON_DEFAULT_DATASET_VALUE = false; + static final String ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME = "EnableProjectDiscovery"; + static final boolean DEFAULT_ENABLE_PROJECT_DISCOVERY_VALUE = false; static final String REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME = "RequestGoogleDriveScope"; static final String SSL_TRUST_STORE_PROPERTY_NAME = "SSLTrustStore"; static final String SSL_TRUST_STORE_PWD_PROPERTY_NAME = "SSLTrustStorePwd"; @@ -576,6 +578,13 @@ protected boolean removeEldestEntry(Map.Entry> eldes .setDefaultValue( String.valueOf(DEFAULT_FILTER_TABLES_ON_DEFAULT_DATASET_VALUE)) .build(), + BigQueryConnectionProperty.newBuilder() + .setName(ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME) + .setDescription( + "Enables or disables automatic discovery of all accessible Google Cloud projects. " + + "When disabled, only the default ProjectId and AdditionalProjects are listed as catalogs.") + .setDefaultValue(String.valueOf(DEFAULT_ENABLE_PROJECT_DISCOVERY_VALUE)) + .build(), BigQueryConnectionProperty.newBuilder() .setName(REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME) .setDescription( diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java index c97da7bd9ee3..e443eb0ed853 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java @@ -84,6 +84,7 @@ public class DataSource implements javax.sql.DataSource { private Boolean enableWriteAPI; private String additionalProjects; private Boolean filterTablesOnDefaultDataset; + private Boolean enableProjectDiscovery; private Integer requestGoogleDriveScope; private Integer metadataFetchThreadCount; private String sslTrustStorePath; @@ -242,6 +243,12 @@ public class DataSource implements javax.sql.DataSource { BigQueryJdbcUrlUtility.convertIntToBoolean( val, BigQueryJdbcUrlUtility.FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME))) + .put( + BigQueryJdbcUrlUtility.ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME, + (ds, val) -> + ds.setEnableProjectDiscovery( + BigQueryJdbcUrlUtility.convertIntToBoolean( + val, BigQueryJdbcUrlUtility.ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME))) .put( BigQueryJdbcUrlUtility.REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME, (ds, val) -> ds.setRequestGoogleDriveScope(Integer.parseInt(val))) @@ -555,6 +562,11 @@ Properties createProperties() { BigQueryJdbcUrlUtility.FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME, String.valueOf(this.filterTablesOnDefaultDataset)); } + if (this.enableProjectDiscovery != null) { + connectionProperties.setProperty( + BigQueryJdbcUrlUtility.ENABLE_PROJECT_DISCOVERY_PROPERTY_NAME, + String.valueOf(this.enableProjectDiscovery)); + } if (this.requestGoogleDriveScope != null) { connectionProperties.setProperty( BigQueryJdbcUrlUtility.REQUEST_GOOGLE_DRIVE_SCOPE_PROPERTY_NAME, @@ -1059,6 +1071,16 @@ public void setFilterTablesOnDefaultDataset(Boolean filterTablesOnDefaultDataset this.filterTablesOnDefaultDataset = filterTablesOnDefaultDataset; } + public Boolean getEnableProjectDiscovery() { + return enableProjectDiscovery != null + ? enableProjectDiscovery + : BigQueryJdbcUrlUtility.DEFAULT_ENABLE_PROJECT_DISCOVERY_VALUE; + } + + public void setEnableProjectDiscovery(Boolean enableProjectDiscovery) { + this.enableProjectDiscovery = enableProjectDiscovery; + } + public Integer getRequestGoogleDriveScope() { return requestGoogleDriveScope != null ? requestGoogleDriveScope diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java index 58a5a7212066..35e080d58208 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java @@ -3308,4 +3308,128 @@ public void testMetadataAndResultSetMetadataTypeMappingConsistency(StandardSQLTy assertEquals( metadataTypeInfo.jdbcType, (int) resultSetType, "Type mapping mismatch for " + type); } + + @Test + public void testGetCatalogs_WithProjectDiscovery() throws SQLException { + when(bigQueryConnection.getCatalog()).thenReturn("primary-project"); + when(bigQueryConnection.isEnableProjectDiscovery()).thenReturn(true); + when(bigQueryConnection.getDiscoveredProjects()) + .thenReturn(Arrays.asList("discovered-1", "discovered-2")); + when(bigQueryConnection.getAdditionalProjects()).thenReturn("additional-1,additional-2"); + + ResultSet rs = dbMetadata.getCatalogs(); + assertNotNull(rs); + + List catalogs = new ArrayList<>(); + while (rs.next()) { + catalogs.add(rs.getString("TABLE_CAT")); + } + + assertThat(catalogs) + .containsExactly( + "additional-1", "additional-2", "discovered-1", "discovered-2", "primary-project") + .inOrder(); + } + + @Test + public void testGetCatalogs_WithoutProjectDiscovery() throws SQLException { + when(bigQueryConnection.getCatalog()).thenReturn("primary-project"); + when(bigQueryConnection.isEnableProjectDiscovery()).thenReturn(false); + when(bigQueryConnection.getDiscoveredProjects()) + .thenReturn(Arrays.asList("discovered-1", "discovered-2")); + when(bigQueryConnection.getAdditionalProjects()).thenReturn("additional-1,additional-2"); + + ResultSet rs = dbMetadata.getCatalogs(); + assertNotNull(rs); + + List catalogs = new ArrayList<>(); + while (rs.next()) { + catalogs.add(rs.getString("TABLE_CAT")); + } + + assertThat(catalogs) + .containsExactly("additional-1", "additional-2", "primary-project") + .inOrder(); + } + + @Test + public void testGetSchemas_WithProjectDiscovery() throws SQLException { + when(bigQueryConnection.getCatalog()).thenReturn("primary-project"); + when(bigQueryConnection.isEnableProjectDiscovery()).thenReturn(true); + when(bigQueryConnection.getDiscoveredProjects()).thenReturn(Arrays.asList("discovered-1")); + when(bigQueryConnection.getAdditionalProjects()).thenReturn("additional-1"); + + Page pagePrimary = mock(Page.class); + Dataset dsPrimary = mockBigQueryDataset("primary-project", "dataset_p"); + when(pagePrimary.iterateAll()).thenReturn(Collections.singletonList(dsPrimary)); + when(bigqueryClient.listDatasets(eq("primary-project"), any(BigQuery.DatasetListOption.class))) + .thenReturn(pagePrimary); + + Page pageAdditional = mock(Page.class); + Dataset dsAdditional = mockBigQueryDataset("additional-1", "dataset_a"); + when(pageAdditional.iterateAll()).thenReturn(Collections.singletonList(dsAdditional)); + when(bigqueryClient.listDatasets(eq("additional-1"), any(BigQuery.DatasetListOption.class))) + .thenReturn(pageAdditional); + + Page pageDiscovered = mock(Page.class); + Dataset dsDiscovered = mockBigQueryDataset("discovered-1", "dataset_d"); + when(pageDiscovered.iterateAll()).thenReturn(Collections.singletonList(dsDiscovered)); + when(bigqueryClient.listDatasets(eq("discovered-1"), any(BigQuery.DatasetListOption.class))) + .thenReturn(pageDiscovered); + + ResultSet rs = dbMetadata.getSchemas(null, null); + assertNotNull(rs); + + List schemas = new ArrayList<>(); + List catalogs = new ArrayList<>(); + while (rs.next()) { + schemas.add(rs.getString("TABLE_SCHEM")); + catalogs.add(rs.getString("TABLE_CATALOG")); + } + + // Results are sorted by catalog (TABLE_CATALOG) then schema (TABLE_SCHEM) + // alphabetical catalog: "additional-1", "discovered-1", "primary-project" + assertThat(catalogs) + .containsExactly("additional-1", "discovered-1", "primary-project") + .inOrder(); + assertThat(schemas).containsExactly("dataset_a", "dataset_d", "dataset_p").inOrder(); + } + + @Test + public void testGetSchemas_WithoutProjectDiscovery() throws SQLException { + when(bigQueryConnection.getCatalog()).thenReturn("primary-project"); + when(bigQueryConnection.isEnableProjectDiscovery()).thenReturn(false); + when(bigQueryConnection.getDiscoveredProjects()).thenReturn(Arrays.asList("discovered-1")); + when(bigQueryConnection.getAdditionalProjects()).thenReturn("additional-1"); + + Page pagePrimary = mock(Page.class); + Dataset dsPrimary = mockBigQueryDataset("primary-project", "dataset_p"); + when(pagePrimary.iterateAll()).thenReturn(Collections.singletonList(dsPrimary)); + when(bigqueryClient.listDatasets(eq("primary-project"), any(BigQuery.DatasetListOption.class))) + .thenReturn(pagePrimary); + + Page pageAdditional = mock(Page.class); + Dataset dsAdditional = mockBigQueryDataset("additional-1", "dataset_a"); + when(pageAdditional.iterateAll()).thenReturn(Collections.singletonList(dsAdditional)); + when(bigqueryClient.listDatasets(eq("additional-1"), any(BigQuery.DatasetListOption.class))) + .thenReturn(pageAdditional); + + ResultSet rs = dbMetadata.getSchemas(null, null); + assertNotNull(rs); + + List schemas = new ArrayList<>(); + List catalogs = new ArrayList<>(); + while (rs.next()) { + schemas.add(rs.getString("TABLE_SCHEM")); + catalogs.add(rs.getString("TABLE_CATALOG")); + } + + // Results are sorted by catalog (TABLE_CATALOG) then schema (TABLE_SCHEM) + // alphabetical catalog: "additional-1", "primary-project" (discovered-1 is ignored) + assertThat(catalogs).containsExactly("additional-1", "primary-project").inOrder(); + assertThat(schemas).containsExactly("dataset_a", "dataset_p").inOrder(); + + verify(bigqueryClient, never()) + .listDatasets(eq("discovered-1"), any(BigQuery.DatasetListOption.class)); + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtilityTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtilityTest.java index 3a09813a035e..0bc580391b12 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtilityTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtilityTest.java @@ -260,4 +260,23 @@ public void testUnrecognizedConnectionProperties() { String url2 = "jdbc:bigquery://;MalformedProperty"; assertThrows(BigQueryJdbcRuntimeException.class, () -> DataSource.fromUrl(url2)); } + + @Test + public void testParseEnableProjectDiscovery() { + String url = + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" + + "ProjectId=MyBigQueryProject;" + + "EnableProjectDiscovery=true"; + + String result = BigQueryJdbcUrlUtility.parseUriProperty(url, "EnableProjectDiscovery"); + assertThat(result).isEqualTo("true"); + + String url2 = + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;" + + "ProjectId=MyBigQueryProject;" + + "EnableProjectDiscovery=false"; + + String result2 = BigQueryJdbcUrlUtility.parseUriProperty(url2, "EnableProjectDiscovery"); + assertThat(result2).isEqualTo("false"); + } } From 81e80a0e5a7a61760d6c2b20b6bb51b5a22a2acf Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Wed, 3 Jun 2026 18:29:53 +0000 Subject: [PATCH 2/4] address pr feedback --- .../bigquery/jdbc/BigQueryConnection.java | 30 +++++++++++---- .../jdbc/BigQueryDatabaseMetaData.java | 38 +++++++++---------- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index b0797253d4b8..1d376775ac3b 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -16,6 +16,9 @@ package com.google.cloud.bigquery.jdbc; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.gson.GsonFactory; import com.google.api.gax.core.CredentialsProvider; import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; @@ -39,7 +42,6 @@ import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.exception.BigQueryJdbcRuntimeException; import com.google.cloud.bigquery.exception.BigQueryJdbcSqlFeatureNotSupportedException; -import com.google.cloud.bigquery.spi.v2.BigQueryRpc; import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; import com.google.cloud.bigquery.storage.v1.BigQueryReadSettings; import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; @@ -49,7 +51,6 @@ import com.google.common.collect.ImmutableSortedSet; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.Field; import java.sql.CallableStatement; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -83,6 +84,10 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { private final String connectionId; private static final String DEFAULT_JDBC_TOKEN_VALUE = "Google-BigQuery-JDBC-Driver"; private static final String DEFAULT_VERSION = "0.0.0"; + private static final String BIGQUERY_SERVICE_NAME = "bigquery"; + private static final long MAX_PROJECTS_PER_PAGE = 10000L; + private static final String PROJECT_LIST_FIELDS = + "projects/projectReference/projectId,nextPageToken"; private static final Set SAFE_TO_LOG_PROPERTIES = ImmutableSortedSet.orderedBy(String.CASE_INSENSITIVE_ORDER) .add( @@ -1242,15 +1247,26 @@ public synchronized List getDiscoveredProjects() { try { BigQueryOptions options = (BigQueryOptions) getBigQuery().getOptions(); - BigQueryRpc rpc = (BigQueryRpc) options.getRpc(); - Field bqField = rpc.getClass().getDeclaredField("bigquery"); - bqField.setAccessible(true); - Bigquery lowLevelBq = (Bigquery) bqField.get(rpc); + HttpTransportOptions transportOptions = (HttpTransportOptions) options.getTransportOptions(); + HttpTransport transport = transportOptions.getHttpTransportFactory().create(); + HttpRequestInitializer initializer = transportOptions.getHttpRequestInitializer(options); + Bigquery lowLevelBq = + new Bigquery.Builder(transport, new GsonFactory(), initializer) + .setRootUrl(options.getResolvedApiaryHost(BIGQUERY_SERVICE_NAME)) + .setApplicationName(DEFAULT_JDBC_TOKEN_VALUE) + .build(); List projects = new ArrayList<>(); String pageToken = null; do { - ProjectList projectList = lowLevelBq.projects().list().setPageToken(pageToken).execute(); + ProjectList projectList = + lowLevelBq + .projects() + .list() + .setPageToken(pageToken) + .setMaxResults(MAX_PROJECTS_PER_PAGE) + .setFields(PROJECT_LIST_FIELDS) + .execute(); if (projectList.getProjects() != null) { for (Projects p : projectList.getProjects()) { projects.add(p.getProjectReference().getProjectId()); diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index ce83e57385a5..a721308551c8 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -3653,13 +3653,11 @@ public ResultSet getSchemas(String catalog, String schemaPattern) { } ExecutorService apiExecutor = null; + final List>> apiFutures = new ArrayList<>(); try { apiExecutor = Executors.newFixedThreadPool(API_EXECUTOR_POOL_SIZE); - List>> apiFutures = new ArrayList<>(); for (String currentProjectToScan : projectsToScanList) { - if (Thread.currentThread().isInterrupted()) { - break; - } + checkInterrupted(apiFutures); Callable> apiCallable = () -> findMatchingBigQueryObjects( @@ -3678,21 +3676,17 @@ public ResultSet getSchemas(String catalog, String schemaPattern) { apiExecutor.shutdown(); for (Future> apiFuture : apiFutures) { - if (Thread.currentThread().isInterrupted()) { - break; - } + checkInterrupted(apiFutures); try { List datasetsInProject = apiFuture.get(); if (datasetsInProject != null) { for (Dataset dataset : datasetsInProject) { - if (Thread.currentThread().isInterrupted()) break; processSchemaInfo(dataset, collectedResults, localResultSchemaFields); } } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - LOG.warning("Fetcher thread interrupted while waiting for API future result."); - break; + checkInterrupted(apiFutures); } catch (ExecutionException e) { LOG.warning("Error executing findMatchingDatasets task: " + e.getMessage()); } catch (CancellationException e) { @@ -3700,18 +3694,17 @@ public ResultSet getSchemas(String catalog, String schemaPattern) { } } - if (!Thread.currentThread().isInterrupted()) { - Comparator comparator = - defineGetSchemasComparator(localResultSchemaFields); - sortResults(collectedResults, comparator, "getSchemas", LOG); - } - - if (!Thread.currentThread().isInterrupted()) { - populateQueue(collectedResults, queue, localResultSchemaFields); - } + checkInterrupted(apiFutures); + Comparator comparator = + defineGetSchemasComparator(localResultSchemaFields); + sortResults(collectedResults, comparator, "getSchemas", LOG); + populateQueue(collectedResults, queue, localResultSchemaFields); + } catch (CancellationException e) { + LOG.warning("Schema fetcher task was cancelled/interrupted."); } catch (Throwable t) { LOG.severe("Unexpected error in schema fetcher runnable: " + t.getMessage()); + apiFutures.forEach(f -> f.cancel(true)); } finally { shutdownExecutor(apiExecutor); signalEndOfData(queue, localResultSchemaFields); @@ -5155,6 +5148,13 @@ private void signalEndOfData( } } + private void checkInterrupted(List> futures) { + if (Thread.currentThread().isInterrupted()) { + futures.forEach(f -> f.cancel(true)); + throw new CancellationException("Fetcher thread was interrupted."); + } + } + private void shutdownExecutor(ExecutorService executor) { if (executor == null || executor.isShutdown()) { return; From 86d62b1ea9bf7104f33d7b2785d4cb456a956b52 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Wed, 3 Jun 2026 18:59:45 +0000 Subject: [PATCH 3/4] address pr feedback --- .../cloud/bigquery/jdbc/BigQueryConnection.java | 13 +++++++++++-- .../bigquery/jdbc/BigQueryDatabaseMetaData.java | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index 1d376775ac3b..4a9c06b9a128 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -16,6 +16,7 @@ package com.google.cloud.bigquery.jdbc; +import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.gson.GsonFactory; @@ -1251,7 +1252,7 @@ public synchronized List getDiscoveredProjects() { HttpTransport transport = transportOptions.getHttpTransportFactory().create(); HttpRequestInitializer initializer = transportOptions.getHttpRequestInitializer(options); Bigquery lowLevelBq = - new Bigquery.Builder(transport, new GsonFactory(), initializer) + new Bigquery.Builder(transport, GsonFactory.getDefaultInstance(), initializer) .setRootUrl(options.getResolvedApiaryHost(BIGQUERY_SERVICE_NAME)) .setApplicationName(DEFAULT_JDBC_TOKEN_VALUE) .build(); @@ -1276,9 +1277,17 @@ public synchronized List getDiscoveredProjects() { } while (pageToken != null); this.discoveredProjectsCache = ImmutableList.copyOf(projects); + } catch (GoogleJsonResponseException e) { + LOG.warning(e, "Failed to list all accessible projects due to Google API error."); + int statusCode = e.getStatusCode(); + // Only cache empty list for non-transient auth/permission errors (400, 401, 403) + if (statusCode == 400 || statusCode == 401 || statusCode == 403) { + this.discoveredProjectsCache = ImmutableList.of(); + } + return ImmutableList.of(); } catch (Exception e) { LOG.warning(e, "Failed to list all accessible projects, falling back to connection default."); - this.discoveredProjectsCache = ImmutableList.of(); + return ImmutableList.of(); } return this.discoveredProjectsCache; } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index a721308551c8..bf1b385a04d1 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -3688,7 +3688,7 @@ public ResultSet getSchemas(String catalog, String schemaPattern) { Thread.currentThread().interrupt(); checkInterrupted(apiFutures); } catch (ExecutionException e) { - LOG.warning("Error executing findMatchingDatasets task: " + e.getMessage()); + LOG.warning(e, "Error executing findMatchingDatasets task."); } catch (CancellationException e) { LOG.warning("A findMatchingDatasets task was cancelled."); } From f4403674925fe881d55e80684f16c772af2f6e07 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Fri, 5 Jun 2026 16:13:19 +0000 Subject: [PATCH 4/4] chore: address pr feedback --- .../google/cloud/bigquery/jdbc/BigQueryConnection.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index 4a9c06b9a128..00af23b2331a 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -86,7 +86,6 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { private static final String DEFAULT_JDBC_TOKEN_VALUE = "Google-BigQuery-JDBC-Driver"; private static final String DEFAULT_VERSION = "0.0.0"; private static final String BIGQUERY_SERVICE_NAME = "bigquery"; - private static final long MAX_PROJECTS_PER_PAGE = 10000L; private static final String PROJECT_LIST_FIELDS = "projects/projectReference/projectId,nextPageToken"; private static final Set SAFE_TO_LOG_PROPERTIES = @@ -1254,7 +1253,10 @@ public synchronized List getDiscoveredProjects() { Bigquery lowLevelBq = new Bigquery.Builder(transport, GsonFactory.getDefaultInstance(), initializer) .setRootUrl(options.getResolvedApiaryHost(BIGQUERY_SERVICE_NAME)) - .setApplicationName(DEFAULT_JDBC_TOKEN_VALUE) + .setApplicationName( + options.getApplicationName() != null + ? options.getApplicationName() + : DEFAULT_JDBC_TOKEN_VALUE) .build(); List projects = new ArrayList<>(); @@ -1265,7 +1267,7 @@ public synchronized List getDiscoveredProjects() { .projects() .list() .setPageToken(pageToken) - .setMaxResults(MAX_PROJECTS_PER_PAGE) + .setMaxResults(getMaxResults()) .setFields(PROJECT_LIST_FIELDS) .execute(); if (projectList.getProjects() != null) {