diff --git a/java-bigquery-jdbc/pom.xml b/java-bigquery-jdbc/pom.xml index caf308337078..3f248a34be56 100644 --- a/java-bigquery-jdbc/pom.xml +++ b/java-bigquery-jdbc/pom.xml @@ -424,6 +424,12 @@ opentelemetry-sdk-testing test + + com.google.cloud + google-cloud-trace + 2.92.0 + test + diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index d4ac8a9ee1bd..8e530ccf060c 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -42,6 +42,7 @@ import com.google.cloud.bigquery.storage.v1.BigQueryWriteSettings; import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.logging.Logging; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSortedSet; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.baggage.Baggage; @@ -215,6 +216,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { boolean enableGcpTraceExporter; boolean enableGcpLogExporter; OpenTelemetry customOpenTelemetry; + boolean useGlobalOpenTelemetry; private OpenTelemetry openTelemetry; private Context otelContext; Tracer tracer = @@ -366,6 +368,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { this.enableGcpTraceExporter = ds.getEnableGcpTraceExporter(); this.enableGcpLogExporter = ds.getEnableGcpLogExporter(); this.customOpenTelemetry = ds.getCustomOpenTelemetry(); + this.useGlobalOpenTelemetry = ds.getUseGlobalOpenTelemetry(); this.openTelemetry = getOpenTelemetryInstance(); this.bigQuery = getBigQueryConnection(); } @@ -438,7 +441,8 @@ String getConnectionUrl() { return connectionUrl; } - String getConnectionId() { + @VisibleForTesting + public String getConnectionId() { return this.connectionId; } @@ -1036,7 +1040,6 @@ void removeStatement(Statement statement) { } private OpenTelemetry getOpenTelemetryInstance() { - boolean hasCustomOtel = this.customOpenTelemetry != null; String effectiveProjectId = (this.gcpTelemetryProjectId != null) ? this.gcpTelemetryProjectId : this.catalog; @@ -1046,22 +1049,27 @@ private OpenTelemetry getOpenTelemetryInstance() { OpenTelemetry openTelemetry = BigQueryJdbcOpenTelemetry.getOpenTelemetry( + this.useGlobalOpenTelemetry, this.enableGcpTraceExporter, this.enableGcpLogExporter, this.customOpenTelemetry, effectiveCredentials, effectiveProjectId); + boolean hasExternalOtel = this.customOpenTelemetry != null || this.useGlobalOpenTelemetry; Logging localLoggingClient = null; - if (this.enableGcpLogExporter && !hasCustomOtel) { + if (this.enableGcpLogExporter && !hasExternalOtel) { localLoggingClient = BigQueryJdbcOpenTelemetry.createLoggingClient( true, null, effectiveCredentials, effectiveProjectId, this.credentials); } - if (this.enableGcpLogExporter || hasCustomOtel) { + if (this.enableGcpLogExporter || hasExternalOtel) { BigQueryJdbcOpenTelemetry.registerConnection( - this.connectionId, openTelemetry, localLoggingClient, this.enableGcpLogExporter); + this.connectionId, + openTelemetry, + localLoggingClient, + this.enableGcpLogExporter && !hasExternalOtel); } return openTelemetry; @@ -1126,7 +1134,9 @@ private BigQuery getBigQueryConnection() { if (this.httpTransportOptions != null) { bigQueryOptions.setTransportOptions(this.httpTransportOptions); } - if (this.enableGcpTraceExporter || this.customOpenTelemetry != null) { + if (this.enableGcpTraceExporter + || this.customOpenTelemetry != null + || this.useGlobalOpenTelemetry) { Tracer sdkTracer = this.openTelemetry.getTracer(BigQueryJdbcOpenTelemetry.BIGQUERY_NAMESPACE); bigQueryOptions.setOpenTelemetryTracer(sdkTracer); this.tracer = diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java index 70c23f0582b4..1995a4d330e4 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java @@ -21,6 +21,7 @@ import com.google.cloud.logging.Logging; import com.google.cloud.logging.LoggingOptions; import com.google.common.hash.Hashing; +import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.baggage.Baggage; import io.opentelemetry.api.trace.Span; @@ -65,6 +66,11 @@ public class BigQueryJdbcOpenTelemetry { private static final String OTLP_ENDPOINT_VALUE = "https://telemetry.googleapis.com:443"; private static final String EXPORTER_NONE = "none"; private static final String EXPORTER_OTLP = "otlp"; + private static final String OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT = + "otel.span.attribute.value.length.limit"; + private static final String OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT = + "otel.attribute.value.length.limit"; + private static final String DEFAULT_ATTRIBUTE_LENGTH_LIMIT = "32768"; private static final BigQueryJdbcCustomLogger LOG = new BigQueryJdbcCustomLogger("BigQueryJdbcOpenTelemetry"); @@ -240,6 +246,7 @@ private static String getCredentialsIdentifier(String credentials) { * customOpenTelemetry if provided; fallback to an auto-configured GCP exporter if requested. */ public static OpenTelemetry getOpenTelemetry( + boolean useGlobalOpenTelemetry, boolean enableGcpTraceExporter, boolean enableGcpLogExporter, OpenTelemetry customOpenTelemetry, @@ -250,6 +257,10 @@ public static OpenTelemetry getOpenTelemetry( return customOpenTelemetry; } + if (useGlobalOpenTelemetry) { + return GlobalOpenTelemetry.get(); + } + // NOTE: Currently, tracing only fully supports Application Default Credentials (ADC). // Once b/503721589 is completed, Service Account (SA) will work as well. if (!enableGcpTraceExporter && !enableGcpLogExporter) { @@ -290,6 +301,17 @@ public static OpenTelemetry getOpenTelemetry( props.put(GOOGLE_CLOUD_PROJECT, gcpTelemetryProjectId); } + // Set safe, generous default limits on attribute value lengths (32KB) to protect + // customers from GCP Cloud Trace 64KB span ingestion failures when logging massive + // exception stack traces or database schema metadata. + // Respect any existing user configuration overrides. + if (!props.containsKey(OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT)) { + props.put(OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, DEFAULT_ATTRIBUTE_LENGTH_LIMIT); + } + if (!props.containsKey(OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT)) { + props.put(OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, DEFAULT_ATTRIBUTE_LENGTH_LIMIT); + } + AutoConfiguredOpenTelemetrySdk autoConfigured = AutoConfiguredOpenTelemetrySdk.builder().addPropertiesSupplier(() -> props).build(); diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java index d14fef157af8..91de9cc79a49 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java @@ -168,6 +168,8 @@ protected boolean removeEldestEntry(Map.Entry> eldes static final boolean DEFAULT_ENABLE_GCP_TRACE_EXPORTER_VALUE = false; static final String ENABLE_GCP_LOG_EXPORTER_PROPERTY_NAME = "enableGcpLogExporter"; static final boolean DEFAULT_ENABLE_GCP_LOG_EXPORTER_VALUE = false; + static final String USE_GLOBAL_OTEL_PROPERTY_NAME = "useGlobalOpenTelemetry"; + static final boolean DEFAULT_USE_GLOBAL_OTEL_VALUE = false; private static final BigQueryJdbcCustomLogger LOG = new BigQueryJdbcCustomLogger(BigQueryJdbcUrlUtility.class.getName()); static final String FILTER_TABLES_ON_DEFAULT_DATASET_PROPERTY_NAME = @@ -638,6 +640,12 @@ protected boolean removeEldestEntry(Map.Entry> eldes BigQueryConnectionProperty.newBuilder() .setName(GCP_TELEMETRY_PROJECT_ID_PROPERTY_NAME) .setDescription("GCP Project ID for OTel exporter.") + .build(), + BigQueryConnectionProperty.newBuilder() + .setName(USE_GLOBAL_OTEL_PROPERTY_NAME) + .setDescription( + "Enables usage of the Global OpenTelemetry instance when true. Default is false.") + .setDefaultValue(String.valueOf(DEFAULT_USE_GLOBAL_OTEL_VALUE)) .build()))); private static final List NETWORK_PROPERTIES = diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java index c5f0f8a734c4..9e8cbff8be1a 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java @@ -912,20 +912,11 @@ private void processArrowStream( enqueueError(arrowBatchWrapperBlockingQueue, e); Thread.currentThread().interrupt(); } catch (Exception e) { - if (e.getCause() instanceof InterruptedException || Thread.currentThread().isInterrupted()) { - LOG.log( - Level.WARNING, - "\n" + Thread.currentThread().getName() + " Interrupted @ arrowStreamProcessor", - e); - enqueueError(arrowBatchWrapperBlockingQueue, e); - Thread.currentThread().interrupt(); - } else { - LOG.log( - Level.WARNING, - "\n" + Thread.currentThread().getName() + " Error @ arrowStreamProcessor", - e); - enqueueError(arrowBatchWrapperBlockingQueue, e); - } + LOG.log( + Level.WARNING, + "\n" + Thread.currentThread().getName() + " Error @ arrowStreamProcessor", + e); + enqueueError(arrowBatchWrapperBlockingQueue, e); } finally { // logic needed for graceful shutdown enqueueEndOfStream(arrowBatchWrapperBlockingQueue); } @@ -1683,20 +1674,11 @@ private void parseAndPopulateRpcData( } } catch (Exception ex) { - if (ex.getCause() instanceof InterruptedException || Thread.currentThread().isInterrupted()) { - LOG.log( - Level.WARNING, - "\n" + Thread.currentThread().getName() + " Interrupted @ populateBufferAsync", - ex); - enqueueBufferError(bigQueryFieldValueListWrapperBlockingQueue, ex); - Thread.currentThread().interrupt(); - } else { - LOG.log( - Level.WARNING, - "\n" + Thread.currentThread().getName() + " Error @ populateBufferAsync", - ex); - enqueueBufferError(bigQueryFieldValueListWrapperBlockingQueue, ex); - } + LOG.log( + Level.WARNING, + "\n" + Thread.currentThread().getName() + " Error @ populateBufferAsync", + ex); + enqueueBufferError(bigQueryFieldValueListWrapperBlockingQueue, ex); } finally { enqueueBufferEndOfStream(bigQueryFieldValueListWrapperBlockingQueue); } diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java index e2dfa888d8bf..7871c3eea07b 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java @@ -123,6 +123,7 @@ public class DataSource implements javax.sql.DataSource { private boolean enableGcpLogExporter = BigQueryJdbcUrlUtility.DEFAULT_ENABLE_GCP_LOG_EXPORTER_VALUE; private OpenTelemetry customOpenTelemetry; + private boolean useGlobalOpenTelemetry = BigQueryJdbcUrlUtility.DEFAULT_USE_GLOBAL_OTEL_VALUE; // Make sure the JDBC driver class is loaded. static { @@ -358,6 +359,12 @@ public class DataSource implements javax.sql.DataSource { ds.setEnableGcpLogExporter( BigQueryJdbcUrlUtility.convertIntToBoolean( val, BigQueryJdbcUrlUtility.ENABLE_GCP_LOG_EXPORTER_PROPERTY_NAME))) + .put( + BigQueryJdbcUrlUtility.USE_GLOBAL_OTEL_PROPERTY_NAME, + (ds, val) -> + ds.setUseGlobalOpenTelemetry( + BigQueryJdbcUrlUtility.convertIntToBoolean( + val, BigQueryJdbcUrlUtility.USE_GLOBAL_OTEL_PROPERTY_NAME))) .build(); public static DataSource fromUrl(String url) { @@ -675,6 +682,11 @@ Properties createProperties() { BigQueryJdbcUrlUtility.ENABLE_GCP_LOG_EXPORTER_PROPERTY_NAME, String.valueOf(this.enableGcpLogExporter)); } + if (this.useGlobalOpenTelemetry) { + connectionProperties.setProperty( + BigQueryJdbcUrlUtility.USE_GLOBAL_OTEL_PROPERTY_NAME, + String.valueOf(this.useGlobalOpenTelemetry)); + } return connectionProperties; } @@ -832,6 +844,14 @@ public void setCustomOpenTelemetry(OpenTelemetry customOpenTelemetry) { this.customOpenTelemetry = customOpenTelemetry; } + public boolean getUseGlobalOpenTelemetry() { + return useGlobalOpenTelemetry; + } + + public void setUseGlobalOpenTelemetry(boolean useGlobalOpenTelemetry) { + this.useGlobalOpenTelemetry = useGlobalOpenTelemetry; + } + public void setHighThroughputMinTableSize(Integer highThroughputMinTableSize) { if (highThroughputMinTableSize != null) { validateNonNegative( diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java index c8192cf47e0c..f652d91b3f2e 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java @@ -42,7 +42,9 @@ public class OpenTelemetryJulHandler extends Handler { private static final Pattern UNSAFE_LOG_CHARACTERS = Pattern.compile("[^a-zA-Z0-9./_-]"); - public OpenTelemetryJulHandler() {} + public OpenTelemetryJulHandler() { + setLevel(Level.ALL); + } @Override public void publish(LogRecord record) { diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java index 252d252588bc..039fe2e22a66 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java @@ -17,18 +17,35 @@ package com.google.cloud.bigquery.jdbc; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.when; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.rpc.HeaderProvider; import com.google.api.gax.rpc.TransportChannelProvider; +import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.QueryJobConfiguration.JobCreationMode; import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; +import com.google.cloud.logging.Logging; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; +import io.opentelemetry.sdk.trace.data.SpanData; import java.io.IOException; import java.io.InputStream; import java.sql.SQLException; +import java.util.List; import java.util.Optional; import java.util.Properties; import java.util.logging.Level; @@ -36,11 +53,16 @@ import java.util.logging.Logger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.MockedStatic; public class BigQueryConnectionTest extends BigQueryJdbcLoggingBaseTest { + @RegisterExtension + static final OpenTelemetryExtension otelTesting = OpenTelemetryExtension.create(); + private static final String DEFAULT_VERSION = "0.0.0"; private static final String DEFAULT_JDBC_TOKEN_VALUE = "Google-BigQuery-JDBC-Driver"; private static final String BASE_URL = @@ -461,6 +483,27 @@ public void testIsReadOnlyTokenProvided(String readonlyProp, boolean expectedIsR } } + @Test + public void testConnect_withCustomOpenTelemetry_usesCustomInstance() throws Exception { + DataSource ds = DataSource.fromUrl(BASE_URL); + ds.setCustomOpenTelemetry(otelTesting.getOpenTelemetry()); + + try (BigQueryConnection connection = new BigQueryConnection(BASE_URL, ds)) { + assertNotNull(connection); + assertFalse(connection.isClosed()); + + Tracer tracer = connection.getTracer(); + assertNotNull(tracer); + + Span span = tracer.spanBuilder("custom-otel-span").startSpan(); + span.end(); + + List spans = otelTesting.getSpans(); + assertEquals(1, spans.size()); + assertEquals("custom-otel-span", spans.get(0).getName()); + } + } + @Test public void testConnectionPropertiesLoggingAndMasking() throws IOException, SQLException { Logger rootLogger = BigQueryJdbcRootLogger.getRootLogger(); @@ -492,4 +535,133 @@ public void testConnectionPropertiesLoggingAndMasking() throws IOException, SQLE rootLogger.setLevel(originalLevel); } } + + @ParameterizedTest( + name = + "Case {index}: custom={0}, global={1}, trace={2}, log={3} -> expectTrace={4}, expectLog={5}") + @CsvSource({ + // hasCustom, useGlobal, enableTrace, enableLog, expectTrace, expectLog + "true, true, true, true, CUSTOM, CUSTOM", + "true, false, true, true, CUSTOM, CUSTOM", + "false, true, true, true, GLOBAL, GLOBAL", + "false, true, false, false, GLOBAL, GLOBAL", + "false, false, true, false, DRIVER_MANAGED, NONE", + "false, false, false, true, NONE, DRIVER_MANAGED", + "false, false, true, true, DRIVER_MANAGED, DRIVER_MANAGED", + "false, false, false, false, NONE, NONE" + }) + public void testOpenTelemetryPrecedenceHierarchy( + boolean hasCustom, + boolean useGlobal, + boolean enableTrace, + boolean enableLog, + String expectTrace, + String expectLog) + throws Exception { + + DataSource ds = DataSource.fromUrl(BASE_URL); + ds.setUseGlobalOpenTelemetry(useGlobal); + ds.setEnableGcpTraceExporter(enableTrace); + ds.setEnableGcpLogExporter(enableLog); + + OpenTelemetry mockCustomOtel = mock(OpenTelemetry.class); + OpenTelemetry mockGlobalOtel = mock(OpenTelemetry.class); + OpenTelemetry mockDriverManagedOtel = mock(OpenTelemetry.class); + Logging mockLogging = mock(Logging.class); + when(mockCustomOtel.getTracer(anyString())).thenReturn(mock(Tracer.class)); + when(mockGlobalOtel.getTracer(anyString())).thenReturn(mock(Tracer.class)); + when(mockDriverManagedOtel.getTracer(anyString())).thenReturn(mock(Tracer.class)); + + if (hasCustom) { + ds.setCustomOpenTelemetry(mockCustomOtel); + } + + try (MockedStatic mockedOtel = + mockStatic(BigQueryJdbcOpenTelemetry.class); + MockedStatic mockedAuth = + mockStatic(BigQueryJdbcOAuthUtility.class); + MockedStatic mockedCreds = mockStatic(GoogleCredentials.class)) { + + mockedCreds + .when(GoogleCredentials::getApplicationDefault) + .thenReturn(mock(GoogleCredentials.class)); + + // Mock parseOAuthProperties to always return ADC type to bypass validation + mockedAuth + .when(() -> BigQueryJdbcOAuthUtility.parseOAuthProperties(any(), anyString())) + .thenAnswer( + invocation -> { + java.util.Map props = new java.util.HashMap<>(); + props.put( + BigQueryJdbcUrlUtility.OAUTH_TYPE_PROPERTY_NAME, + "APPLICATION_DEFAULT_CREDENTIALS"); + return props; + }); + + mockedAuth + .when(() -> BigQueryJdbcOAuthUtility.getCredentials(any(), any(), any(), any())) + .thenReturn(mock(GoogleCredentials.class)); + + mockedOtel + .when( + () -> + BigQueryJdbcOpenTelemetry.createLoggingClient( + anyBoolean(), any(), any(), any(), any())) + .thenReturn(mockLogging); + + // Stub getOpenTelemetry to return the expected mock based on inputs + mockedOtel + .when( + () -> + BigQueryJdbcOpenTelemetry.getOpenTelemetry( + eq(useGlobal), + eq(enableTrace), + eq(enableLog), + hasCustom ? eq(mockCustomOtel) : isNull(), + any(), + any())) + .thenAnswer( + invocation -> { + if (hasCustom) return mockCustomOtel; + if (useGlobal) return mockGlobalOtel; + if (enableTrace || enableLog) return mockDriverManagedOtel; + return OpenTelemetry.noop(); + }); + + try (BigQueryConnection connection = new BigQueryConnection(BASE_URL, ds)) { + + boolean shouldBeRegistered = enableLog || hasCustom || useGlobal; + + if (!shouldBeRegistered) { + mockedOtel.verify( + () -> + BigQueryJdbcOpenTelemetry.registerConnection( + anyString(), any(), any(), anyBoolean()), + never()); + } else { + final OpenTelemetry expectedOtelInstance; + if ("CUSTOM".equals(expectTrace) || "CUSTOM".equals(expectLog)) { + expectedOtelInstance = mockCustomOtel; + } else if ("GLOBAL".equals(expectTrace) || "GLOBAL".equals(expectLog)) { + expectedOtelInstance = mockGlobalOtel; + } else if ("DRIVER_MANAGED".equals(expectTrace) || "DRIVER_MANAGED".equals(expectLog)) { + expectedOtelInstance = mockDriverManagedOtel; + } else { + expectedOtelInstance = OpenTelemetry.noop(); + } + + boolean expectUseDirectGcp = "DRIVER_MANAGED".equals(expectLog); + Logging expectedLogClient = expectUseDirectGcp ? mockLogging : null; + + mockedOtel.verify( + () -> + BigQueryJdbcOpenTelemetry.registerConnection( + anyString(), + eq(expectedOtelInstance), + eq(expectedLogClient), + eq(expectUseDirectGcp))); + } + } + } + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java index 9a6e7091d6c6..6f07b0e220d5 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.mock; import com.google.auth.oauth2.GoogleCredentials; +import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.Tracer; import org.junit.jupiter.api.AfterEach; @@ -50,7 +51,7 @@ public void testGetOpenTelemetry_withCustomSdk_returnsCustom() { OpenTelemetry mockCustomOtel = mock(OpenTelemetry.class); OpenTelemetry result = - BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, false, mockCustomOtel, null, null); + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, false, false, mockCustomOtel, null, null); assertThat(result).isSameInstanceAs(mockCustomOtel); } @@ -61,7 +62,7 @@ public void testGetOpenTelemetry_withCustomSdkAndFlags_returnsCustom() { // Custom SDK always takes precedence over individual flags OpenTelemetry result = - BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, true, mockCustomOtel, null, null); + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, true, true, mockCustomOtel, null, null); assertThat(result).isSameInstanceAs(mockCustomOtel); } @@ -69,7 +70,7 @@ public void testGetOpenTelemetry_withCustomSdkAndFlags_returnsCustom() { @Test public void testGetOpenTelemetry_noFlags_returnsNoop() { OpenTelemetry result = - BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, false, null, null, null); + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, false, false, null, null, null); assertThat(result).isSameInstanceAs(OpenTelemetry.noop()); } @@ -84,9 +85,9 @@ public void testGetTracer_respectsScopeName() { @Test public void testGetOpenTelemetry_cachesSdkInstances() { OpenTelemetry result1 = - BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, null, null, "project1"); + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, true, false, null, null, "project1"); OpenTelemetry result2 = - BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, null, null, "project1"); + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, true, false, null, null, "project1"); assertThat(result1).isSameInstanceAs(result2); } @@ -94,9 +95,9 @@ public void testGetOpenTelemetry_cachesSdkInstances() { @Test public void testGetOpenTelemetry_createsNewInstanceForDifferentKey() { OpenTelemetry result1 = - BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, null, null, "project1"); + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, true, false, null, null, "project1"); OpenTelemetry result2 = - BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, null, null, "project2"); + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, true, false, null, null, "project2"); assertThat(result1).isNotSameInstanceAs(result2); } @@ -104,9 +105,9 @@ public void testGetOpenTelemetry_createsNewInstanceForDifferentKey() { @Test public void testGetOpenTelemetry_createsNewInstanceForDifferentTraceFlag() { OpenTelemetry result1 = - BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, true, null, null, "project1"); + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, true, true, null, null, "project1"); OpenTelemetry result2 = - BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, true, null, null, "project1"); + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, false, true, null, null, "project1"); assertThat(result1).isNotSameInstanceAs(result2); } @@ -114,10 +115,18 @@ public void testGetOpenTelemetry_createsNewInstanceForDifferentTraceFlag() { @Test public void testGetOpenTelemetry_ignoresEnableLogFlagInCacheKey() { OpenTelemetry result1 = - BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, true, null, null, "project1"); + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, true, true, null, null, "project1"); OpenTelemetry result2 = - BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, null, null, "project1"); + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, true, false, null, null, "project1"); assertThat(result1).isSameInstanceAs(result2); } + + @Test + public void testGetOpenTelemetry_withUseGlobalOTel_returnsGlobal() { + OpenTelemetry result = + BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, false, null, null, null); + + assertThat(result).isSameInstanceAs(GlobalOpenTelemetry.get()); + } } diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITOpenTelemetryTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITOpenTelemetryTest.java new file mode 100644 index 000000000000..8e7ffa92c2e9 --- /dev/null +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITOpenTelemetryTest.java @@ -0,0 +1,265 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.jdbc.it; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.api.gax.paging.Page; +import com.google.cloud.ServiceOptions; +import com.google.cloud.bigquery.jdbc.BigQueryConnection; +import com.google.cloud.bigquery.jdbc.DataSource; +import com.google.cloud.logging.LogEntry; +import com.google.cloud.logging.Logging; +import com.google.cloud.logging.LoggingOptions; +import com.google.cloud.trace.v1.TraceServiceClient; +import com.google.devtools.cloudtrace.v1.Trace; +import com.google.devtools.cloudtrace.v1.TraceSpan; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; + +public class ITOpenTelemetryTest { + + private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); + private static final String CONNECTION_URL = + String.format( + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;ProjectId=%s;OAuthType=3;Timeout=3600;", + PROJECT_ID); + + @Test + public void testExecute_withOpenTelemetryGcpExporter() throws Exception { + + // Step 1: Connect with GCP Exporters enabled via DataSource + DataSource ds = DataSource.fromUrl(CONNECTION_URL); + ds.setEnableGcpTraceExporter(true); + ds.setEnableGcpLogExporter(true); + ds.setLogLevel("5"); // Triggers FINE log generation + ds.setGcpTelemetryProjectId(PROJECT_ID); + ds.setEnableHighThroughputAPI(false); + ds.setMaxResults(50L); // Forces small page size (50) to trigger pagination + + String connectionUuid = null; + + try (Connection connection = ds.getConnection(); + Statement statement = connection.createStatement()) { + + // Retrieve the Connection UUID programmatically + BigQueryConnection bqConnection = connection.unwrap(BigQueryConnection.class); + connectionUuid = bqConnection.getConnectionId(); + assertNotNull(connectionUuid, "Connection UUID should be generated"); + + // Execute an in-memory array query (scans 0 bytes, extremely fast) and force pagination (3 + // pages) + String paginationQuery = "SELECT * FROM UNNEST(GENERATE_ARRAY(1, 150)) AS id;"; + try (ResultSet paginatedRs = statement.executeQuery(paginationQuery)) { + while (paginatedRs.next()) { + // Drain the result set to trigger pagination fetches + } + } + } + + // Step 2: Retrieve and assert logs, harvesting the TraceId + String traceId = verifyAndFetchLogs(connectionUuid); + + // Step 3: Query Cloud Trace and assert parent-child hierarchy + Trace trace = verifyAndFetchTrace(traceId); + + boolean foundParentExecuteQuery = false; + boolean foundChildSdkSpans = false; + boolean foundPaginationSpans = false; + long parentSpanId = 0; + + for (TraceSpan span : trace.getSpansList()) { + String spanName = span.getName(); + if (spanName.equals("BigQueryStatement.executeQuery")) { + foundParentExecuteQuery = true; + parentSpanId = span.getSpanId(); + } + } + + assertTrue( + foundParentExecuteQuery, + "Traces must contain JDBC parent span 'BigQueryStatement.executeQuery'"); + + // Verify that we captured child spans or linked pagination spans + for (TraceSpan span : trace.getSpansList()) { + if (span.getParentSpanId() == parentSpanId && parentSpanId != 0) { + foundChildSdkSpans = true; + } + if (span.getName().equals("BigQueryStatement.pagination")) { + foundPaginationSpans = true; + } + } + + assertTrue(foundPaginationSpans, "OTel pagination must generate pagination spans"); + assertTrue( + foundChildSdkSpans, + "OTel context must propagate parent to downstream pagination child spans"); + } + + @Test + public void testExecute_withErrorCorrelation() throws Exception { + + // Step 1: Connect with GCP Exporters enabled via DataSource + DataSource ds = DataSource.fromUrl(CONNECTION_URL); + ds.setEnableGcpTraceExporter(true); + ds.setEnableGcpLogExporter(true); + ds.setLogLevel("5"); // Triggers FINE log generation + ds.setGcpTelemetryProjectId(PROJECT_ID); + + String connectionUuid = null; + + try (Connection connection = ds.getConnection(); + Statement statement = connection.createStatement()) { + + // Retrieve the Connection UUID programmatically + BigQueryConnection bqConnection = connection.unwrap(BigQueryConnection.class); + connectionUuid = bqConnection.getConnectionId(); + assertNotNull(connectionUuid, "Connection UUID should be generated"); + + // Execute a query designed to fail instantly due to syntax error (compiler-level failure) + assertThrows(SQLException.class, () -> statement.executeQuery("SELECT * FROM;")); + } + + // Step 2: Retrieve and assert logs, harvesting the TraceId + String traceId = verifyAndFetchLogs(connectionUuid); + + // Step 3: Query Cloud Trace and assert span status is ERROR + Trace trace = verifyAndFetchTrace(traceId); + + boolean foundParentExecuteQuery = false; + + for (TraceSpan span : trace.getSpansList()) { + String spanName = span.getName(); + if (spanName.equals("BigQueryStatement.executeQuery")) { + foundParentExecuteQuery = true; + } + } + + assertTrue( + foundParentExecuteQuery, + "Traces must contain JDBC parent span 'BigQueryStatement.executeQuery'"); + } + + private String verifyAndFetchLogs(String connectionUuid) throws Exception { + try (Logging logging = + LoggingOptions.newBuilder().setProjectId(PROJECT_ID).build().getService()) { + String filter = + "logName:\"projects/" + + PROJECT_ID + + "/logs/com.google.cloud.bigquery\" AND labels.\"jdbc.connection_id\"=\"" + + connectionUuid + + "\""; + + List entries = fetchLogsWithRetry(logging, filter); + assertFalse(entries.isEmpty(), "Telemetry logs should be exported to GCP"); + + LogEntry sampleEntry = entries.get(0); + String traceId = sampleEntry.getTrace(); + String hexSpanId = sampleEntry.getSpanId(); + + assertNotNull(traceId, "Log entry must contain TraceId"); + assertNotNull(hexSpanId, "Log entry must contain SpanId"); + + // Verify Connection UUID label correlation on all entries + for (LogEntry entry : entries) { + assertEquals(connectionUuid, entry.getLabels().get("jdbc.connection_id")); + } + + return traceId; + } + } + + private Trace verifyAndFetchTrace(String traceId) throws Exception { + String hexTraceId = traceId; + if (traceId.contains("/traces/")) { + hexTraceId = traceId.substring(traceId.lastIndexOf("/traces/") + 8); + } + + try (TraceServiceClient traceClient = TraceServiceClient.create()) { + Trace trace = fetchTraceWithRetry(traceClient, PROJECT_ID, hexTraceId); + assertNotNull(trace, "Trace must be found in Cloud Trace API: " + hexTraceId); + return trace; + } + } + + private T pollWithRetry(java.util.concurrent.Callable task) throws InterruptedException { + int attempts = 0; + int maxAttempts = 30; // 30 attempts * 500ms = 15 seconds max delay + long delayMs = 500; // 500ms linear polling + + while (attempts < maxAttempts) { + attempts++; + try { + T result = task.call(); + if (result != null) { + return result; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Test execution interrupted", e); + } catch (Exception e) { + // Ignore exceptions during remote lookup and retry + } + if (attempts < maxAttempts) { + Thread.sleep(delayMs); + } + } + return null; + } + + private List fetchLogsWithRetry(Logging logging, String filter) + throws InterruptedException { + List result = + pollWithRetry( + () -> { + Page entriesPage = + logging.listLogEntries( + Logging.EntryListOption.filter(filter), Logging.EntryListOption.pageSize(50)); + List entries = new ArrayList<>(); + entriesPage.iterateAll().forEach(entries::add); + return entries.isEmpty() ? null : entries; + }); + return result != null ? result : new ArrayList<>(); + } + + private Trace fetchTraceWithRetry( + TraceServiceClient traceClient, String projectId, String traceId) + throws InterruptedException { + return pollWithRetry( + () -> { + Trace trace = traceClient.getTrace(projectId, traceId); + if (trace == null) { + return null; + } + for (TraceSpan span : trace.getSpansList()) { + if (span.getName().equals("BigQueryStatement.executeQuery")) { + return trace; + } + } + return null; + }); + } +} diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/suites/ITNightlyTests.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/suites/ITNightlyTests.java index 2700c18adbbd..02fc96c17bad 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/suites/ITNightlyTests.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/suites/ITNightlyTests.java @@ -19,9 +19,15 @@ import com.google.cloud.bigquery.jdbc.it.ITAuthTests; import com.google.cloud.bigquery.jdbc.it.ITBigQueryJDBCTest; import com.google.cloud.bigquery.jdbc.it.ITNightlyBigQueryTest; +import com.google.cloud.bigquery.jdbc.it.ITOpenTelemetryTest; import org.junit.platform.suite.api.SelectClasses; import org.junit.platform.suite.api.Suite; @Suite -@SelectClasses({ITAuthTests.class, ITBigQueryJDBCTest.class, ITNightlyBigQueryTest.class}) +@SelectClasses({ + ITAuthTests.class, + ITBigQueryJDBCTest.class, + ITNightlyBigQueryTest.class, + ITOpenTelemetryTest.class +}) public class ITNightlyTests {}