From 6e3e07a22b8397e1e9d5b567589e44abc55961f2 Mon Sep 17 00:00:00 2001 From: Liam Huffman <44932470+whuffman36@users.noreply.github.com> Date: Mon, 9 Jun 2025 10:30:36 -0700 Subject: [PATCH 01/24] feat(bigquery): Integrate Otel in client lib (#3747) * feat(bigquery): Integrate Otel in client lib * Refactor Otel code into separate file * Add copyright comment * fix style * remove unused dependencies * remove unused test dependencies * fix dependency issue * fix it test * refactor out OtelHelper class * change test name to start with lowercase * change datasetId key to dataset * remove OtelHelper class * Clean up attribute values * Add spans to query and listPartitions * remove extraneous attributes * add attribute namespaces, change to snake_case * Resolve merge conflict * change previewEnable to JobCreationMode * fix style --- google-cloud-bigquery/pom.xml | 27 + .../google/cloud/bigquery/BigQueryImpl.java | 1012 +++++++++++++---- .../cloud/bigquery/BigQueryOptions.java | 50 + .../com/google/cloud/bigquery/DatasetId.java | 8 + .../google/cloud/bigquery/DatasetInfo.java | 13 + .../cloud/bigquery/InsertAllRequest.java | 12 + .../java/com/google/cloud/bigquery/JobId.java | 13 + .../com/google/cloud/bigquery/ModelId.java | 9 + .../com/google/cloud/bigquery/ModelInfo.java | 16 + .../com/google/cloud/bigquery/RoutineId.java | 9 + .../google/cloud/bigquery/RoutineInfo.java | 14 + .../com/google/cloud/bigquery/TableId.java | 9 + .../com/google/cloud/bigquery/TableInfo.java | 16 + .../cloud/bigquery/it/ITBigQueryTest.java | 202 ++++ pom.xml | 9 + 15 files changed, 1225 insertions(+), 194 deletions(-) diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 07cc6b0fb..9599d760c 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -156,6 +156,16 @@ error_prone_annotations + + + io.opentelemetry + opentelemetry-api + + + io.opentelemetry + opentelemetry-context + + com.google.api @@ -208,6 +218,23 @@ proto-google-cloud-datacatalog-v1 test + + + + io.opentelemetry + opentelemetry-sdk + test + + + io.opentelemetry + opentelemetry-sdk-common + test + + + io.opentelemetry + opentelemetry-sdk-trace + test + diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index 9ede02838..2efe8f92a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -51,6 +51,9 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Scope; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -268,7 +271,18 @@ public Dataset create(DatasetInfo datasetInfo, DatasetOption... options) { : datasetInfo.getDatasetId().getProject()) .toPb(); final Map optionsMap = optionMap(options); - try { + Span datasetCreate = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + datasetCreate = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.createDataset") + .setAllAttributes(datasetInfo.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope datasetCreateScope = datasetCreate != null ? datasetCreate.makeCurrent() : null) { return Dataset.fromPb( this, BigQueryRetryHelper.runWithRetries( @@ -284,6 +298,10 @@ public com.google.api.services.bigquery.model.Dataset call() throws IOException EMPTY_RETRY_CONFIG)); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (datasetCreate != null) { + datasetCreate.end(); + } } } @@ -298,7 +316,18 @@ public Table create(TableInfo tableInfo, TableOption... options) { .toPb(); handleExternalTableSchema(tablePb); final Map optionsMap = optionMap(options); - try { + Span tableCreate = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + tableCreate = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.createTable") + .setAllAttributes(tableInfo.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope tableCreateScope = tableCreate != null ? tableCreate.makeCurrent() : null) { return Table.fromPb( this, BigQueryRetryHelper.runWithRetries( @@ -314,6 +343,10 @@ public com.google.api.services.bigquery.model.Table call() throws IOException { EMPTY_RETRY_CONFIG)); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (tableCreate != null) { + tableCreate.end(); + } } } @@ -337,7 +370,18 @@ public Routine create(RoutineInfo routineInfo, RoutineOption... options) { : routineInfo.getRoutineId().getProject()) .toPb(); final Map optionsMap = optionMap(options); - try { + Span routineCreate = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + routineCreate = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.createRoutine") + .setAllAttributes(routineInfo.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope createRoutineScope = routineCreate != null ? routineCreate.makeCurrent() : null) { return Routine.fromPb( this, BigQueryRetryHelper.runWithRetries( @@ -353,6 +397,10 @@ public com.google.api.services.bigquery.model.Routine call() throws IOException EMPTY_RETRY_CONFIG)); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (routineCreate != null) { + routineCreate.end(); + } } } @@ -388,6 +436,17 @@ Job create(JobInfo jobInfo, Supplier idProvider, JobOption... options) { final boolean idRandom = (jobInfo.getJobId() == null); final Map optionsMap = optionMap(options); + Span jobCreate = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + jobCreate = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.createJob") + .setAllAttributes(jobInfo.getJobId().getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } BigQueryException createException; // NOTE(pongad): This double-try structure is admittedly odd. // translateAndThrow itself throws, and pretends to return an exception only @@ -396,90 +455,96 @@ Job create(JobInfo jobInfo, Supplier idProvider, JobOption... options) { // Fixing this entails some work on BaseServiceException.translate. // Since that affects a bunch of APIs, we should fix this as a separate change. final JobId[] finalJobId = new JobId[1]; - try { + try (Scope jobCreateScope = jobCreate != null ? jobCreate.makeCurrent() : null) { try { - return Job.fromPb( - this, - BigQueryRetryHelper.runWithRetries( - new Callable() { - @Override - public com.google.api.services.bigquery.model.Job call() throws IOException { - if (idRandom) { - // re-generate a new random job with the same jobInfo when jobId is not - // provided by the user - JobInfo recreatedJobInfo = - jobInfo.toBuilder().setJobId(idProvider.get()).build(); - com.google.api.services.bigquery.model.Job newJobPb = - recreatedJobInfo.setProjectId(getOptions().getProjectId()).toPb(); - finalJobId[0] = recreatedJobInfo.getJobId(); - return bigQueryRpc.createSkipExceptionTranslation(newJobPb, optionsMap); - } else { - com.google.api.services.bigquery.model.Job jobPb = - jobInfo.setProjectId(getOptions().getProjectId()).toPb(); - return bigQueryRpc.createSkipExceptionTranslation(jobPb, optionsMap); + try { + return Job.fromPb( + this, + BigQueryRetryHelper.runWithRetries( + new Callable() { + @Override + public com.google.api.services.bigquery.model.Job call() throws IOException { + if (idRandom) { + // re-generate a new random job with the same jobInfo when jobId is not + // provided by the user + JobInfo recreatedJobInfo = + jobInfo.toBuilder().setJobId(idProvider.get()).build(); + com.google.api.services.bigquery.model.Job newJobPb = + recreatedJobInfo.setProjectId(getOptions().getProjectId()).toPb(); + finalJobId[0] = recreatedJobInfo.getJobId(); + return bigQueryRpc.createSkipExceptionTranslation(newJobPb, optionsMap); + } else { + com.google.api.services.bigquery.model.Job jobPb = + jobInfo.setProjectId(getOptions().getProjectId()).toPb(); + return bigQueryRpc.createSkipExceptionTranslation(jobPb, optionsMap); + } } - } - }, - getRetryOptions(optionsMap) != null - ? RetryOption.mergeToSettings( - getOptions().getRetrySettings(), getRetryOptions(optionsMap)) - : getOptions().getRetrySettings(), - BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, - getOptions().getClock(), - getBigQueryRetryConfig(optionsMap) != null - ? getBigQueryRetryConfig(optionsMap) - : DEFAULT_RETRY_CONFIG)); - } catch (BigQueryRetryHelperException e) { - throw BigQueryException.translateAndThrow(e); + }, + getRetryOptions(optionsMap) != null + ? RetryOption.mergeToSettings( + getOptions().getRetrySettings(), getRetryOptions(optionsMap)) + : getOptions().getRetrySettings(), + BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, + getOptions().getClock(), + getBigQueryRetryConfig(optionsMap) != null + ? getBigQueryRetryConfig(optionsMap) + : DEFAULT_RETRY_CONFIG)); + } catch (BigQueryRetryHelperException e) { + throw BigQueryException.translateAndThrow(e); + } + } catch (BigQueryException e) { + createException = e; } - } catch (BigQueryException e) { - createException = e; - } - - if (!idRandom) { - if (createException instanceof BigQueryException - && createException.getCause() != null - && createException.getCause().getMessage() != null) { - - Pattern pattern = Pattern.compile(".*Already.*Exists:.*Job.*", Pattern.CASE_INSENSITIVE); - Matcher matcher = pattern.matcher(createException.getCause().getMessage()); - - if (matcher.find()) { - // If the Job ALREADY EXISTS, retrieve it. - Job job = this.getJob(jobInfo.getJobId(), JobOption.fields(JobField.STATISTICS)); - - long jobCreationTime = job.getStatistics().getCreationTime(); - long jobMinStaleTime = System.currentTimeMillis(); - long jobMaxStaleTime = - java.time.Instant.ofEpochMilli(jobMinStaleTime) - .minus(1, java.time.temporal.ChronoUnit.DAYS) - .toEpochMilli(); - - // Only return the job if it has been created in the past 24 hours. - // This is assuming any job older than 24 hours is a valid duplicate JobID - // and not a false positive like b/290419183 - if (jobCreationTime >= jobMaxStaleTime && jobCreationTime <= jobMinStaleTime) { - return job; + + if (!idRandom) { + if (createException instanceof BigQueryException + && createException.getCause() != null + && createException.getCause().getMessage() != null) { + + Pattern pattern = Pattern.compile(".*Already.*Exists:.*Job.*", Pattern.CASE_INSENSITIVE); + Matcher matcher = pattern.matcher(createException.getCause().getMessage()); + + if (matcher.find()) { + // If the Job ALREADY EXISTS, retrieve it. + Job job = this.getJob(jobInfo.getJobId(), JobOption.fields(JobField.STATISTICS)); + + long jobCreationTime = job.getStatistics().getCreationTime(); + long jobMinStaleTime = System.currentTimeMillis(); + long jobMaxStaleTime = + java.time.Instant.ofEpochMilli(jobMinStaleTime) + .minus(1, java.time.temporal.ChronoUnit.DAYS) + .toEpochMilli(); + + // Only return the job if it has been created in the past 24 hours. + // This is assuming any job older than 24 hours is a valid duplicate JobID + // and not a false positive like b/290419183 + if (jobCreationTime >= jobMaxStaleTime && jobCreationTime <= jobMinStaleTime) { + return job; + } } } + throw createException; } - throw createException; - } - // If create RPC fails, it's still possible that the job has been successfully - // created, and get might work. - // We can only do this if we randomly generated the ID. Otherwise we might - // mistakenly fetch a job created by someone else. - Job job; - try { - job = getJob(finalJobId[0]); - } catch (BigQueryException e) { - throw createException; - } - if (job == null) { - throw createException; + // If create RPC fails, it's still possible that the job has been successfully + // created, and get might work. + // We can only do this if we randomly generated the ID. Otherwise we might + // mistakenly fetch a job created by someone else. + Job job; + try { + job = getJob(finalJobId[0]); + } catch (BigQueryException e) { + throw createException; + } + if (job == null) { + throw createException; + } + return job; + } finally { + if (jobCreate != null) { + jobCreate.end(); + } } - return job; } @Override @@ -491,7 +556,18 @@ public Dataset getDataset(String datasetId, DatasetOption... options) { public Dataset getDataset(final DatasetId datasetId, DatasetOption... options) { final DatasetId completeDatasetId = datasetId.setProjectId(getOptions().getProjectId()); final Map optionsMap = optionMap(options); - try { + Span datasetGet = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + datasetGet = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.getDataset") + .setAllAttributes(completeDatasetId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope datasetGetScope = datasetGet != null ? datasetGet.makeCurrent() : null) { com.google.api.services.bigquery.model.Dataset answer = BigQueryRetryHelper.runWithRetries( new Callable() { @@ -514,6 +590,10 @@ public com.google.api.services.bigquery.model.Dataset call() throws IOException return null; } throw BigQueryException.translateAndThrow(e); + } finally { + if (datasetGet != null) { + datasetGet.end(); + } } } @@ -524,7 +604,24 @@ public Page listDatasets(DatasetListOption... options) { @Override public Page listDatasets(String projectId, DatasetListOption... options) { - return listDatasets(projectId, getOptions(), optionMap(options)); + Span datasetsList = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + datasetsList = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.listDatasets") + .setAttribute("bq.dataset.project_id", projectId) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope datasetsListScope = datasetsList != null ? datasetsList.makeCurrent() : null) { + return listDatasets(projectId, getOptions(), optionMap(options)); + } finally { + if (datasetsList != null) { + datasetsList.end(); + } + } } private static Page listDatasets( @@ -574,7 +671,18 @@ public boolean delete(String datasetId, DatasetDeleteOption... options) { public boolean delete(DatasetId datasetId, DatasetDeleteOption... options) { final DatasetId completeDatasetId = datasetId.setProjectId(getOptions().getProjectId()); final Map optionsMap = optionMap(options); - try { + Span datasetDelete = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + datasetDelete = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.deleteDataset") + .setAllAttributes(datasetId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope datasetDeleteScope = datasetDelete != null ? datasetDelete.makeCurrent() : null) { return BigQueryRetryHelper.runWithRetries( new Callable() { @Override @@ -592,6 +700,10 @@ public Boolean call() throws IOException { return false; } throw BigQueryException.translateAndThrow(e); + } finally { + if (datasetDelete != null) { + datasetDelete.end(); + } } } @@ -607,7 +719,17 @@ public boolean delete(TableId tableId) { Strings.isNullOrEmpty(tableId.getProject()) ? getOptions().getProjectId() : tableId.getProject()); - try { + Span tableDelete = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + tableDelete = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.deleteTable") + .setAllAttributes(tableId.getOtelAttributes()) + .startSpan(); + } + try (Scope tableDeleteScope = tableDelete != null ? tableDelete.makeCurrent() : null) { return BigQueryRetryHelper.runWithRetries( new Callable() { @Override @@ -627,6 +749,10 @@ public Boolean call() throws IOException { return false; } throw BigQueryException.translateAndThrow(e); + } finally { + if (tableDelete != null) { + tableDelete.end(); + } } } @@ -637,7 +763,17 @@ public boolean delete(ModelId modelId) { Strings.isNullOrEmpty(modelId.getProject()) ? getOptions().getProjectId() : modelId.getProject()); - try { + Span modelDelete = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + modelDelete = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.deleteModel") + .setAllAttributes(modelId.getOtelAttributes()) + .startSpan(); + } + try (Scope modelDeleteScope = modelDelete != null ? modelDelete.makeCurrent() : null) { return BigQueryRetryHelper.runWithRetries( new Callable() { @Override @@ -657,6 +793,10 @@ public Boolean call() throws IOException { return false; } throw BigQueryException.translateAndThrow(e); + } finally { + if (modelDelete != null) { + modelDelete.end(); + } } } @@ -667,7 +807,17 @@ public boolean delete(RoutineId routineId) { Strings.isNullOrEmpty(routineId.getProject()) ? getOptions().getProjectId() : routineId.getProject()); - try { + Span routineDelete = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + routineDelete = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.deleteRoutine") + .setAllAttributes(routineId.getOtelAttributes()) + .startSpan(); + } + try (Scope routineDeleteScope = routineDelete != null ? routineDelete.makeCurrent() : null) { return BigQueryRetryHelper.runWithRetries( new Callable() { @Override @@ -687,6 +837,10 @@ public Boolean call() throws IOException { return false; } throw BigQueryException.translateAndThrow(e); + } finally { + if (routineDelete != null) { + routineDelete.end(); + } } } @@ -697,6 +851,16 @@ public boolean delete(JobId jobId) { Strings.isNullOrEmpty(jobId.getProject()) ? getOptions().getProjectId() : jobId.getProject()); + Span jobDelete = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + jobDelete = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.deleteJob") + .setAllAttributes(completeJobId.getOtelAttributes()) + .startSpan(); + } try { return BigQueryRetryHelper.runWithRetries( new Callable() { @@ -712,6 +876,10 @@ public Boolean call() throws IOException { EMPTY_RETRY_CONFIG); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (jobDelete != null) { + jobDelete.end(); + } } } @@ -720,7 +888,18 @@ public Dataset update(DatasetInfo datasetInfo, DatasetOption... options) { final com.google.api.services.bigquery.model.Dataset datasetPb = datasetInfo.setProjectId(getOptions().getProjectId()).toPb(); final Map optionsMap = optionMap(options); - try { + Span datasetUpdate = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + datasetUpdate = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.updateDataset") + .setAllAttributes(datasetInfo.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope datasetUpdateScope = datasetUpdate != null ? datasetUpdate.makeCurrent() : null) { return Dataset.fromPb( this, BigQueryRetryHelper.runWithRetries( @@ -736,6 +915,10 @@ public com.google.api.services.bigquery.model.Dataset call() throws IOException EMPTY_RETRY_CONFIG)); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (datasetUpdate != null) { + datasetUpdate.end(); + } } } @@ -750,7 +933,18 @@ public Table update(TableInfo tableInfo, TableOption... options) { .toPb(); handleExternalTableSchema(tablePb); final Map optionsMap = optionMap(options); - try { + Span tableUpdate = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + tableUpdate = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.updateTable") + .setAllAttributes(tableInfo.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope tableUpdateScope = tableUpdate != null ? tableUpdate.makeCurrent() : null) { return Table.fromPb( this, BigQueryRetryHelper.runWithRetries( @@ -766,6 +960,10 @@ public com.google.api.services.bigquery.model.Table call() throws IOException { EMPTY_RETRY_CONFIG)); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (tableUpdate != null) { + tableUpdate.end(); + } } } @@ -779,7 +977,18 @@ public Model update(ModelInfo modelInfo, ModelOption... options) { : modelInfo.getModelId().getProject()) .toPb(); final Map optionsMap = optionMap(options); - try { + Span modelUpdate = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + modelUpdate = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.updateModel") + .setAllAttributes(modelInfo.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope modelUpdateScope = modelUpdate != null ? modelUpdate.makeCurrent() : null) { return Model.fromPb( this, BigQueryRetryHelper.runWithRetries( @@ -795,6 +1004,10 @@ public com.google.api.services.bigquery.model.Model call() throws IOException { EMPTY_RETRY_CONFIG)); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (modelUpdate != null) { + modelUpdate.end(); + } } } @@ -808,7 +1021,18 @@ public Routine update(RoutineInfo routineInfo, RoutineOption... options) { : routineInfo.getRoutineId().getProject()) .toPb(); final Map optionsMap = optionMap(options); - try { + Span routineUpdate = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + routineUpdate = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.updateRoutine") + .setAllAttributes(routineInfo.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope routineUpdateScope = routineUpdate != null ? routineUpdate.makeCurrent() : null) { return Routine.fromPb( this, BigQueryRetryHelper.runWithRetries( @@ -824,6 +1048,10 @@ public com.google.api.services.bigquery.model.Routine call() throws IOException EMPTY_RETRY_CONFIG)); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (routineUpdate != null) { + routineUpdate.end(); + } } } @@ -842,7 +1070,18 @@ public Table getTable(TableId tableId, TableOption... options) { ? getOptions().getProjectId() : tableId.getProject()); final Map optionsMap = optionMap(options); - try { + Span tableGet = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + tableGet = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.getTable") + .setAllAttributes(tableId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope tableGetScope = tableGet != null ? tableGet.makeCurrent() : null) { com.google.api.services.bigquery.model.Table answer = BigQueryRetryHelper.runWithRetries( new Callable() { @@ -868,6 +1107,10 @@ public com.google.api.services.bigquery.model.Table call() throws IOException { return null; } throw BigQueryException.translateAndThrow(e); + } finally { + if (tableGet != null) { + tableGet.end(); + } } } @@ -884,7 +1127,18 @@ public Model getModel(ModelId modelId, ModelOption... options) { ? getOptions().getProjectId() : modelId.getProject()); final Map optionsMap = optionMap(options); - try { + Span modelGet = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + modelGet = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.getModel") + .setAllAttributes(modelId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope modelGetScope = modelGet != null ? modelGet.makeCurrent() : null) { com.google.api.services.bigquery.model.Model answer = BigQueryRetryHelper.runWithRetries( new Callable() { @@ -910,6 +1164,10 @@ public com.google.api.services.bigquery.model.Model call() throws IOException { return null; } throw BigQueryException.translateAndThrow(e); + } finally { + if (modelGet != null) { + modelGet.end(); + } } } @@ -926,7 +1184,18 @@ public Routine getRoutine(RoutineId routineId, RoutineOption... options) { ? getOptions().getProjectId() : routineId.getProject()); final Map optionsMap = optionMap(options); - try { + Span routineGet = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + routineGet = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.getRoutine") + .setAllAttributes(routineId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope routineGetScope = routineGet != null ? routineGet.makeCurrent() : null) { com.google.api.services.bigquery.model.Routine answer = BigQueryRetryHelper.runWithRetries( new Callable() { @@ -952,67 +1221,189 @@ public com.google.api.services.bigquery.model.Routine call() throws IOException return null; } throw BigQueryException.translateAndThrow(e); + } finally { + if (routineGet != null) { + routineGet.end(); + } } } @Override public Page listTables(String datasetId, TableListOption... options) { - return listTables( - DatasetId.of(getOptions().getProjectId(), datasetId), getOptions(), optionMap(options)); + Span tablesList = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + tablesList = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.listTables") + .setAllAttributes(DatasetId.of(datasetId).getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope tablesListScope = tablesList != null ? tablesList.makeCurrent() : null) { + return listTables( + DatasetId.of(getOptions().getProjectId(), datasetId), getOptions(), optionMap(options)); + } finally { + if (tablesList != null) { + tablesList.end(); + } + } } @Override public Page
listTables(DatasetId datasetId, TableListOption... options) { DatasetId completeDatasetId = datasetId.setProjectId(getOptions().getProjectId()); - return listTables(completeDatasetId, getOptions(), optionMap(options)); + Span tablesList = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + tablesList = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.listTables") + .setAllAttributes(completeDatasetId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope tablesListScope = tablesList != null ? tablesList.makeCurrent() : null) { + return listTables(completeDatasetId, getOptions(), optionMap(options)); + } finally { + if (tablesList != null) { + tablesList.end(); + } + } } @Override public Page listModels(String datasetId, ModelListOption... options) { - return listModels( - DatasetId.of(getOptions().getProjectId(), datasetId), getOptions(), optionMap(options)); + Span modelsList = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + modelsList = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.listModels") + .setAllAttributes(DatasetId.of(datasetId).getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope modelsListScope = modelsList != null ? modelsList.makeCurrent() : null) { + return listModels( + DatasetId.of(getOptions().getProjectId(), datasetId), getOptions(), optionMap(options)); + } finally { + if (modelsList != null) { + modelsList.end(); + } + } } @Override public Page listModels(DatasetId datasetId, ModelListOption... options) { DatasetId completeDatasetId = datasetId.setProjectId(getOptions().getProjectId()); - return listModels(completeDatasetId, getOptions(), optionMap(options)); + Span modelsList = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + modelsList = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.listModels") + .setAllAttributes(datasetId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope modelsListScope = modelsList != null ? modelsList.makeCurrent() : null) { + return listModels(completeDatasetId, getOptions(), optionMap(options)); + } finally { + if (modelsList != null) { + modelsList.end(); + } + } } @Override public Page listRoutines(String datasetId, RoutineListOption... options) { - return listRoutines( - DatasetId.of(getOptions().getProjectId(), datasetId), getOptions(), optionMap(options)); + Span routinesList = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + routinesList = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.listRoutines") + .setAllAttributes(DatasetId.of(datasetId).getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope routinesListScope = routinesList != null ? routinesList.makeCurrent() : null) { + return listRoutines( + DatasetId.of(getOptions().getProjectId(), datasetId), getOptions(), optionMap(options)); + } finally { + if (routinesList != null) { + routinesList.end(); + } + } } @Override public Page listRoutines(DatasetId datasetId, RoutineListOption... options) { DatasetId completeDatasetId = datasetId.setProjectId(getOptions().getProjectId()); - return listRoutines(completeDatasetId, getOptions(), optionMap(options)); + Span routinesList = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + routinesList = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.listRoutines") + .setAllAttributes(datasetId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope routinesListScope = routinesList != null ? routinesList.makeCurrent() : null) { + return listRoutines(completeDatasetId, getOptions(), optionMap(options)); + } finally { + if (routinesList != null) { + routinesList.end(); + } + } } @Override public List listPartitions(TableId tableId) { - List partitions = new ArrayList(); - String partitionsTable = tableId.getTable() + "$__PARTITIONS_SUMMARY__"; - TableId metaTableId = - tableId.getProject() == null - ? TableId.of(tableId.getDataset(), partitionsTable) - : TableId.of(tableId.getProject(), tableId.getDataset(), partitionsTable); - Table metaTable = getTable(metaTableId); - Schema metaSchema = metaTable.getDefinition().getSchema(); - String partition_id = null; - for (Field field : metaSchema.getFields()) { - if (field.getName().equals("partition_id")) { - partition_id = field.getName(); - break; - } + Span listPartitions = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + listPartitions = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.listPartitions") + .setAllAttributes(tableId.getOtelAttributes()) + .startSpan(); } - TableResult result = metaTable.list(metaSchema); - for (FieldValueList list : result.iterateAll()) { - partitions.add(list.get(partition_id).getStringValue()); + try (Scope listPartitionsScope = listPartitions != null ? listPartitions.makeCurrent() : null) { + List partitions = new ArrayList(); + String partitionsTable = tableId.getTable() + "$__PARTITIONS_SUMMARY__"; + TableId metaTableId = + tableId.getProject() == null + ? TableId.of(tableId.getDataset(), partitionsTable) + : TableId.of(tableId.getProject(), tableId.getDataset(), partitionsTable); + Table metaTable = getTable(metaTableId); + Schema metaSchema = metaTable.getDefinition().getSchema(); + String partition_id = null; + for (Field field : metaSchema.getFields()) { + if (field.getName().equals("partition_id")) { + partition_id = field.getName(); + break; + } + } + TableResult result = metaTable.list(metaSchema); + for (FieldValueList list : result.iterateAll()) { + partitions.add(list.get(partition_id).getStringValue()); + } + return partitions; + } finally { + if (listPartitions != null) { + listPartitions.end(); + } } - return partitions; } private static Page
listTables( @@ -1165,33 +1556,52 @@ public Rows apply(RowToInsert rowToInsert) { requestPb.setRows(rowsPb); TableDataInsertAllResponse responsePb; - if (allInsertIdsSet[0]) { - // allowing retries only if all row insertIds are set (used for deduplication) - try { + Span insertAll = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + insertAll = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.insertAll") + .setAllAttributes(request.getOtelAttributes()) + .startSpan(); + } + try (Scope insertAllScope = insertAll != null ? insertAll.makeCurrent() : null) { + if (allInsertIdsSet[0]) { + // allowing retries only if all row insertIds are set (used for deduplication) + try { + responsePb = + BigQueryRetryHelper.runWithRetries( + new Callable() { + @Override + public TableDataInsertAllResponse call() throws Exception { + return bigQueryRpc.insertAllSkipExceptionTranslation( + tableId.getProject(), + tableId.getDataset(), + tableId.getTable(), + requestPb); + } + }, + getOptions().getRetrySettings(), + BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, + getOptions().getClock(), + EMPTY_RETRY_CONFIG); + } catch (BigQueryRetryHelperException e) { + throw BigQueryException.translateAndThrow(e); + } + } else { + // Use insertAll that translate the exception as we are not retrying. responsePb = - BigQueryRetryHelper.runWithRetries( - new Callable() { - @Override - public TableDataInsertAllResponse call() throws Exception { - return bigQueryRpc.insertAllSkipExceptionTranslation( - tableId.getProject(), tableId.getDataset(), tableId.getTable(), requestPb); - } - }, - getOptions().getRetrySettings(), - BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, - getOptions().getClock(), - EMPTY_RETRY_CONFIG); - } catch (BigQueryRetryHelperException e) { - throw BigQueryException.translateAndThrow(e); + bigQueryRpc.insertAll( + tableId.getProject(), tableId.getDataset(), tableId.getTable(), requestPb); } - } else { - // Use insertAll that translate the exception as we are not retrying. - responsePb = - bigQueryRpc.insertAll( - tableId.getProject(), tableId.getDataset(), tableId.getTable(), requestPb); - } - return InsertAllResponse.fromPb(responsePb); + return InsertAllResponse.fromPb(responsePb); + } finally { + if (insertAll != null) { + insertAll.end(); + } + } } @Override @@ -1215,11 +1625,28 @@ public TableResult listTableData( public TableResult listTableData(TableId tableId, Schema schema, TableDataListOption... options) { Tuple, Long> data = listTableData(tableId, schema, getOptions(), optionMap(options)); - return TableResult.newBuilder() - .setSchema(schema) - .setTotalRows(data.y()) - .setPageNoSchema(data.x()) - .build(); + Span tableDataList = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + tableDataList = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.listTableData") + .setAllAttributes(tableId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope tableDataListScope = tableDataList != null ? tableDataList.makeCurrent() : null) { + return TableResult.newBuilder() + .setSchema(schema) + .setTotalRows(data.y()) + .setPageNoSchema(data.x()) + .build(); + } finally { + if (tableDataList != null) { + tableDataList.end(); + } + } } private static Tuple, Long> listTableData( @@ -1295,7 +1722,18 @@ public Job getJob(JobId jobId, JobOption... options) { jobId.getLocation() == null && getOptions().getLocation() != null ? getOptions().getLocation() : jobId.getLocation()); - try { + Span jobGet = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + jobGet = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.getJob") + .setAllAttributes(completeJobId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope jobGetScope = jobGet != null ? jobGet.makeCurrent() : null) { com.google.api.services.bigquery.model.Job answer = BigQueryRetryHelper.runWithRetries( new Callable() { @@ -1321,12 +1759,32 @@ public com.google.api.services.bigquery.model.Job call() throws IOException { return null; } throw BigQueryException.translateAndThrow(e); + } finally { + if (jobGet != null) { + jobGet.end(); + } } } @Override public Page listJobs(JobListOption... options) { - return listJobs(getOptions(), optionMap(options)); + Span jobsList = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + jobsList = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.listJobs") + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope jobsListScope = jobsList != null ? jobsList.makeCurrent() : null) { + return listJobs(getOptions(), optionMap(options)); + } finally { + if (jobsList != null) { + jobsList.end(); + } + } } private static Page listJobs( @@ -1377,7 +1835,17 @@ public boolean cancel(JobId jobId) { jobId.getLocation() == null && getOptions().getLocation() != null ? getOptions().getLocation() : jobId.getLocation()); - try { + Span jobCancel = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + jobCancel = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.cancelJob") + .setAllAttributes(completeJobId.getOtelAttributes()) + .startSpan(); + } + try (Scope jobCancelScope = jobCancel != null ? jobCancel.makeCurrent() : null) { return BigQueryRetryHelper.runWithRetries( new Callable() { @Override @@ -1395,6 +1863,10 @@ public Boolean call() throws IOException { return false; } throw BigQueryException.translateAndThrow(e); + } finally { + if (jobCancel != null) { + jobCancel.end(); + } } } @@ -1408,27 +1880,56 @@ public TableResult query(QueryJobConfiguration configuration, JobOption... optio .setJobCreationMode(getOptions().getDefaultJobCreationMode()) .build(); - // If all parameters passed in configuration are supported by the query() method on the backend, - // put on fast path - QueryRequestInfo requestInfo = - new QueryRequestInfo(configuration, getOptions().getUseInt64Timestamps()); - if (requestInfo.isFastQuerySupported(null)) { - String projectId = getOptions().getProjectId(); - QueryRequest content = requestInfo.toPb(); - if (getOptions().getLocation() != null) { - content.setLocation(getOptions().getLocation()); + Span querySpan = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + querySpan = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.query") + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope queryScope = querySpan != null ? querySpan.makeCurrent() : null) { + // If all parameters passed in configuration are supported by the query() method on the + // backend, + // put on fast path + QueryRequestInfo requestInfo = + new QueryRequestInfo(configuration, getOptions().getUseInt64Timestamps()); + if (requestInfo.isFastQuerySupported(null)) { + String projectId = getOptions().getProjectId(); + QueryRequest content = requestInfo.toPb(); + if (getOptions().getLocation() != null) { + content.setLocation(getOptions().getLocation()); + } + return queryRpc(projectId, content, options); + } + // Otherwise, fall back to the existing create query job logic + return create(JobInfo.of(configuration), options).getQueryResults(); + } finally { + if (querySpan != null) { + querySpan.end(); } - return queryRpc(projectId, content, options); } - // Otherwise, fall back to the existing create query job logic - return create(JobInfo.of(configuration), options).getQueryResults(); } private TableResult queryRpc( final String projectId, final QueryRequest content, JobOption... options) throws InterruptedException { com.google.api.services.bigquery.model.QueryResponse results; - try { + Span queryRpc = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + queryRpc = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.queryRpc") + .setAttribute("bq.query.project_id", projectId) + .setAllAttributes(otelAttributesFromQueryRequest(content)) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope queryRpcScope = queryRpc != null ? queryRpc.makeCurrent() : null) { results = BigQueryRetryHelper.runWithRetries( new Callable() { @@ -1444,6 +1945,10 @@ public com.google.api.services.bigquery.model.QueryResponse call() DEFAULT_RETRY_CONFIG); } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (queryRpc != null) { + queryRpc.end(); + } } if (results.getErrors() != null) { @@ -1513,37 +2018,74 @@ public TableResult query(QueryJobConfiguration configuration, JobId jobId, JobOp throws InterruptedException, JobException { Job.checkNotDryRun(configuration, "query"); - // If all parameters passed in configuration are supported by the query() method on the backend, - // put on fast path - QueryRequestInfo requestInfo = - new QueryRequestInfo(configuration, getOptions().getUseInt64Timestamps()); - if (requestInfo.isFastQuerySupported(jobId)) { - // Be careful when setting the projectID in JobId, if a projectID is specified in the JobId, - // the job created by the query method will use that project. This may cause the query to - // fail with "Access denied" if the project do not have enough permissions to run the job. - - String projectId = - jobId.getProject() != null ? jobId.getProject() : getOptions().getProjectId(); - QueryRequest content = requestInfo.toPb(); - // Be careful when setting the location, if a location is specified in the BigQueryOption or - // JobId the job created by the query method will be in that location, even if the table to be - // queried is in a different location. This may cause the query to fail with - // "BigQueryException: Not found" - if (jobId.getLocation() != null) { - content.setLocation(jobId.getLocation()); - } else if (getOptions().getLocation() != null) { - content.setLocation(getOptions().getLocation()); - } + Span querySpan = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + querySpan = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.query") + .setAllAttributes(jobId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope queryScope = querySpan != null ? querySpan.makeCurrent() : null) { + // If all parameters passed in configuration are supported by the query() method on the + // backend, + // put on fast path + QueryRequestInfo requestInfo = + new QueryRequestInfo(configuration, getOptions().getUseInt64Timestamps()); + if (requestInfo.isFastQuerySupported(jobId)) { + // Be careful when setting the projectID in JobId, if a projectID is specified in the JobId, + // the job created by the query method will use that project. This may cause the query to + // fail with "Access denied" if the project do not have enough permissions to run the job. + + String projectId = + jobId.getProject() != null ? jobId.getProject() : getOptions().getProjectId(); + QueryRequest content = requestInfo.toPb(); + // Be careful when setting the location, if a location is specified in the BigQueryOption or + // JobId the job created by the query method will be in that location, even if the table to + // be + // queried is in a different location. This may cause the query to fail with + // "BigQueryException: Not found" + if (jobId.getLocation() != null) { + content.setLocation(jobId.getLocation()); + } else if (getOptions().getLocation() != null) { + content.setLocation(getOptions().getLocation()); + } - return queryRpc(projectId, content, options); + return queryRpc(projectId, content, options); + } + return create(JobInfo.of(jobId, configuration), options).getQueryResults(); + } finally { + if (querySpan != null) { + querySpan.end(); + } } - return create(JobInfo.of(jobId, configuration), options).getQueryResults(); } @Override public QueryResponse getQueryResults(JobId jobId, QueryResultsOption... options) { Map optionsMap = optionMap(options); - return getQueryResults(jobId, getOptions(), optionsMap); + Span getQueryResults = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + getQueryResults = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.getQueryResults") + .setAllAttributes(jobId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope getQueryResultsScope = + getQueryResults != null ? getQueryResults.makeCurrent() : null) { + return getQueryResults(jobId, getOptions(), optionsMap); + } finally { + if (getQueryResults != null) { + getQueryResults.end(); + } + } } private static QueryResponse getQueryResults( @@ -1619,8 +2161,19 @@ public Policy getIamPolicy(TableId tableId, IAMOption... options) { ? getOptions().getProjectId() : tableId.getProject()); - try { - final Map optionsMap = optionMap(options); + final Map optionsMap = optionMap(options); + Span iamPolicyGet = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + iamPolicyGet = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.getIamPolicy") + .setAllAttributes(tableId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope iamPolicyGetScope = iamPolicyGet != null ? iamPolicyGet.makeCurrent() : null) { return convertFromApiPolicy( BigQueryRetryHelper.runWithRetries( new Callable() { @@ -1636,6 +2189,10 @@ public com.google.api.services.bigquery.model.Policy call() throws IOException { EMPTY_RETRY_CONFIG)); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (iamPolicyGet != null) { + iamPolicyGet.end(); + } } } @@ -1646,8 +2203,21 @@ public Policy setIamPolicy(TableId tableId, final Policy policy, IAMOption... op Strings.isNullOrEmpty(tableId.getProject()) ? getOptions().getProjectId() : tableId.getProject()); - try { - final Map optionsMap = optionMap(options); + + final Map optionsMap = optionMap(options); + Span iamPolicySet = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + iamPolicySet = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.setIamPolicy") + .setAllAttributes(tableId.getOtelAttributes()) + .setAllAttributes(otelAttributesFromPolicy(policy)) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope iamPolicySetScope = iamPolicySet != null ? iamPolicySet.makeCurrent() : null) { return convertFromApiPolicy( BigQueryRetryHelper.runWithRetries( new Callable() { @@ -1663,6 +2233,10 @@ public com.google.api.services.bigquery.model.Policy call() throws IOException { EMPTY_RETRY_CONFIG)); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (iamPolicySet != null) { + iamPolicySet.end(); + } } } @@ -1674,8 +2248,21 @@ public List testIamPermissions( Strings.isNullOrEmpty(tableId.getProject()) ? getOptions().getProjectId() : tableId.getProject()); - try { - final Map optionsMap = optionMap(options); + final Map optionsMap = optionMap(options); + Span testIamPermissions = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + testIamPermissions = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.BigQuery.testIamPermissions") + .setAllAttributes(tableId.getOtelAttributes()) + .setAttribute("bq.iam.permissions", permissions.toString()) + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + try (Scope testIamPermissionsScope = + testIamPermissions != null ? testIamPermissions.makeCurrent() : null) { com.google.api.services.bigquery.model.TestIamPermissionsResponse response = BigQueryRetryHelper.runWithRetries( new Callable() { @@ -1695,6 +2282,10 @@ public com.google.api.services.bigquery.model.TestIamPermissionsResponse call() : ImmutableList.copyOf(response.getPermissions()); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (testIamPermissions != null) { + testIamPermissions.end(); + } } } @@ -1708,6 +2299,39 @@ public com.google.api.services.bigquery.model.TestIamPermissionsResponse call() return optionMap; } + private static String getFieldAsString(Object field) { + return field == null ? "null" : field.toString(); + } + + private static Attributes otelAttributesFromOptions(Option... options) { + Attributes attributes = Attributes.builder().build(); + for (Option option : options) { + attributes = + attributes.toBuilder() + .put("bq.option." + option.getRpcOption().toString(), option.getValue().toString()) + .build(); + } + return attributes; + } + + private static Attributes otelAttributesFromQueryRequest(QueryRequest request) { + return Attributes.builder() + .put("bq.query.dry_run", getFieldAsString(request.getDryRun())) + .put("bq.query.job_creation_mode", getFieldAsString(request.getJobCreationMode())) + .put("bq.query.kind", getFieldAsString(request.getKind())) + .put("bq.query.location", getFieldAsString(request.getLocation())) + .put("bq.query.request_id", getFieldAsString(request.getRequestId())) + .put("bq.query.use_query_cache", getFieldAsString(request.getUseQueryCache())) + .build(); + } + + private static Attributes otelAttributesFromPolicy(Policy policy) { + return Attributes.builder() + .put("bq.policy.version", getFieldAsString(policy.getVersion())) + .put("bq.policy.bindings", getFieldAsString(policy.getBindings())) + .build(); + } + static BigQueryRetryConfig getBigQueryRetryConfig(Map options) { return (BigQueryRetryConfig) options.getOrDefault(BigQueryRpc.Option.BIGQUERY_RETRY_CONFIG, null); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java index a1dacc4cf..36eeb2086 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java @@ -16,6 +16,7 @@ package com.google.cloud.bigquery; +import com.google.api.core.BetaApi; import com.google.cloud.ServiceDefaults; import com.google.cloud.ServiceOptions; import com.google.cloud.ServiceRpc; @@ -25,6 +26,7 @@ import com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc; import com.google.cloud.http.HttpTransportOptions; import com.google.common.collect.ImmutableSet; +import io.opentelemetry.api.trace.Tracer; import java.util.Set; public class BigQueryOptions extends ServiceOptions { @@ -39,6 +41,8 @@ public class BigQueryOptions extends ServiceOptions { private boolean setThrowNotFound; private boolean useInt64Timestamps; private JobCreationMode defaultJobCreationMode = JobCreationMode.JOB_CREATION_MODE_UNSPECIFIED; + private boolean enableOpenTelemetryTracing; + private Tracer openTelemetryTracer; public static class DefaultBigQueryFactory implements BigQueryFactory { @@ -64,6 +68,8 @@ public static class Builder extends ServiceOptions.Builder { @@ -171,6 +201,26 @@ public JobCreationMode getDefaultJobCreationMode() { return defaultJobCreationMode; } + /** + * Returns whether this BigQuery instance has OpenTelemetry tracing enabled + * + * @return true if tracing is enabled, false if not + */ + @BetaApi("Span names and attributes are subject to change without notice") + public boolean isOpenTelemetryTracingEnabled() { + return enableOpenTelemetryTracing; + } + + /** + * Returns the OpenTelemetry tracer used by this BigQuery instance + * + * @return OpenTelemetry tracer object or {@code null} if not set + */ + @BetaApi("Span names and attributes are subject to change without notice") + public Tracer getOpenTelemetryTracer() { + return openTelemetryTracer; + } + @SuppressWarnings("unchecked") @Override public Builder toBuilder() { diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetId.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetId.java index 0e2ad02b2..c06d257d7 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetId.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetId.java @@ -20,6 +20,7 @@ import static com.google.common.base.Strings.isNullOrEmpty; import com.google.api.services.bigquery.model.DatasetReference; +import io.opentelemetry.api.common.Attributes; import java.io.Serializable; import java.util.Objects; @@ -84,4 +85,11 @@ DatasetReference toPb() { static DatasetId fromPb(DatasetReference datasetRef) { return new DatasetId(datasetRef.getProjectId(), datasetRef.getDatasetId()); } + + protected Attributes getOtelAttributes() { + return Attributes.builder() + .put("bq.dataset.project", this.getProject()) + .put("bq.dataset.id", this.getDataset()) + .build(); + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java index 90d6c2cd5..918761995 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java @@ -27,6 +27,7 @@ import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import io.opentelemetry.api.common.Attributes; import java.io.Serializable; import java.util.List; import java.util.Map; @@ -746,6 +747,18 @@ public static DatasetInfo of(String datasetId) { return newBuilder(datasetId).build(); } + private static String getFieldAsString(Object field) { + return field == null ? "null" : field.toString(); + } + + protected Attributes getOtelAttributes() { + return Attributes.builder() + .putAll(this.getDatasetId().getOtelAttributes()) + .put("bq.dataset.last_modified", getFieldAsString(this.getLastModified())) + .put("bq.dataset.location", getFieldAsString(this.getLocation())) + .build(); + } + static DatasetInfo fromPb(Dataset datasetPb) { return new BuilderImpl(datasetPb).build(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/InsertAllRequest.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/InsertAllRequest.java index 53952bc6c..ce9380098 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/InsertAllRequest.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/InsertAllRequest.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.opentelemetry.api.common.Attributes; import java.io.Serializable; import java.util.Collections; import java.util.HashMap; @@ -479,4 +480,15 @@ public boolean equals(Object obj) { && Objects.equals(skipInvalidRows, other.skipInvalidRows) && Objects.equals(templateSuffix, other.templateSuffix); } + + private static String getFieldAsString(Object field) { + return field == null ? "null" : field.toString(); + } + + public Attributes getOtelAttributes() { + return Attributes.builder() + .put("bq.insert_all.table", getFieldAsString(this.getTable().getTable())) + .put("bq.insert_all.template_suffix", getFieldAsString(this.getTemplateSuffix())) + .build(); + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobId.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobId.java index b966cebe1..4bfc2aa00 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobId.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobId.java @@ -21,6 +21,7 @@ import com.google.api.services.bigquery.model.JobReference; import com.google.auto.value.AutoValue; +import io.opentelemetry.api.common.Attributes; import java.io.Serializable; import java.util.UUID; import javax.annotation.Nullable; @@ -123,4 +124,16 @@ static JobId fromPb(JobReference jobRef) { .setLocation(jobRef.getLocation()) .build(); } + + private static String getFieldAsString(Object field) { + return field == null ? "null" : field.toString(); + } + + protected Attributes getOtelAttributes() { + return Attributes.builder() + .put("bq.job.id", getFieldAsString(this.getJob())) + .put("bq.job.location", getFieldAsString(this.getLocation())) + .put("bq.job.project", getFieldAsString(this.getProject())) + .build(); + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ModelId.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ModelId.java index 9356c6935..adb497ffa 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ModelId.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ModelId.java @@ -21,6 +21,7 @@ import com.google.api.services.bigquery.model.ModelReference; import com.google.common.base.Function; +import io.opentelemetry.api.common.Attributes; import java.io.Serializable; import java.util.Objects; @@ -105,4 +106,12 @@ ModelReference toPb() { static ModelId fromPb(ModelReference modelRef) { return new ModelId(modelRef.getProjectId(), modelRef.getDatasetId(), modelRef.getModelId()); } + + protected Attributes getOtelAttributes() { + return Attributes.builder() + .put("bq.model.project", this.getProject()) + .put("bq.model.dataset", this.getDataset()) + .put("bq.model.id", this.getModel()) + .build(); + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ModelInfo.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ModelInfo.java index 922ed441c..3039483b5 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ModelInfo.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ModelInfo.java @@ -26,6 +26,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import io.opentelemetry.api.common.Attributes; import java.io.Serializable; import java.util.Collections; import java.util.List; @@ -453,4 +454,19 @@ Model toPb() { static ModelInfo fromPb(Model modelPb) { return new BuilderImpl(modelPb).build(); } + + private static String getFieldAsString(Object field) { + return field == null ? "null" : field.toString(); + } + + protected Attributes getOtelAttributes() { + return Attributes.builder() + .putAll(this.getModelId().getOtelAttributes()) + .put("bq.model.type", getFieldAsString(this.getModelType())) + .put("bq.model.creation_time", getFieldAsString(this.getCreationTime())) + .put("bq.model.last_modified_time", getFieldAsString(this.getLastModifiedTime())) + .put("bq.model.expiration_time", getFieldAsString(this.getExpirationTime())) + .put("bq.model.location", getFieldAsString(this.getLocation())) + .build(); + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RoutineId.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RoutineId.java index c5c3dadb3..a632fc5a1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RoutineId.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RoutineId.java @@ -21,6 +21,7 @@ import com.google.api.services.bigquery.model.RoutineReference; import com.google.common.base.Function; +import io.opentelemetry.api.common.Attributes; import java.io.Serializable; import java.util.Objects; @@ -108,4 +109,12 @@ static RoutineId fromPb(RoutineReference routineRef) { return new RoutineId( routineRef.getProjectId(), routineRef.getDatasetId(), routineRef.getRoutineId()); } + + protected Attributes getOtelAttributes() { + return Attributes.builder() + .put("bq.routine.project", this.getProject()) + .put("bq.routine.dataset", this.getDataset()) + .put("bq.routine.id", this.getRoutine()) + .build(); + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RoutineInfo.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RoutineInfo.java index 646a2fdea..c13b90b41 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RoutineInfo.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RoutineInfo.java @@ -24,6 +24,7 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Strings; import com.google.common.collect.Lists; +import io.opentelemetry.api.common.Attributes; import java.io.Serializable; import java.util.Collections; import java.util.List; @@ -544,4 +545,17 @@ Routine toPb() { static RoutineInfo fromPb(Routine routinePb) { return new BuilderImpl(routinePb).build(); } + + private static String getFieldAsString(Object field) { + return field == null ? "null" : field.toString(); + } + + protected Attributes getOtelAttributes() { + return Attributes.builder() + .putAll(this.getRoutineId().getOtelAttributes()) + .put("bq.routine.type", getFieldAsString(this.getRoutineType())) + .put("bq.routine.creation_time", getFieldAsString(this.getCreationTime())) + .put("bq.routine.last_modified_time", getFieldAsString(this.getLastModifiedTime())) + .build(); + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableId.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableId.java index b74055d4f..fcc208453 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableId.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableId.java @@ -22,6 +22,7 @@ import com.google.api.services.bigquery.model.TableReference; import com.google.common.base.Function; +import io.opentelemetry.api.common.Attributes; import java.io.Serializable; import java.util.Objects; @@ -114,4 +115,12 @@ TableReference toPb() { static TableId fromPb(TableReference tableRef) { return new TableId(tableRef.getProjectId(), tableRef.getDatasetId(), tableRef.getTableId()); } + + protected Attributes getOtelAttributes() { + return Attributes.builder() + .put("bq.table.project", this.getProject()) + .put("bq.table.dataset", this.getDataset()) + .put("bq.table.id", this.getTable()) + .build(); + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableInfo.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableInfo.java index 6e050d064..b3236f4c3 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableInfo.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableInfo.java @@ -25,6 +25,7 @@ import com.google.api.services.bigquery.model.Table; import com.google.common.base.Function; import com.google.common.base.MoreObjects; +import io.opentelemetry.api.common.Attributes; import java.io.Serializable; import java.math.BigInteger; import java.util.Map; @@ -763,4 +764,19 @@ Table toPb() { static TableInfo fromPb(Table tablePb) { return new BuilderImpl(tablePb).build(); } + + private static String getFieldAsString(Object field) { + return field == null ? "null" : field.toString(); + } + + protected Attributes getOtelAttributes() { + return Attributes.builder() + .putAll(this.getTableId().getOtelAttributes()) + .put("bq.table.creation_time", getFieldAsString(this.getCreationTime())) + .put("bq.table.expiration_time", getFieldAsString(this.getExpirationTime())) + .put("bq.table.last_modified_time", getFieldAsString(this.getLastModifiedTime())) + .put("bq.table.num_bytes", getFieldAsString(this.getNumBytes())) + .put("bq.table.num_rows", getFieldAsString(this.getNumRows())) + .build(); + } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index bd17b14fc..eceadcbca 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -161,6 +161,17 @@ import com.google.common.io.BaseEncoding; import com.google.common.util.concurrent.ListenableFuture; import com.google.gson.JsonObject; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -220,6 +231,11 @@ public class ITBigQueryTest { private static final String STORAGE_BILLING_MODEL = "LOGICAL"; private static final Long MAX_TIME_TRAVEL_HOURS = 120L; private static final Long MAX_TIME_TRAVEL_HOURS_DEFAULT = 168L; + private static final Map, Object>> OTEL_ATTRIBUTES = + new HashMap, Object>>(); + private static final Map OTEL_PARENT_SPAN_IDS = new HashMap<>(); + private static final Map OTEL_SPAN_IDS_TO_NAMES = new HashMap<>(); + private static final String OTEL_PARENT_SPAN_ID = "0000000000000000"; private static final String CLOUD_SAMPLES_DATA = Optional.fromNullable(System.getenv("CLOUD_SAMPLES_DATA_BUCKET")).or("cloud-samples-data"); private static final Map LABELS = @@ -1017,6 +1033,32 @@ public class ITBigQueryTest { private static BigQuery bigquery; private static Storage storage; + private static OpenTelemetry otel; + + private static class TestSpanExporter implements io.opentelemetry.sdk.trace.export.SpanExporter { + @Override + public CompletableResultCode export(Collection collection) { + if (collection.isEmpty()) { + return CompletableResultCode.ofFailure(); + } + for (SpanData data : collection) { + OTEL_ATTRIBUTES.put(data.getName(), data.getAttributes().asMap()); + OTEL_PARENT_SPAN_IDS.put(data.getName(), data.getParentSpanId()); + OTEL_SPAN_IDS_TO_NAMES.put(data.getSpanId(), data.getName()); + } + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode flush() { + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } + } @Rule public Timeout globalTimeout = Timeout.seconds(300); @@ -1025,6 +1067,13 @@ public static void beforeClass() throws InterruptedException, IOException { RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); RemoteStorageHelper storageHelper = RemoteStorageHelper.create(); Map labels = ImmutableMap.of("test-job-name", "test-load-job"); + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(new TestSpanExporter())) + .setSampler(Sampler.alwaysOn()) + .build(); + otel = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal(); + bigquery = bigqueryHelper.getOptions().getService(); storage = storageHelper.getOptions().getService(); storage.create(BucketInfo.of(BUCKET)); @@ -7493,4 +7542,157 @@ public void testStatementType() throws InterruptedException { assertTrue(remoteTable.getDefinition() instanceof MaterializedViewDefinition); assertTrue(remoteTable.delete()); } + + @Test + public void testOpenTelemetryTracingDatasets() { + Tracer tracer = otel.getTracer("Test Tracer"); + BigQueryOptions otelOptions = + BigQueryOptions.newBuilder() + .setEnableOpenTelemetryTracing(true) + .setOpenTelemetryTracer(tracer) + .build(); + BigQuery bigquery = otelOptions.getService(); + + Span parentSpan = + tracer + .spanBuilder("Test Parent Span") + .setNoParent() + .setAttribute("test-attribute", "test-value") + .startSpan(); + String billingModelDataset = RemoteBigQueryHelper.generateDatasetName(); + + try (Scope parentScope = parentSpan.makeCurrent()) { + DatasetInfo info = + DatasetInfo.newBuilder(billingModelDataset) + .setDescription(DESCRIPTION) + .setMaxTimeTravelHours(72L) + .setLabels(LABELS) + .build(); + + Dataset dataset = bigquery.create(info); + assertNotNull(dataset); + dataset = bigquery.getDataset(dataset.getDatasetId().getDataset()); + assertNotNull(dataset); + + DatasetInfo updatedInfo = + DatasetInfo.newBuilder(billingModelDataset) + .setDescription("Updated Description") + .setMaxTimeTravelHours(96L) + .setLabels(LABELS) + .build(); + + dataset = bigquery.update(updatedInfo, DatasetOption.accessPolicyVersion(2)); + assertEquals(dataset.getDescription(), "Updated Description"); + assertTrue(bigquery.delete(dataset.getDatasetId())); + } finally { + parentSpan.end(); + Map, Object> createMap = + OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.createDataset"); + assertEquals(createMap.get(AttributeKey.stringKey("bq.dataset.location")), "null"); + + Map, Object> getMap = + OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.getDataset"); + assertEquals(getMap.get(AttributeKey.stringKey("bq.dataset.id")), billingModelDataset); + + Map, Object> updateMap = + OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.updateDataset"); + assertEquals(updateMap.get(AttributeKey.stringKey("bq.option.ACCESS_POLICY_VERSION")), "2"); + + Map, Object> deleteMap = + OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.deleteDataset"); + assertEquals(deleteMap.get(AttributeKey.stringKey("bq.dataset.id")), billingModelDataset); + + // All should be children spans of parentSpan + assertEquals( + OTEL_SPAN_IDS_TO_NAMES.get( + OTEL_PARENT_SPAN_IDS.get("com.google.cloud.bigquery.BigQuery.getDataset")), + "Test Parent Span"); + assertEquals( + OTEL_SPAN_IDS_TO_NAMES.get( + OTEL_PARENT_SPAN_IDS.get("com.google.cloud.bigquery.BigQuery.createDataset")), + "Test Parent Span"); + assertEquals( + OTEL_SPAN_IDS_TO_NAMES.get( + OTEL_PARENT_SPAN_IDS.get("com.google.cloud.bigquery.BigQuery.deleteDataset")), + "Test Parent Span"); + assertEquals(OTEL_PARENT_SPAN_IDS.get("Test Parent Span"), OTEL_PARENT_SPAN_ID); + RemoteBigQueryHelper.forceDelete(bigquery, billingModelDataset); + } + } + + @Test + public void testOpenTelemetryTracingTables() { + Tracer tracer = otel.getTracer("Test Tracer"); + BigQueryOptions otelOptions = + BigQueryOptions.newBuilder() + .setEnableOpenTelemetryTracing(true) + .setOpenTelemetryTracer(tracer) + .build(); + BigQuery bigquery = otelOptions.getService(); + + String tableName = "test_otel_table"; + StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); + TableInfo tableInfo = + TableInfo.newBuilder(TableId.of(DATASET, tableName), tableDefinition) + .setDescription("Some Description") + .build(); + Table createdTable = bigquery.create(tableInfo); + assertThat(createdTable.getDescription()).isEqualTo("Some Description"); + + assertEquals( + OTEL_PARENT_SPAN_IDS.get("com.google.cloud.bigquery.BigQuery.createTable"), + OTEL_PARENT_SPAN_ID); + assertEquals( + OTEL_ATTRIBUTES + .get("com.google.cloud.bigquery.BigQuery.createTable") + .get(AttributeKey.stringKey("bq.table.id")), + tableName); + assertEquals( + OTEL_ATTRIBUTES + .get("com.google.cloud.bigquery.BigQuery.createTable") + .get(AttributeKey.stringKey("bq.table.creation_time")), + "null"); + + Table updatedTable = + bigquery.update(createdTable.toBuilder().setDescription("Updated Description").build()); + assertThat(updatedTable.getDescription()).isEqualTo("Updated Description"); + + assertNotNull(OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.updateTable")); + assertEquals( + OTEL_PARENT_SPAN_IDS.get("com.google.cloud.bigquery.BigQuery.updateTable"), + OTEL_PARENT_SPAN_ID); + } + + @Test + public void testOpenTelemetryTracingQuery() throws InterruptedException { + Tracer tracer = otel.getTracer("Test Tracer"); + BigQueryOptions otelOptions = + BigQueryOptions.newBuilder() + .setEnableOpenTelemetryTracing(true) + .setOpenTelemetryTracer(tracer) + .build(); + BigQuery bigquery = otelOptions.getService(); + + // Stateless query + bigquery.getOptions().setDefaultJobCreationMode(JobCreationMode.JOB_CREATION_OPTIONAL); + TableResult tableResult = executeSimpleQuery(bigquery); + assertNotNull(tableResult.getQueryId()); + assertNull(tableResult.getJobId()); + + assertNotNull(OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.queryRpc")); + + // Query job + String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); + QueryJobConfiguration config = + QueryJobConfiguration.newBuilder(query).setDefaultDataset(DatasetId.of(DATASET)).build(); + Job job = bigquery.create(JobInfo.of(JobId.of(), config)); + + TableResult result = job.getQueryResults(); + assertNotNull(result.getJobId()); + assertEquals(QUERY_RESULT_SCHEMA, result.getSchema()); + + assertNotNull(OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.getQueryResults")); + assertNotNull(OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.listTableData")); + assertNotNull(OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.createJob")); + } } diff --git a/pom.xml b/pom.xml index e281937a3..b65439ecb 100644 --- a/pom.xml +++ b/pom.xml @@ -59,6 +59,15 @@ + + + io.opentelemetry + opentelemetry-bom + 1.48.0 + pom + import + + com.google.cloud google-cloud-shared-dependencies From b1962a7f0084ee4c3e248266b50406cf575cd657 Mon Sep 17 00:00:00 2001 From: Liam Huffman <44932470+whuffman36@users.noreply.github.com> Date: Mon, 9 Jun 2025 10:30:59 -0700 Subject: [PATCH 02/24] fix(bigquery): Add MY_VIEW_DATASET_NAME_TEST_ to resource clean up sample (#3838) --- .../src/main/java/com/example/bigquery/ResourceCleanUp.java | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/snippets/src/main/java/com/example/bigquery/ResourceCleanUp.java b/samples/snippets/src/main/java/com/example/bigquery/ResourceCleanUp.java index cc294f7a5..a530885cf 100644 --- a/samples/snippets/src/main/java/com/example/bigquery/ResourceCleanUp.java +++ b/samples/snippets/src/main/java/com/example/bigquery/ResourceCleanUp.java @@ -54,6 +54,7 @@ public static void main(String[] args) { || datasetName.contains("gcloud_test_") || datasetName.contains("SHARED_DATASET_TEST_") || datasetName.contains("WRITE_STREAM_TEST")) + || datasetName.contains("MY_VIEW_DATASET_NAME_TEST_") && dataset.getCreationTime() > sixHourAgo) { System.out.format("\tDeleting Dataset: %s\n", datasetName); bigquery.delete( From 69be5e7345fb8ca69d633d9dc99cf6c15fa5227b Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 12 Jun 2025 18:20:54 +0200 Subject: [PATCH 03/24] deps: update dependency com.google.api.grpc:proto-google-cloud-bigqueryconnection-v1 to v2.66.0 (#3835) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b65439ecb..a713e0f59 100644 --- a/pom.xml +++ b/pom.xml @@ -158,7 +158,7 @@ com.google.api.grpc proto-google-cloud-bigqueryconnection-v1 - 2.65.0 + 2.66.0 test From e9e5b690769491cd779ac275dfe6779d2af4e110 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 12 Jun 2025 18:31:33 +0200 Subject: [PATCH 04/24] chore(deps): update google cloud java monorepo updates (#3834) --- pom.xml | 4 ++-- samples/install-without-bom/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index a713e0f59..f02c1908a 100644 --- a/pom.xml +++ b/pom.xml @@ -88,7 +88,7 @@ com.google.cloud google-cloud-datacatalog-bom - 1.69.0 + 1.70.0 pom import @@ -152,7 +152,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.65.0 + 2.66.0 test diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 0719a5045..679ea72bc 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -69,7 +69,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.65.0 + 2.66.0 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index f81a2b0d9..4c62e8c10 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -67,7 +67,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.65.0 + 2.66.0 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 9ddab8dc9..3ec7e666e 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -85,7 +85,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.65.0 + 2.66.0 test From 08772af8d04a4200b1bb5539246208f16cea714b Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Thu, 12 Jun 2025 13:39:39 -0400 Subject: [PATCH 05/24] build: include arrow dependency suggested compiler flags (#3844) * build: include arrow dependency suggested compiler flags This follows https://arrow.apache.org/docs/java/install.html, where we add the indicated `--add-opens` flag. This is meant to follow from https://github.com/googleapis/java-bigquery/pull/3811#issuecomment-2941443802 * fix: only activate on JDK 9+ --- google-cloud-bigquery/pom.xml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 9599d760c..1bdc43fae 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -275,6 +275,27 @@ + + arrow-config + + [9,) + + + + + org.apache.maven.plugins + maven-compiler-plugin + + UTF-8 + true + + -J--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + + + + java17 From 4c203c1e7f063fe9a8ea77974899f3a79a2d0fd7 Mon Sep 17 00:00:00 2001 From: Phong Chuong <147636638+PhongChuong@users.noreply.github.com> Date: Thu, 12 Jun 2025 14:40:32 -0400 Subject: [PATCH 06/24] chore(deps): update dependency com.google.cloud:libraries-bom to v26.62.0 (#3848) --- README.md | 2 +- samples/snippets/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 42b0b9f7e..d8f6c46d8 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ See https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google com.google.cloud libraries-bom - 26.43.0 + 26.62.0 pom import diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 3ec7e666e..e233d277e 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -47,7 +47,7 @@ com.google.cloud libraries-bom - 26.43.0 + 26.62.0 pom import From d0cea7d2cb958adcc85f5861cbca6f9c3d1c8183 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 12 Jun 2025 20:40:54 +0200 Subject: [PATCH 07/24] chore(deps): update dependency com.google.cloud:google-cloud-bigtable to v2.60.0 (#3836) --- samples/install-without-bom/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 679ea72bc..b7dd48f28 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -63,7 +63,7 @@ com.google.cloud google-cloud-bigtable - 2.59.0 + 2.60.0 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 4c62e8c10..d176a10cc 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -61,7 +61,7 @@ com.google.cloud google-cloud-bigtable - 2.59.0 + 2.60.0 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index e233d277e..478359e9a 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -79,7 +79,7 @@ com.google.cloud google-cloud-bigtable - 2.59.0 + 2.60.0 test From 51321c22778fd41134cc0cdfc70bdc47f05883f1 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 12 Jun 2025 20:43:15 +0200 Subject: [PATCH 08/24] deps: update dependency io.opentelemetry:opentelemetry-bom to v1.51.0 (#3840) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f02c1908a..97712df9f 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ io.opentelemetry opentelemetry-bom - 1.48.0 + 1.51.0 pom import From 057f7626d8b0d178fd04d40e7eb31839e891d42a Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Fri, 13 Jun 2025 13:54:37 -0400 Subject: [PATCH 09/24] chore(main): release 2.51.1-SNAPSHOT (#3839) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- benchmark/pom.xml | 2 +- google-cloud-bigquery-bom/pom.xml | 4 ++-- google-cloud-bigquery/pom.xml | 4 ++-- pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 6285defb6..cd645c356 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -6,7 +6,7 @@ google-cloud-bigquery-parent com.google.cloud - 2.51.0 + 2.51.1-SNAPSHOT diff --git a/google-cloud-bigquery-bom/pom.xml b/google-cloud-bigquery-bom/pom.xml index 66387cf2e..3d31594b7 100644 --- a/google-cloud-bigquery-bom/pom.xml +++ b/google-cloud-bigquery-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery-bom - 2.51.0 + 2.51.1-SNAPSHOT pom com.google.cloud @@ -54,7 +54,7 @@ com.google.cloud google-cloud-bigquery - 2.51.0 + 2.51.1-SNAPSHOT diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 1bdc43fae..98d88fdbb 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery - 2.51.0 + 2.51.1-SNAPSHOT jar BigQuery https://github.com/googleapis/java-bigquery @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquery-parent - 2.51.0 + 2.51.1-SNAPSHOT google-cloud-bigquery diff --git a/pom.xml b/pom.xml index 97712df9f..fd844f088 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquery-parent pom - 2.51.0 + 2.51.1-SNAPSHOT BigQuery Parent https://github.com/googleapis/java-bigquery @@ -102,7 +102,7 @@ com.google.cloud google-cloud-bigquery - 2.51.0 + 2.51.1-SNAPSHOT diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index d176a10cc..5bfcf1bf0 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -44,7 +44,7 @@ com.google.cloud google-cloud-bigquery - 2.51.0 + 2.51.1-SNAPSHOT diff --git a/versions.txt b/versions.txt index b17f78a7b..7622bb1dc 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-bigquery:2.51.0:2.51.0 \ No newline at end of file +google-cloud-bigquery:2.51.0:2.51.1-SNAPSHOT \ No newline at end of file From 414f61d7efcfa568c1446bd41945d7a8e2450649 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 16 Jun 2025 19:33:29 +0200 Subject: [PATCH 10/24] deps: update ossf/scorecard-action action to v2.4.2 (#3810) --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 87e8e13c4..1524dc8d2 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -37,7 +37,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2 with: results_file: results.sarif results_format: sarif From fde9d671933d01cfd43493fcf917382361d8f384 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 16 Jun 2025 19:35:56 +0200 Subject: [PATCH 11/24] chore(deps): update dependency com.google.cloud:google-cloud-bigquery to v2.51.0 (#3797) --- samples/install-without-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index b7dd48f28..dee35288d 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -45,7 +45,7 @@ com.google.cloud google-cloud-bigquery - 2.50.0 + 2.51.0 From 286728fb1c6bb5d1a8afe796a1d3514704b8c6d1 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 16 Jun 2025 19:38:43 +0200 Subject: [PATCH 12/24] test(deps): update dependency com.google.cloud:google-cloud-storage to v2.53.0 (#3798) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fd844f088..16e2df2e4 100644 --- a/pom.xml +++ b/pom.xml @@ -146,7 +146,7 @@ com.google.cloud google-cloud-storage - 2.52.2 + 2.53.0 test From ab8499ae927101d7bf9c67130a790a31aa00ac25 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 16 Jun 2025 19:57:03 +0200 Subject: [PATCH 13/24] build(deps): update dependency org.codehaus.mojo:build-helper-maven-plugin to v3.6.1 (#3830) --- google-cloud-bigquery/pom.xml | 2 +- samples/install-without-bom/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 98d88fdbb..4162718fb 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -251,7 +251,7 @@ org.codehaus.mojo build-helper-maven-plugin - 3.6.0 + 3.6.1 add-source diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index dee35288d..ec9180d6a 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -92,7 +92,7 @@ org.codehaus.mojo build-helper-maven-plugin - 3.6.0 + 3.6.1 add-snippets-source diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 5bfcf1bf0..b8ba50d01 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -90,7 +90,7 @@ org.codehaus.mojo build-helper-maven-plugin - 3.6.0 + 3.6.1 add-snippets-source From 6f9f77d47596b00b7317c8a0d4a10c3d849ad57b Mon Sep 17 00:00:00 2001 From: Phong Chuong <147636638+PhongChuong@users.noreply.github.com> Date: Tue, 17 Jun 2025 11:06:50 -0400 Subject: [PATCH 14/24] deps: remove version declaration of open-telemetry-bom (#3855) --- pom.xml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pom.xml b/pom.xml index 16e2df2e4..7e07ef3ef 100644 --- a/pom.xml +++ b/pom.xml @@ -59,15 +59,6 @@ - - - io.opentelemetry - opentelemetry-bom - 1.51.0 - pom - import - - com.google.cloud google-cloud-shared-dependencies From eabf2a276231faa0ca413a60d33943f581b30aaa Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 17 Jun 2025 18:25:59 +0200 Subject: [PATCH 15/24] chore(deps): update dependency com.google.cloud:google-cloud-bigquerystorage-bom to v3.15.2 (#3852) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7e07ef3ef..ebdc3d45b 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ com.google.cloud google-cloud-bigquerystorage-bom - 3.15.0 + 3.15.2 pom import From cf864df739bbb820e99999b7c1592a3635fea4ec Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 18 Jun 2025 17:49:02 +0200 Subject: [PATCH 16/24] deps: update dependency com.google.cloud:sdk-platform-java-config to v3.49.2 (#3853) --- .github/workflows/unmanaged_dependency_check.yaml | 2 +- .kokoro/continuous/graalvm-native-a.cfg | 2 +- .kokoro/continuous/graalvm-native-b.cfg | 2 +- .kokoro/continuous/graalvm-native-c.cfg | 2 +- .kokoro/presubmit/graalvm-native-a.cfg | 2 +- .kokoro/presubmit/graalvm-native-b.cfg | 2 +- .kokoro/presubmit/graalvm-native-c.cfg | 2 +- google-cloud-bigquery-bom/pom.xml | 2 +- pom.xml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unmanaged_dependency_check.yaml b/.github/workflows/unmanaged_dependency_check.yaml index 3f2945215..8449bf074 100644 --- a/.github/workflows/unmanaged_dependency_check.yaml +++ b/.github/workflows/unmanaged_dependency_check.yaml @@ -17,7 +17,7 @@ jobs: # repository .kokoro/build.sh - name: Unmanaged dependency check - uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.49.0 + uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.49.2 with: # java-bigquery does not produce a BOM. Fortunately the root pom.xml # defines google-cloud-bigquery in dependencyManagement section. So diff --git a/.kokoro/continuous/graalvm-native-a.cfg b/.kokoro/continuous/graalvm-native-a.cfg index 54d873dc1..b6a5a9d3e 100644 --- a/.kokoro/continuous/graalvm-native-a.cfg +++ b/.kokoro/continuous/graalvm-native-a.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.49.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.49.2" } env_vars: { diff --git a/.kokoro/continuous/graalvm-native-b.cfg b/.kokoro/continuous/graalvm-native-b.cfg index b7bd7ed2f..d0d6dea71 100644 --- a/.kokoro/continuous/graalvm-native-b.cfg +++ b/.kokoro/continuous/graalvm-native-b.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.49.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.49.2" } env_vars: { diff --git a/.kokoro/continuous/graalvm-native-c.cfg b/.kokoro/continuous/graalvm-native-c.cfg index 7090b33d1..a4a1fe79e 100644 --- a/.kokoro/continuous/graalvm-native-c.cfg +++ b/.kokoro/continuous/graalvm-native-c.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_c:3.49.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_c:3.49.2" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native-a.cfg b/.kokoro/presubmit/graalvm-native-a.cfg index 54d873dc1..b6a5a9d3e 100644 --- a/.kokoro/presubmit/graalvm-native-a.cfg +++ b/.kokoro/presubmit/graalvm-native-a.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.49.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.49.2" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native-b.cfg b/.kokoro/presubmit/graalvm-native-b.cfg index b7bd7ed2f..d0d6dea71 100644 --- a/.kokoro/presubmit/graalvm-native-b.cfg +++ b/.kokoro/presubmit/graalvm-native-b.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.49.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.49.2" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native-c.cfg b/.kokoro/presubmit/graalvm-native-c.cfg index 7090b33d1..a4a1fe79e 100644 --- a/.kokoro/presubmit/graalvm-native-c.cfg +++ b/.kokoro/presubmit/graalvm-native-c.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_c:3.49.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_c:3.49.2" } env_vars: { diff --git a/google-cloud-bigquery-bom/pom.xml b/google-cloud-bigquery-bom/pom.xml index 3d31594b7..9c1967a6d 100644 --- a/google-cloud-bigquery-bom/pom.xml +++ b/google-cloud-bigquery-bom/pom.xml @@ -8,7 +8,7 @@ com.google.cloud sdk-platform-java-config - 3.49.0 + 3.49.2 diff --git a/pom.xml b/pom.xml index ebdc3d45b..362c1f144 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ com.google.cloud sdk-platform-java-config - 3.49.0 + 3.49.2 From af9084141d615179e0f2952ccb138b9d7c90ed17 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Fri, 20 Jun 2025 16:50:58 +0200 Subject: [PATCH 17/24] test(deps): update dependency com.google.cloud:google-cloud-storage to v2.53.1 (#3856) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 362c1f144..2cdc232f0 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,7 @@ com.google.cloud google-cloud-storage - 2.53.0 + 2.53.1 test From 4b28c479c1bc22326c8d2501354fb86ec2ce1744 Mon Sep 17 00:00:00 2001 From: Liam Huffman <44932470+whuffman36@users.noreply.github.com> Date: Mon, 23 Jun 2025 14:50:27 -0700 Subject: [PATCH 18/24] feat(bigquery): Integrate Otel into retries, jobs, and more (#3842) * feat(bigquery): Integrate Otel into retries, jobs, and tableDataWriteChannel * Add justification for changing func signature * fix ignored-differences syntax * fix ignored-differences syntax...again * update IT tests * Remove config and algorithm from retry span * remove otel support from connection interface --- .../clirr-ignored-differences.xml | 6 + .../google/cloud/bigquery/BigQueryImpl.java | 124 +++++-- .../cloud/bigquery/BigQueryRetryHelper.java | 20 +- .../google/cloud/bigquery/ConnectionImpl.java | 24 +- .../java/com/google/cloud/bigquery/Job.java | 335 +++++++++++++----- .../cloud/bigquery/TableDataWriteChannel.java | 45 ++- .../cloud/bigquery/it/ITBigQueryTest.java | 25 ++ 7 files changed, 446 insertions(+), 133 deletions(-) diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index 37e349413..e048f4ce8 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -2,6 +2,12 @@ + + 7004 + com/google/cloud/bigquery/BigQueryRetryHelper + java.lang.Object runWithRetries(java.util.concurrent.Callable, com.google.api.gax.retrying.RetrySettings, com.google.api.gax.retrying.ResultRetryAlgorithm, com.google.api.core.ApiClock, com.google.cloud.bigquery.BigQueryRetryConfig) + A Tracer object is needed to use Otel and runWithRetries is only called in a few files, so it should be fine to update the signature + 7004 com/google/cloud/bigquery/spi/v2/BigQueryRpc diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index 2efe8f92a..a32a600c3 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -295,7 +295,9 @@ public com.google.api.services.bigquery.model.Dataset call() throws IOException getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG)); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer())); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } finally { @@ -340,7 +342,9 @@ public com.google.api.services.bigquery.model.Table call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG)); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer())); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } finally { @@ -394,7 +398,9 @@ public com.google.api.services.bigquery.model.Routine call() throws IOException getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG)); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer())); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } finally { @@ -488,7 +494,9 @@ public com.google.api.services.bigquery.model.Job call() throws IOException { getOptions().getClock(), getBigQueryRetryConfig(optionsMap) != null ? getBigQueryRetryConfig(optionsMap) - : DEFAULT_RETRY_CONFIG)); + : DEFAULT_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer())); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } @@ -580,7 +588,9 @@ public com.google.api.services.bigquery.model.Dataset call() throws IOException getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); return Dataset.fromPb(this, answer); } catch (BigQueryRetryHelperException e) { if (isRetryErrorCodeHttpNotFound(e)) { @@ -644,7 +654,9 @@ private static Page listDatasets( serviceOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, serviceOptions.getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + serviceOptions.isOpenTelemetryTracingEnabled(), + serviceOptions.getOpenTelemetryTracer()); String cursor = result.x(); return new PageImpl<>( new DatasetPageFetcher(projectId, serviceOptions, cursor, optionsMap), @@ -694,7 +706,9 @@ public Boolean call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); } catch (BigQueryRetryHelperException e) { if (isRetryErrorCodeHttpNotFound(e)) { return false; @@ -743,7 +757,9 @@ public Boolean call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); } catch (BigQueryRetryHelperException e) { if (isRetryErrorCodeHttpNotFound(e)) { return false; @@ -787,7 +803,9 @@ public Boolean call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); } catch (BigQueryRetryHelperException e) { if (isRetryErrorCodeHttpNotFound(e)) { return false; @@ -831,7 +849,9 @@ public Boolean call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); } catch (BigQueryRetryHelperException e) { if (isRetryErrorCodeHttpNotFound(e)) { return false; @@ -873,7 +893,9 @@ public Boolean call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } finally { @@ -912,7 +934,9 @@ public com.google.api.services.bigquery.model.Dataset call() throws IOException getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG)); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer())); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } finally { @@ -957,7 +981,9 @@ public com.google.api.services.bigquery.model.Table call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG)); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer())); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } finally { @@ -1001,7 +1027,9 @@ public com.google.api.services.bigquery.model.Model call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG)); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer())); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } finally { @@ -1045,7 +1073,9 @@ public com.google.api.services.bigquery.model.Routine call() throws IOException getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG)); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer())); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } finally { @@ -1097,7 +1127,9 @@ public com.google.api.services.bigquery.model.Table call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); return Table.fromPb(this, answer); } catch (BigQueryRetryHelperException e) { if (isRetryErrorCodeHttpNotFound(e)) { @@ -1154,7 +1186,9 @@ public com.google.api.services.bigquery.model.Model call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); return Model.fromPb(this, answer); } catch (BigQueryRetryHelperException e) { if (isRetryErrorCodeHttpNotFound(e)) { @@ -1211,7 +1245,9 @@ public com.google.api.services.bigquery.model.Routine call() throws IOException getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); return Routine.fromPb(this, answer); } catch (BigQueryRetryHelperException e) { if (isRetryErrorCodeHttpNotFound(e)) { @@ -1427,7 +1463,9 @@ public Tuple> cal serviceOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, serviceOptions.getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + serviceOptions.isOpenTelemetryTracingEnabled(), + serviceOptions.getOpenTelemetryTracer()); String cursor = result.x(); Iterable
tables = Iterables.transform( @@ -1466,7 +1504,9 @@ public Tuple> cal serviceOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, serviceOptions.getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + serviceOptions.isOpenTelemetryTracingEnabled(), + serviceOptions.getOpenTelemetryTracer()); String cursor = result.x(); Iterable models = Iterables.transform( @@ -1505,7 +1545,9 @@ private static Page listRoutines( serviceOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, serviceOptions.getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + serviceOptions.isOpenTelemetryTracingEnabled(), + serviceOptions.getOpenTelemetryTracer()); String cursor = result.x(); Iterable routines = Iterables.transform( @@ -1585,7 +1627,9 @@ public TableDataInsertAllResponse call() throws Exception { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } @@ -1677,7 +1721,9 @@ public TableDataList call() throws IOException { serviceOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, serviceOptions.getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + serviceOptions.isOpenTelemetryTracingEnabled(), + serviceOptions.getOpenTelemetryTracer()); String cursor = result.getPageToken(); Map pageOptionMap = Strings.isNullOrEmpty(cursor) ? optionsMap : optionMap(TableDataListOption.startIndex(0)); @@ -1749,7 +1795,9 @@ public com.google.api.services.bigquery.model.Job call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); return Job.fromPb(this, answer); } catch (BigQueryRetryHelperException e) { if (isRetryErrorCodeHttpNotFound(e)) { @@ -1804,7 +1852,9 @@ public Tuple> call( serviceOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, serviceOptions.getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + serviceOptions.isOpenTelemetryTracingEnabled(), + serviceOptions.getOpenTelemetryTracer()); String cursor = result.x(); Iterable jobs = Iterables.transform( @@ -1857,7 +1907,9 @@ public Boolean call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); } catch (BigQueryRetryHelperException e) { if (isRetryErrorCodeHttpNotFound(e)) { return false; @@ -1942,7 +1994,9 @@ public com.google.api.services.bigquery.model.QueryResponse call() getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - DEFAULT_RETRY_CONFIG); + DEFAULT_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } finally { @@ -2117,7 +2171,9 @@ public GetQueryResultsResponse call() throws IOException { serviceOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, serviceOptions.getClock(), - DEFAULT_RETRY_CONFIG); + DEFAULT_RETRY_CONFIG, + serviceOptions.isOpenTelemetryTracingEnabled(), + serviceOptions.getOpenTelemetryTracer()); TableSchema schemaPb = results.getSchema(); @@ -2186,7 +2242,9 @@ public com.google.api.services.bigquery.model.Policy call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG)); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer())); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } finally { @@ -2230,7 +2288,9 @@ public com.google.api.services.bigquery.model.Policy call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG)); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer())); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } finally { @@ -2276,7 +2336,9 @@ public com.google.api.services.bigquery.model.TestIamPermissionsResponse call() getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); return response.getPermissions() == null ? ImmutableList.of() : ImmutableList.copyOf(response.getPermissions()); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java index d315241a3..9c7083046 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java @@ -25,6 +25,9 @@ import com.google.api.gax.retrying.RetryingFuture; import com.google.api.gax.retrying.TimedRetryAlgorithm; import com.google.cloud.RetryHelper; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; import java.io.IOException; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -40,9 +43,18 @@ public static V runWithRetries( RetrySettings retrySettings, ResultRetryAlgorithm resultRetryAlgorithm, ApiClock clock, - BigQueryRetryConfig bigQueryRetryConfig) + BigQueryRetryConfig bigQueryRetryConfig, + boolean isOpenTelemetryEnabled, + Tracer openTelemetryTracer) throws RetryHelperException { - try { + Span runWithRetries = null; + if (isOpenTelemetryEnabled && openTelemetryTracer != null) { + runWithRetries = + openTelemetryTracer + .spanBuilder("com.google.cloud.bigquery.BigQueryRetryHelper.runWithRetries") + .startSpan(); + } + try (Scope runWithRetriesScope = runWithRetries != null ? runWithRetries.makeCurrent() : null) { // Suppressing should be ok as a workaraund. Current and only ResultRetryAlgorithm // implementation does not use response at all, so ignoring its type is ok. @SuppressWarnings("unchecked") @@ -59,6 +71,10 @@ public static V runWithRetries( throw new BigQueryRetryHelperException(new BigQueryException((IOException) e.getCause())); } throw new BigQueryRetryHelperException(e.getCause()); + } finally { + if (runWithRetries != null) { + runWithRetries.end(); + } } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index eb3a32db7..b12ad8556 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -476,7 +476,9 @@ private BigQueryResult queryRpc( bigQueryOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, bigQueryOptions.getClock(), - retryConfig); + retryConfig, + false, + null); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } @@ -932,7 +934,9 @@ private Job getQueryJobRpc(JobId jobId) { bigQueryOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, bigQueryOptions.getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + false, + null); } catch (BigQueryRetryHelperException e) { if (e.getCause() instanceof BigQueryException) { if (((BigQueryException) e.getCause()).getCode() == HTTP_NOT_FOUND) { @@ -977,7 +981,9 @@ TableDataList tableDataListRpc(TableId destinationTable, String pageToken) { bigQueryOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, bigQueryOptions.getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + false, + null); return results; } catch (BigQueryRetryHelperException e) { @@ -1208,7 +1214,9 @@ GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId) { bigQueryOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, bigQueryOptions.getClock(), - retryConfig); + retryConfig, + false, + null); if (results.getErrors() != null) { List bigQueryErrors = @@ -1471,7 +1479,9 @@ com.google.api.services.bigquery.model.Job createQueryJob( bigQueryOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, bigQueryOptions.getClock(), - retryConfig); + retryConfig, + false, + null); } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { logger.log(Level.WARNING, "\n Error occurred while calling createJobForQuery", e); throw BigQueryException.translateAndThrow(e); @@ -1514,7 +1524,9 @@ com.google.api.services.bigquery.model.Job createDryRunJob(String sql) { bigQueryOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, bigQueryOptions.getClock(), - retryConfig); + retryConfig, + false, + null); } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Job.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Job.java index 00edefb9f..88950b9fb 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Job.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Job.java @@ -29,6 +29,9 @@ import com.google.cloud.bigquery.BigQuery.TableDataListOption; import com.google.cloud.bigquery.JobConfiguration.Type; import com.google.common.collect.ImmutableList; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Scope; import java.io.IOException; import java.io.ObjectInputStream; import java.time.Duration; @@ -172,7 +175,21 @@ public Job build() { */ public boolean exists() { checkNotDryRun("exists"); - return bigquery.getJob(getJobId(), JobOption.fields()) != null; + Span exists = null; + if (options.isOpenTelemetryTracingEnabled() && options.getOpenTelemetryTracer() != null) { + exists = + options + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.Job.exists") + .startSpan(); + } + try (Scope existsScope = exists != null ? exists.makeCurrent() : null) { + return bigquery.getJob(getJobId(), JobOption.fields()) != null; + } finally { + if (exists != null) { + exists.end(); + } + } } /** @@ -193,8 +210,22 @@ public boolean exists() { */ public boolean isDone() { checkNotDryRun("isDone"); - Job job = bigquery.getJob(getJobId(), JobOption.fields(BigQuery.JobField.STATUS)); - return job == null || JobStatus.State.DONE.equals(job.getStatus().getState()); + Span isDone = null; + if (options.isOpenTelemetryTracingEnabled() && options.getOpenTelemetryTracer() != null) { + isDone = + options + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.Job.isDone") + .startSpan(); + } + try (Scope isDoneScope = isDone != null ? isDone.makeCurrent() : null) { + Job job = bigquery.getJob(getJobId(), JobOption.fields(BigQuery.JobField.STATUS)); + return job == null || JobStatus.State.DONE.equals(job.getStatus().getState()); + } finally { + if (isDone != null) { + isDone.end(); + } + } } /** See {@link #waitFor(BigQueryRetryConfig, RetryOption...)} */ @@ -275,19 +306,34 @@ public Job waitFor(BigQueryRetryConfig bigQueryRetryConfig, RetryOption... waitO private Job waitForInternal(BigQueryRetryConfig bigQueryRetryConfig, RetryOption... waitOptions) throws InterruptedException { checkNotDryRun("waitFor"); - Object completedJobResponse; - if (getConfiguration().getType() == Type.QUERY) { - completedJobResponse = - waitForQueryResults( - RetryOption.mergeToSettings(DEFAULT_JOB_WAIT_SETTINGS, waitOptions), - bigQueryRetryConfig, - DEFAULT_QUERY_WAIT_OPTIONS); - } else { - completedJobResponse = - waitForJob(RetryOption.mergeToSettings(DEFAULT_QUERY_JOB_WAIT_SETTINGS, waitOptions)); + Span waitFor = null; + if (this.options.isOpenTelemetryTracingEnabled() + && this.options.getOpenTelemetryTracer() != null) { + waitFor = + this.options + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.Job.waitFor") + .startSpan(); } + try (Scope waitForScope = waitFor != null ? waitFor.makeCurrent() : null) { + Object completedJobResponse; + if (getConfiguration().getType() == Type.QUERY) { + completedJobResponse = + waitForQueryResults( + RetryOption.mergeToSettings(DEFAULT_JOB_WAIT_SETTINGS, waitOptions), + bigQueryRetryConfig, + DEFAULT_QUERY_WAIT_OPTIONS); + } else { + completedJobResponse = + waitForJob(RetryOption.mergeToSettings(DEFAULT_QUERY_JOB_WAIT_SETTINGS, waitOptions)); + } - return completedJobResponse == null ? null : reload(); + return completedJobResponse == null ? null : reload(); + } finally { + if (waitFor != null) { + waitFor.end(); + } + } } /** @@ -304,72 +350,91 @@ private Job waitForInternal(BigQueryRetryConfig bigQueryRetryConfig, RetryOption public TableResult getQueryResults(QueryResultsOption... options) throws InterruptedException, JobException { checkNotDryRun("getQueryResults"); - if (getConfiguration().getType() != Type.QUERY) { - throw new UnsupportedOperationException( - "Getting query results is supported only for " + Type.QUERY + " jobs"); - } - - List waitOptions = - new ArrayList<>(Arrays.asList(DEFAULT_QUERY_WAIT_OPTIONS)); - List listOptions = new ArrayList<>(); - for (QueryResultsOption option : options) { - switch (option.getRpcOption()) { - case MAX_RESULTS: - listOptions.add(TableDataListOption.pageSize((Long) option.getValue())); - break; - case PAGE_TOKEN: - listOptions.add(TableDataListOption.pageToken((String) option.getValue())); - break; - case START_INDEX: - listOptions.add(TableDataListOption.startIndex((Long) option.getValue())); - break; - case TIMEOUT: - waitOptions.add(QueryResultsOption.maxWaitTime((Long) option.getValue())); - break; - } + + Span getQueryResults = null; + if (this.options.isOpenTelemetryTracingEnabled() + && this.options.getOpenTelemetryTracer() != null) { + getQueryResults = + this.options + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.Job.getQueryResults") + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); } + try (Scope getQueryResultsScope = + getQueryResults != null ? getQueryResults.makeCurrent() : null) { - QueryResponse response = - waitForQueryResults( - DEFAULT_JOB_WAIT_SETTINGS, - DEFAULT_RETRY_CONFIG, - waitOptions.toArray(new QueryResultsOption[0])); - - // Get the job resource to determine if it has errored. - Job job = this; - if (job.getStatus() == null || !JobStatus.State.DONE.equals(job.getStatus().getState())) { - job = reload(); - } - if (job.getStatus() != null && job.getStatus().getError() != null) { - throw new BigQueryException( - job.getStatus().getExecutionErrors() == null - ? ImmutableList.of(job.getStatus().getError()) - : ImmutableList.copyOf(job.getStatus().getExecutionErrors())); - } - - // If there are no rows in the result, this may have been a DDL query. - // Listing table data might fail, such as with CREATE VIEW queries. - // Avoid a tabledata.list API request by returning an empty TableResult. - if (response.getTotalRows() == 0) { - TableResult emptyTableResult = - TableResult.newBuilder() - .setSchema(response.getSchema()) - .setJobId(job.getJobId()) - .setTotalRows(0L) - .setPageNoSchema(new PageImpl(null, "", null)) - .build(); - return emptyTableResult; - } - - TableId table = - ((QueryJobConfiguration) getConfiguration()).getDestinationTable() == null - ? ((QueryJobConfiguration) job.getConfiguration()).getDestinationTable() - : ((QueryJobConfiguration) getConfiguration()).getDestinationTable(); - TableResult tableResult = - bigquery.listTableData( - table, response.getSchema(), listOptions.toArray(new TableDataListOption[0])); - TableResult tableResultWithJobId = tableResult.toBuilder().setJobId(job.getJobId()).build(); - return tableResultWithJobId; + if (getConfiguration().getType() != Type.QUERY) { + throw new UnsupportedOperationException( + "Getting query results is supported only for " + Type.QUERY + " jobs"); + } + + List waitOptions = + new ArrayList<>(Arrays.asList(DEFAULT_QUERY_WAIT_OPTIONS)); + List listOptions = new ArrayList<>(); + for (QueryResultsOption option : options) { + switch (option.getRpcOption()) { + case MAX_RESULTS: + listOptions.add(TableDataListOption.pageSize((Long) option.getValue())); + break; + case PAGE_TOKEN: + listOptions.add(TableDataListOption.pageToken((String) option.getValue())); + break; + case START_INDEX: + listOptions.add(TableDataListOption.startIndex((Long) option.getValue())); + break; + case TIMEOUT: + waitOptions.add(QueryResultsOption.maxWaitTime((Long) option.getValue())); + break; + } + } + + QueryResponse response = + waitForQueryResults( + DEFAULT_JOB_WAIT_SETTINGS, + DEFAULT_RETRY_CONFIG, + waitOptions.toArray(new QueryResultsOption[0])); + + // Get the job resource to determine if it has errored. + Job job = this; + if (job.getStatus() == null || !JobStatus.State.DONE.equals(job.getStatus().getState())) { + job = reload(); + } + if (job.getStatus() != null && job.getStatus().getError() != null) { + throw new BigQueryException( + job.getStatus().getExecutionErrors() == null + ? ImmutableList.of(job.getStatus().getError()) + : ImmutableList.copyOf(job.getStatus().getExecutionErrors())); + } + + // If there are no rows in the result, this may have been a DDL query. + // Listing table data might fail, such as with CREATE VIEW queries. + // Avoid a tabledata.list API request by returning an empty TableResult. + if (response.getTotalRows() == 0) { + TableResult emptyTableResult = + TableResult.newBuilder() + .setSchema(response.getSchema()) + .setJobId(job.getJobId()) + .setTotalRows(0L) + .setPageNoSchema(new PageImpl(null, "", null)) + .build(); + return emptyTableResult; + } + + TableId table = + ((QueryJobConfiguration) getConfiguration()).getDestinationTable() == null + ? ((QueryJobConfiguration) job.getConfiguration()).getDestinationTable() + : ((QueryJobConfiguration) getConfiguration()).getDestinationTable(); + TableResult tableResult = + bigquery.listTableData( + table, response.getSchema(), listOptions.toArray(new TableDataListOption[0])); + TableResult tableResultWithJobId = tableResult.toBuilder().setJobId(job.getJobId()).build(); + return tableResultWithJobId; + } finally { + if (getQueryResults != null) { + getQueryResults.end(); + } + } } private QueryResponse waitForQueryResults( @@ -382,7 +447,17 @@ private QueryResponse waitForQueryResults( "Waiting for query results is supported only for " + Type.QUERY + " jobs"); } - try { + Span waitForQueryResults = null; + if (options.isOpenTelemetryTracingEnabled() && options.getOpenTelemetryTracer() != null) { + waitForQueryResults = + options + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.Job.waitForQueryResults") + .setAllAttributes(otelAttributesFromOptions(resultsOptions)) + .startSpan(); + } + try (Scope waitForQueryResultsScope = + waitForQueryResults != null ? waitForQueryResults.makeCurrent() : null) { return BigQueryRetryHelper.runWithRetries( new Callable() { @Override @@ -401,14 +476,43 @@ public boolean shouldRetry( } }, options.getClock(), - bigQueryRetryConfig); + bigQueryRetryConfig, + options.isOpenTelemetryTracingEnabled(), + options.getOpenTelemetryTracer()); } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (waitForQueryResults != null) { + waitForQueryResults.end(); + } } } private Job waitForJob(RetrySettings waitSettings) throws InterruptedException { - try { + Span waitForJob = null; + if (options.isOpenTelemetryTracingEnabled() && options.getOpenTelemetryTracer() != null) { + waitForJob = + this.options + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.Job.waitForJob") + .setAttribute( + "bq.job.wait_settings.total_timeout", + getFieldAsString(waitSettings.getTotalTimeoutDuration())) + .setAttribute( + "bq.job.wait_settings.initial_retry_delay", + getFieldAsString(waitSettings.getInitialRetryDelayDuration())) + .setAttribute( + "bq.job.wait_settings.max_retry_delay", + getFieldAsString(waitSettings.getMaxRetryDelayDuration())) + .setAttribute( + "bq.job.wait_settings.initial_rpc_timeout", + getFieldAsString(waitSettings.getInitialRpcTimeoutDuration())) + .setAttribute( + "bq.job.wait_settings.max_rpc_timeout", + getFieldAsString(waitSettings.getMaxRpcTimeoutDuration())) + .startSpan(); + } + try (Scope waitForJobScope = waitForJob != null ? waitForJob.makeCurrent() : null) { return RetryHelper.poll( new Callable() { @Override @@ -433,6 +537,10 @@ public boolean shouldRetry(Throwable prevThrowable, Job prevResponse) { options.getClock()); } catch (ExecutionException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (waitForJob != null) { + waitForJob.end(); + } } } @@ -463,14 +571,31 @@ public boolean shouldRetry(Throwable prevThrowable, Job prevResponse) { */ public Job reload(JobOption... options) { checkNotDryRun("reload"); - Job job = bigquery.getJob(getJobId(), options); - if (job != null && job.getStatus().getError() != null) { - throw new BigQueryException( - job.getStatus().getExecutionErrors() == null - ? ImmutableList.of(job.getStatus().getError()) - : ImmutableList.copyOf(job.getStatus().getExecutionErrors())); - } - return job; + Span reload = null; + if (this.options.isOpenTelemetryTracingEnabled() + && this.options.getOpenTelemetryTracer() != null) { + reload = + this.options + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.Job.reload") + .setAllAttributes(otelAttributesFromOptions(options)) + .startSpan(); + } + + try (Scope reloadScope = reload != null ? reload.makeCurrent() : null) { + Job job = bigquery.getJob(getJobId(), options); + if (job != null && job.getStatus().getError() != null) { + throw new BigQueryException( + job.getStatus().getExecutionErrors() == null + ? ImmutableList.of(job.getStatus().getError()) + : ImmutableList.copyOf(job.getStatus().getExecutionErrors())); + } + return job; + } finally { + if (reload != null) { + reload.end(); + } + } } /** @@ -492,7 +617,22 @@ public Job reload(JobOption... options) { */ public boolean cancel() { checkNotDryRun("cancel"); - return bigquery.cancel(getJobId()); + Span cancel = null; + if (options.isOpenTelemetryTracingEnabled() && options.getOpenTelemetryTracer() != null) { + cancel = + options + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.Job.cancel") + .startSpan(); + } + + try (Scope cancelScope = cancel != null ? cancel.makeCurrent() : null) { + return bigquery.cancel(getJobId()); + } finally { + if (cancel != null) { + cancel.end(); + } + } } private void checkNotDryRun(String op) { @@ -556,4 +696,19 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE static Job fromPb(BigQuery bigquery, com.google.api.services.bigquery.model.Job jobPb) { return new Job(bigquery, new JobInfo.BuilderImpl(jobPb)); } + + private static Attributes otelAttributesFromOptions(Option... options) { + Attributes attributes = Attributes.builder().build(); + for (Option option : options) { + attributes = + attributes.toBuilder() + .put(option.getRpcOption().toString(), option.getValue().toString()) + .build(); + } + return attributes; + } + + private static String getFieldAsString(Object field) { + return field == null ? "null" : field.toString(); + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableDataWriteChannel.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableDataWriteChannel.java index aabd87d47..cc64ec72f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableDataWriteChannel.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableDataWriteChannel.java @@ -20,6 +20,8 @@ import com.google.cloud.RestorableState; import com.google.cloud.WriteChannel; import com.google.cloud.bigquery.BigQueryRetryHelper.BigQueryRetryHelperException; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Scope; import java.io.IOException; import java.util.List; import java.util.Objects; @@ -50,7 +52,19 @@ public class TableDataWriteChannel @Override protected void flushBuffer(final int length, final boolean last) { - try { + Span flushBuffer = null; + if (getOptions().isOpenTelemetryTracingEnabled() + && getOptions().getOpenTelemetryTracer() != null) { + flushBuffer = + getOptions() + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.TableDataWriteChannel.flushBuffer") + .setAttribute("bq.table_data_write_channel.flush_buffer.length", length) + .setAttribute("bq.table_data_write_channel.flush_buffer.last", last) + .startSpan(); + } + + try (Scope flushBufferScope = flushBuffer != null ? flushBuffer.makeCurrent() : null) { com.google.api.services.bigquery.model.Job jobPb = BigQueryRetryHelper.runWithRetries( new Callable() { @@ -65,10 +79,16 @@ public com.google.api.services.bigquery.model.Job call() throws IOException { getOptions().getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, getOptions().getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + getOptions().isOpenTelemetryTracingEnabled(), + getOptions().getOpenTelemetryTracer()); job = jobPb != null ? Job.fromPb(getOptions().getService(), jobPb) : null; } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (flushBuffer != null) { + flushBuffer.end(); + } } } @@ -81,7 +101,18 @@ private static String open( final BigQueryOptions options, final JobId jobId, final WriteChannelConfiguration writeChannelConfiguration) { - try { + Span open = null; + if (options.isOpenTelemetryTracingEnabled() && options.getOpenTelemetryTracer() != null) { + open = + options + .getOpenTelemetryTracer() + .spanBuilder("com.google.cloud.bigquery.TableDataWriteChannel.open") + .setAllAttributes(jobId.getOtelAttributes()) + .setAllAttributes(writeChannelConfiguration.getDestinationTable().getOtelAttributes()) + .startSpan(); + } + + try (Scope openScope = open != null ? open.makeCurrent() : null) { return BigQueryRetryHelper.runWithRetries( new Callable() { @Override @@ -97,9 +128,15 @@ public String call() throws IOException { options.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, options.getClock(), - EMPTY_RETRY_CONFIG); + EMPTY_RETRY_CONFIG, + options.isOpenTelemetryTracingEnabled(), + options.getOpenTelemetryTracer()); } catch (BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); + } finally { + if (open != null) { + open.end(); + } } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index eceadcbca..551e1eb40 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -6489,6 +6489,14 @@ public void testLocation() throws Exception { assertThat(location).isNotEqualTo(wrongLocation); + Tracer tracer = otel.getTracer("Test Tracer"); + bigquery = + bigquery.getOptions().toBuilder() + .setEnableOpenTelemetryTracing(true) + .setOpenTelemetryTracer(tracer) + .build() + .getService(); + Dataset dataset = bigquery.create( DatasetInfo.newBuilder("locationset_" + UUID.randomUUID().toString().replace("-", "_")) @@ -6565,6 +6573,11 @@ public void testLocation() throws Exception { bigquery.writer( JobId.newBuilder().setLocation(location).build(), writeChannelConfiguration)) { writer.write(ByteBuffer.wrap("foo".getBytes())); + assertEquals( + OTEL_ATTRIBUTES + .get("com.google.cloud.bigquery.TableDataWriteChannel.open") + .get(AttributeKey.stringKey("bq.job.location")), + location); } try { @@ -6577,6 +6590,12 @@ public void testLocation() throws Exception { } } finally { bigquery.delete(dataset.getDatasetId(), DatasetDeleteOption.deleteContents()); + bigquery = + bigquery.getOptions().toBuilder() + .setEnableOpenTelemetryTracing(false) + .setOpenTelemetryTracer(null) + .build() + .getService(); } } @@ -7661,6 +7680,7 @@ public void testOpenTelemetryTracingTables() { assertEquals( OTEL_PARENT_SPAN_IDS.get("com.google.cloud.bigquery.BigQuery.updateTable"), OTEL_PARENT_SPAN_ID); + assertTrue(bigquery.delete(updatedTable.getTableId())); } @Test @@ -7680,6 +7700,9 @@ public void testOpenTelemetryTracingQuery() throws InterruptedException { assertNull(tableResult.getJobId()); assertNotNull(OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.queryRpc")); + assertNotNull( + OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQueryRetryHelper.runWithRetries")); + assertTrue(OTEL_ATTRIBUTES.containsKey("com.google.cloud.bigquery.BigQuery.query")); // Query job String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); @@ -7694,5 +7717,7 @@ public void testOpenTelemetryTracingQuery() throws InterruptedException { assertNotNull(OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.getQueryResults")); assertNotNull(OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.listTableData")); assertNotNull(OTEL_ATTRIBUTES.get("com.google.cloud.bigquery.BigQuery.createJob")); + assertTrue(OTEL_ATTRIBUTES.containsKey("com.google.cloud.bigquery.Job.getQueryResults")); + assertTrue(OTEL_ATTRIBUTES.containsKey("com.google.cloud.bigquery.Job.waitForQueryResults")); } } From d2e6b4a58ddc817ec0f7c255f5b6b15ce6b2a649 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 24 Jun 2025 15:52:35 +0200 Subject: [PATCH 19/24] chore(deps): update google cloud java monorepo updates (#3857) --- pom.xml | 4 ++-- samples/install-without-bom/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 2cdc232f0..6e992241d 100644 --- a/pom.xml +++ b/pom.xml @@ -79,7 +79,7 @@ com.google.cloud google-cloud-datacatalog-bom - 1.70.0 + 1.72.0 pom import @@ -143,7 +143,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.66.0 + 2.68.0 test diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index ec9180d6a..4089821fe 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -69,7 +69,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.66.0 + 2.68.0 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index b8ba50d01..775c14fe8 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -67,7 +67,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.66.0 + 2.68.0 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 478359e9a..160e38cb0 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -85,7 +85,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.66.0 + 2.68.0 test From d4ca3535f54f3282aec133337103bbfa2c9a3653 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 24 Jun 2025 16:46:33 +0200 Subject: [PATCH 20/24] deps: update dependency com.google.api.grpc:proto-google-cloud-bigqueryconnection-v1 to v2.68.0 (#3858) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6e992241d..f33a3245a 100644 --- a/pom.xml +++ b/pom.xml @@ -149,7 +149,7 @@ com.google.api.grpc proto-google-cloud-bigqueryconnection-v1 - 2.66.0 + 2.68.0 test From eb26deee37119389aee3962eea5ad67d63f26c70 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 25 Jun 2025 15:59:21 +0200 Subject: [PATCH 21/24] deps: update dependency com.google.cloud:sdk-platform-java-config to v3.50.0 (#3861) --- .github/workflows/unmanaged_dependency_check.yaml | 2 +- .kokoro/continuous/graalvm-native-a.cfg | 2 +- .kokoro/continuous/graalvm-native-b.cfg | 2 +- .kokoro/continuous/graalvm-native-c.cfg | 2 +- .kokoro/presubmit/graalvm-native-a.cfg | 2 +- .kokoro/presubmit/graalvm-native-b.cfg | 2 +- .kokoro/presubmit/graalvm-native-c.cfg | 2 +- google-cloud-bigquery-bom/pom.xml | 2 +- pom.xml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unmanaged_dependency_check.yaml b/.github/workflows/unmanaged_dependency_check.yaml index 8449bf074..63543a40f 100644 --- a/.github/workflows/unmanaged_dependency_check.yaml +++ b/.github/workflows/unmanaged_dependency_check.yaml @@ -17,7 +17,7 @@ jobs: # repository .kokoro/build.sh - name: Unmanaged dependency check - uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.49.2 + uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.50.0 with: # java-bigquery does not produce a BOM. Fortunately the root pom.xml # defines google-cloud-bigquery in dependencyManagement section. So diff --git a/.kokoro/continuous/graalvm-native-a.cfg b/.kokoro/continuous/graalvm-native-a.cfg index b6a5a9d3e..dfa34060b 100644 --- a/.kokoro/continuous/graalvm-native-a.cfg +++ b/.kokoro/continuous/graalvm-native-a.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.49.2" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.50.0" } env_vars: { diff --git a/.kokoro/continuous/graalvm-native-b.cfg b/.kokoro/continuous/graalvm-native-b.cfg index d0d6dea71..37dda4ab5 100644 --- a/.kokoro/continuous/graalvm-native-b.cfg +++ b/.kokoro/continuous/graalvm-native-b.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.49.2" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.50.0" } env_vars: { diff --git a/.kokoro/continuous/graalvm-native-c.cfg b/.kokoro/continuous/graalvm-native-c.cfg index a4a1fe79e..a8ab97784 100644 --- a/.kokoro/continuous/graalvm-native-c.cfg +++ b/.kokoro/continuous/graalvm-native-c.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_c:3.49.2" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_c:3.50.0" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native-a.cfg b/.kokoro/presubmit/graalvm-native-a.cfg index b6a5a9d3e..dfa34060b 100644 --- a/.kokoro/presubmit/graalvm-native-a.cfg +++ b/.kokoro/presubmit/graalvm-native-a.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.49.2" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.50.0" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native-b.cfg b/.kokoro/presubmit/graalvm-native-b.cfg index d0d6dea71..37dda4ab5 100644 --- a/.kokoro/presubmit/graalvm-native-b.cfg +++ b/.kokoro/presubmit/graalvm-native-b.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.49.2" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.50.0" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native-c.cfg b/.kokoro/presubmit/graalvm-native-c.cfg index a4a1fe79e..a8ab97784 100644 --- a/.kokoro/presubmit/graalvm-native-c.cfg +++ b/.kokoro/presubmit/graalvm-native-c.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_c:3.49.2" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_c:3.50.0" } env_vars: { diff --git a/google-cloud-bigquery-bom/pom.xml b/google-cloud-bigquery-bom/pom.xml index 9c1967a6d..091c1d625 100644 --- a/google-cloud-bigquery-bom/pom.xml +++ b/google-cloud-bigquery-bom/pom.xml @@ -8,7 +8,7 @@ com.google.cloud sdk-platform-java-config - 3.49.2 + 3.50.0 diff --git a/pom.xml b/pom.xml index f33a3245a..c4ae2a12f 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ com.google.cloud sdk-platform-java-config - 3.49.2 + 3.50.0 From 87ff9e1632503bb2b01ab93e6b2ae6eae4471cba Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 25 Jun 2025 16:55:34 +0200 Subject: [PATCH 22/24] chore(config): migrate config renovate.json (#3862) --- renovate.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/renovate.json b/renovate.json index 833ec769e..aacfaebeb 100644 --- a/renovate.json +++ b/renovate.json @@ -15,9 +15,9 @@ "customManagers": [ { "customType": "regex", - "fileMatch": [ - "^.kokoro/continuous/graalvm-native.*.cfg$", - "^.kokoro/presubmit/graalvm-native.*.cfg$" + "managerFilePatterns": [ + "/^.kokoro/continuous/graalvm-native.*.cfg$/", + "/^.kokoro/presubmit/graalvm-native.*.cfg$/" ], "matchStrings": [ "value: \"gcr.io/cloud-devrel-public-resources/graalvm.*:(?.*?)\"" @@ -27,8 +27,8 @@ }, { "customType": "regex", - "fileMatch": [ - "^.github/workflows/unmanaged_dependency_check.yaml$" + "managerFilePatterns": [ + "/^.github/workflows/unmanaged_dependency_check.yaml$/" ], "matchStrings": [ "uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v(?.+?)\\n" From 2498739ce5a37f776d158394d2939dd55a9c600a Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 25 Jun 2025 21:52:14 +0200 Subject: [PATCH 23/24] chore(deps): update dependency com.google.cloud:google-cloud-bigquerystorage-bom to v3.15.3 (#3863) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c4ae2a12f..dc0b89ee4 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ com.google.cloud google-cloud-bigquerystorage-bom - 3.15.2 + 3.15.3 pom import From 13894df2d920930319660b220d13a2628ba705c6 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 16:48:52 -0400 Subject: [PATCH 24/24] chore(main): release 2.52.0 (#3849) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 24 ++++++++++++++++++++++++ benchmark/pom.xml | 2 +- google-cloud-bigquery-bom/pom.xml | 4 ++-- google-cloud-bigquery/pom.xml | 4 ++-- pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 2 +- 7 files changed, 33 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9fe4fdd3..71577758b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## [2.52.0](https://github.com/googleapis/java-bigquery/compare/v2.51.0...v2.52.0) (2025-06-25) + + +### Features + +* **bigquery:** Integrate Otel in client lib ([#3747](https://github.com/googleapis/java-bigquery/issues/3747)) ([6e3e07a](https://github.com/googleapis/java-bigquery/commit/6e3e07a22b8397e1e9d5b567589e44abc55961f2)) +* **bigquery:** Integrate Otel into retries, jobs, and more ([#3842](https://github.com/googleapis/java-bigquery/issues/3842)) ([4b28c47](https://github.com/googleapis/java-bigquery/commit/4b28c479c1bc22326c8d2501354fb86ec2ce1744)) + + +### Bug Fixes + +* **bigquery:** Add MY_VIEW_DATASET_NAME_TEST_ to resource clean up sample ([#3838](https://github.com/googleapis/java-bigquery/issues/3838)) ([b1962a7](https://github.com/googleapis/java-bigquery/commit/b1962a7f0084ee4c3e248266b50406cf575cd657)) + + +### Dependencies + +* Remove version declaration of open-telemetry-bom ([#3855](https://github.com/googleapis/java-bigquery/issues/3855)) ([6f9f77d](https://github.com/googleapis/java-bigquery/commit/6f9f77d47596b00b7317c8a0d4a10c3d849ad57b)) +* Update dependency com.google.api.grpc:proto-google-cloud-bigqueryconnection-v1 to v2.66.0 ([#3835](https://github.com/googleapis/java-bigquery/issues/3835)) ([69be5e7](https://github.com/googleapis/java-bigquery/commit/69be5e7345fb8ca69d633d9dc99cf6c15fa5227b)) +* Update dependency com.google.api.grpc:proto-google-cloud-bigqueryconnection-v1 to v2.68.0 ([#3858](https://github.com/googleapis/java-bigquery/issues/3858)) ([d4ca353](https://github.com/googleapis/java-bigquery/commit/d4ca3535f54f3282aec133337103bbfa2c9a3653)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.49.2 ([#3853](https://github.com/googleapis/java-bigquery/issues/3853)) ([cf864df](https://github.com/googleapis/java-bigquery/commit/cf864df739bbb820e99999b7c1592a3635fea4ec)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.50.0 ([#3861](https://github.com/googleapis/java-bigquery/issues/3861)) ([eb26dee](https://github.com/googleapis/java-bigquery/commit/eb26deee37119389aee3962eea5ad67d63f26c70)) +* Update dependency io.opentelemetry:opentelemetry-bom to v1.51.0 ([#3840](https://github.com/googleapis/java-bigquery/issues/3840)) ([51321c2](https://github.com/googleapis/java-bigquery/commit/51321c22778fd41134cc0cdfc70bdc47f05883f1)) +* Update ossf/scorecard-action action to v2.4.2 ([#3810](https://github.com/googleapis/java-bigquery/issues/3810)) ([414f61d](https://github.com/googleapis/java-bigquery/commit/414f61d7efcfa568c1446bd41945d7a8e2450649)) + ## [2.51.0](https://github.com/googleapis/java-bigquery/compare/v2.50.1...v2.51.0) (2025-06-06) diff --git a/benchmark/pom.xml b/benchmark/pom.xml index cd645c356..bb58ca1e1 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -6,7 +6,7 @@ google-cloud-bigquery-parent com.google.cloud - 2.51.1-SNAPSHOT + 2.52.0 diff --git a/google-cloud-bigquery-bom/pom.xml b/google-cloud-bigquery-bom/pom.xml index 091c1d625..836073eb4 100644 --- a/google-cloud-bigquery-bom/pom.xml +++ b/google-cloud-bigquery-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery-bom - 2.51.1-SNAPSHOT + 2.52.0 pom com.google.cloud @@ -54,7 +54,7 @@ com.google.cloud google-cloud-bigquery - 2.51.1-SNAPSHOT + 2.52.0 diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 4162718fb..644fe0ef5 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery - 2.51.1-SNAPSHOT + 2.52.0 jar BigQuery https://github.com/googleapis/java-bigquery @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquery-parent - 2.51.1-SNAPSHOT + 2.52.0 google-cloud-bigquery diff --git a/pom.xml b/pom.xml index dc0b89ee4..b265c66e3 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquery-parent pom - 2.51.1-SNAPSHOT + 2.52.0 BigQuery Parent https://github.com/googleapis/java-bigquery @@ -93,7 +93,7 @@ com.google.cloud google-cloud-bigquery - 2.51.1-SNAPSHOT + 2.52.0 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 775c14fe8..1a1655234 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -44,7 +44,7 @@ com.google.cloud google-cloud-bigquery - 2.51.1-SNAPSHOT + 2.52.0 diff --git a/versions.txt b/versions.txt index 7622bb1dc..2b5922874 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-bigquery:2.51.0:2.51.1-SNAPSHOT \ No newline at end of file +google-cloud-bigquery:2.52.0:2.52.0 \ No newline at end of file pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy