Skip to content

docs: use direct API for audio transcription #2447

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shuoweil merged 2 commits into from
Feb 9, 2026
Merged

docs: use direct API for audio transcription #2447

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
11a8b5e
refactor: use direct API for audio transcription
shuoweil Feb 7, 2026
04d491d
Merge branch 'main' into shuowei-blob-change-audio
shuoweil Feb 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 54 additions & 41 deletions notebooks/multimodal/multimodal_dataframe.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 9,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
Expand Down Expand Up @@ -1459,99 +1459,112 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"### 6. Audio transcribe function"
"### 6. Audio transcribe"
]
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n",
"instead of using `db_dtypes` in the future when available in pandas\n",
"(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n",
" warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n"
]
}
],
"outputs": [],
"source": [
"audio_gcs_path = \"gs://bigframes_blob_test/audio/*\"\n",
"df = bpd.from_glob_path(audio_gcs_path, name=\"audio\")"
]
},
{
"cell_type": "code",
"execution_count": 22,
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n",
"instead of using `db_dtypes` in the future when available in pandas\n",
"(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n",
" warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n",
"/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n",
"/usr/local/google/home/shuowei/src/python-bigquery-dataframes/bigframes/dtypes.py:987: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n",
"instead of using `db_dtypes` in the future when available in pandas\n",
"(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n",
" warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n"
]
},
{
"data": {
"text/html": [
"<pre>0 Now, as all books, not primarily intended as p...</pre>"
],
"text/plain": [
"0 Now, as all books, not primarily intended as p...\n",
"Name: transcribed_content, dtype: string"
]
},
"execution_count": 22,
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"transcribed_series = df['audio'].blob.audio_transcribe(model_name=\"gemini-2.0-flash-001\", verbose=False)\n",
"import bigframes.bigquery as bbq\n",
"import bigframes.operations as ops\n",
"\n",
"# The audio_transcribe function is a convenience wrapper around bigframes.bigquery.ai.generate.\n",
"# Here's how to perform the same operation directly:\n",
"\n",
"audio_series = df['audio']\n",
"prompt_text = (\n",
" \"**Task:** Transcribe the provided audio. **Instructions:** - Your response \"\n",
" \"must contain only the verbatim transcription of the audio. - Do not include \"\n",
" \"any introductory text, summaries, or conversational filler in your response. \"\n",
" \"The output should begin directly with the first word of the audio.\"\n",
")\n",
"\n",
"# Convert the audio series to the runtime representation required by the model.\n",
"# This involves fetching metadata and getting a signed access URL.\n",
"audio_metadata = audio_series._apply_unary_op(ops.obj_fetch_metadata_op)\n",
"audio_runtime = audio_metadata._apply_unary_op(ops.ObjGetAccessUrl(mode=\"R\"))\n",
"\n",
"transcribed_results = bbq.ai.generate(\n",
" prompt=(prompt_text, audio_runtime),\n",
" endpoint=\"gemini-2.0-flash-001\",\n",
" model_params={\"generationConfig\": {\"temperature\": 0.0}},\n",
")\n",
"\n",
"transcribed_series = transcribed_results.struct.field(\"result\").rename(\"transcribed_content\")\n",
"transcribed_series"
]
},
{
"cell_type": "code",
"execution_count": 23,
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n",
"instead of using `db_dtypes` in the future when available in pandas\n",
"(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n",
" warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n",
"/usr/local/google/home/shuowei/src/github.com/googleapis/python-bigquery-dataframes/bigframes/dtypes.py:959: JSONDtypeWarning: JSON columns will be represented as pandas.ArrowDtype(pyarrow.json_())\n",
"instead of using `db_dtypes` in the future when available in pandas\n",
"(https://github.com/pandas-dev/pandas/issues/60958) and pyarrow.\n",
" warnings.warn(msg, bigframes.exceptions.JSONDtypeWarning)\n"
]
},
{
"data": {
"text/html": [
"<pre>0 {'status': '', 'content': 'Now, as all books, ...</pre>"
],
"text/plain": [
"0 {'status': '', 'content': 'Now, as all books, ...\n",
"Name: transcription_results, dtype: struct<status: string, content: string>[pyarrow]"
]
},
"execution_count": 23,
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"transcribed_series_verbose = df['audio'].blob.audio_transcribe(model_name=\"gemini-2.0-flash-001\", verbose=True)\n",
"# To get verbose results (including status), we can extract both fields from the result struct.\n",
"transcribed_content_series = transcribed_results.struct.field(\"result\")\n",
"transcribed_status_series = transcribed_results.struct.field(\"status\")\n",
"\n",
"transcribed_series_verbose = bpd.DataFrame(\n",
" {\n",
" \"status\": transcribed_status_series,\n",
" \"content\": transcribed_content_series,\n",
" }\n",
")\n",
"# Package as a struct for consistent display\n",
"transcribed_series_verbose = bbq.struct(transcribed_series_verbose).rename(\"transcription_results\")\n",
"transcribed_series_verbose"
]
},
Expand Down Expand Up @@ -1657,7 +1670,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.18"
"version": "3.13.0"
}
},
"nbformat": 4,
Expand Down
Loading