Skip to content

feat: add user-agent delegation telemetry for pandas-gbq #18184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
shuoweil wants to merge 4 commits into
base: main
Choose a base branch
from
+282 −0
Open

feat: add user-agent delegation telemetry for pandas-gbq #18184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e9dfcb7
feat(bigquery): inject user-agent telemetry for pandas-gbq delegation
shuoweil Aug 20, 2026
68f68bf
Update packages/google-cloud-bigquery/tests/unit/test_table.py
shuoweil Aug 21, 2026
f033e19
fix(bigquery): defensively update user-agent telemetry
shuoweil Aug 21, 2026
776a74f
fix(bigquery): downgrade telemetry failure log to debug
shuoweil Aug 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions packages/google-cloud-bigquery/google/cloud/bigquery/table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import copy
import datetime
import functools
import logging
import operator
import typing
import warnings
Expand Down Expand Up @@ -86,6 +87,7 @@
from google.cloud import bigquery_storage # type: ignore
from google.cloud.bigquery.dataset import DatasetReference

_LOGGER = logging.getLogger(__name__)

_NO_GEOPANDAS_ERROR = (
"The geopandas library is not installed, please install "
Expand Down Expand Up @@ -2991,6 +2993,21 @@ def to_dataframe(
create_bqstorage_client = False
bqstorage_client = None

if _versions_helpers.PANDAS_GBQ_VERSIONS.is_delegation_supported:
Comment on lines 2995 to +2996

@sycai sycai Aug 21, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you share the link to the code where pandas-gbq is used as the delegation?

@shuoweil shuoweil Aug 31, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In Phase 1 (Storage Read MVP), delegation is implemented in google-cloud-bigquery-storage ReadRowsPage.to_arrow() (merged in PR #17938 / v2.40.0), which delegates row decoding directly to pandas_gbq.arrow.from_read_rows_response() (released in pandas-gbq v0.35.1 via PR #17958).

When RowIterator.to_dataframe() or to_geodataframe() runs, it calls self.to_arrow() using the Storage Read client, executing through this delegation path.

_versions_helpers.PANDAS_GBQ_VERSIONS.is_delegation_supported (merged in PR #17957) checks _internal_delegation_api_version to safely gate the pandas-gbq/{version} user-agent telemetry decoration so we can track delegated usage.

try:
client_info = getattr(
getattr(self.client, "_connection", None), "_client_info", None
)
if client_info:
ua = getattr(client_info, "user_agent", None) or ""
if "pandas-gbq" not in ua:
version = (
_versions_helpers.PANDAS_GBQ_VERSIONS.installed_version
)
client_info.user_agent = f"{ua} pandas-gbq/{version}".strip()
except Exception as exc:
_LOGGER.debug("Failed to update telemetry user-agent: %s", exc)

with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
Expand Down
265 changes: 265 additions & 0 deletions packages/google-cloud-bigquery/tests/unit/test_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import datetime
import logging
import re
import sys
import time
import types
import unittest
Expand Down Expand Up @@ -5795,6 +5796,270 @@ def test_to_geodataframe_does_not_emit_deprecation_warning(self):
]
self.assertEqual(len(deprecation_warnings), 0)

def test_to_dataframe_delegated_updates_user_agent(self):
pytest.importorskip("db_dtypes")
pandas = pytest.importorskip("pandas")
mock_pandas_gbq = mock.Mock()
mock_pandas_gbq.__version__ = "1.0.0"

mock_client_info = mock.Mock()
mock_client_info.user_agent = "gl-python/3.10.0"

mock_client = _mock_client()
mock_client._connection = mock.Mock(_client_info=mock_client_info)

with (
mock.patch(
"google.cloud.bigquery._versions_helpers.PandasGBQVersions.is_delegation_supported",
new_callable=mock.PropertyMock,
return_value=True,
),
mock.patch(
"google.cloud.bigquery._versions_helpers.SUPPORTS_RANGE_PYARROW",
False,
),
mock.patch.dict(sys.modules, {"pandas_gbq": mock_pandas_gbq}),
):
row_iterator = self._make_one_from_data((("name", "STRING"),), (("foo",),))
row_iterator.client = mock_client

df = row_iterator.to_dataframe(progress_bar_type="tqdm", timeout=5.0)

self.assertIsInstance(df, pandas.DataFrame)
self.assertEqual(
mock_client_info.user_agent,
"gl-python/3.10.0 pandas-gbq/1.0.0",
)

def test_to_dataframe_delegated_does_not_duplicate_user_agent(self):
pytest.importorskip("db_dtypes")
pandas = pytest.importorskip("pandas")
mock_pandas_gbq = mock.Mock()
mock_pandas_gbq.__version__ = "1.0.0"

mock_client_info = mock.Mock()
mock_client_info.user_agent = "gl-python/3.10.0 pandas-gbq/1.0.0"

mock_client = _mock_client()
mock_client._connection = mock.Mock(_client_info=mock_client_info)

with (
mock.patch(
"google.cloud.bigquery._versions_helpers.PandasGBQVersions.is_delegation_supported",
new_callable=mock.PropertyMock,
return_value=True,
),
mock.patch(
"google.cloud.bigquery._versions_helpers.SUPPORTS_RANGE_PYARROW",
False,
),
mock.patch.dict(sys.modules, {"pandas_gbq": mock_pandas_gbq}),
):
row_iterator = self._make_one_from_data((("name", "STRING"),), (("foo",),))
row_iterator.client = mock_client

df = row_iterator.to_dataframe(progress_bar_type="tqdm", timeout=5.0)

self.assertIsInstance(df, pandas.DataFrame)
self.assertEqual(
mock_client_info.user_agent,
"gl-python/3.10.0 pandas-gbq/1.0.0",
)

def test_to_dataframe_delegated_when_client_info_is_none(self):
pytest.importorskip("db_dtypes")
pandas = pytest.importorskip("pandas")
mock_pandas_gbq = mock.Mock()
mock_pandas_gbq.__version__ = "1.0.0"

mock_client = _mock_client()
mock_client._connection = mock.Mock(_client_info=None)

with (
mock.patch(
"google.cloud.bigquery._versions_helpers.PandasGBQVersions.is_delegation_supported",
new_callable=mock.PropertyMock,
return_value=True,
),
mock.patch(
"google.cloud.bigquery._versions_helpers.SUPPORTS_RANGE_PYARROW",
False,
),
mock.patch.dict(sys.modules, {"pandas_gbq": mock_pandas_gbq}),
):
row_iterator = self._make_one_from_data((("name", "STRING"),), (("foo",),))
row_iterator.client = mock_client

df = row_iterator.to_dataframe(progress_bar_type="tqdm", timeout=5.0)

self.assertIsInstance(df, pandas.DataFrame)

def test_to_dataframe_delegated_when_user_agent_is_none(self):
pytest.importorskip("db_dtypes")
pandas = pytest.importorskip("pandas")
mock_pandas_gbq = mock.Mock()
mock_pandas_gbq.__version__ = "1.0.0"

mock_client_info = mock.Mock()
mock_client_info.user_agent = None

mock_client = _mock_client()
mock_client._connection = mock.Mock(_client_info=mock_client_info)

with (
mock.patch(
"google.cloud.bigquery._versions_helpers.PandasGBQVersions.is_delegation_supported",
new_callable=mock.PropertyMock,
return_value=True,
),
mock.patch(
"google.cloud.bigquery._versions_helpers.SUPPORTS_RANGE_PYARROW",
False,
),
mock.patch.dict(sys.modules, {"pandas_gbq": mock_pandas_gbq}),
):
row_iterator = self._make_one_from_data((("name", "STRING"),), (("foo",),))
row_iterator.client = mock_client

df = row_iterator.to_dataframe(progress_bar_type="tqdm", timeout=5.0)

self.assertIsInstance(df, pandas.DataFrame)
self.assertEqual(
mock_client_info.user_agent,
"pandas-gbq/1.0.0",
)

def test_to_dataframe_delegated_false_does_not_update_user_agent(self):
pytest.importorskip("db_dtypes")
pandas = pytest.importorskip("pandas")

mock_client_info = mock.Mock()
mock_client_info.user_agent = "gl-python/3.10.0"

mock_client = _mock_client()
mock_client._connection = mock.Mock(_client_info=mock_client_info)

with (
mock.patch(
"google.cloud.bigquery._versions_helpers.PandasGBQVersions.is_delegation_supported",
new_callable=mock.PropertyMock,
return_value=False,
),
mock.patch(
"google.cloud.bigquery._versions_helpers.SUPPORTS_RANGE_PYARROW",
False,
),
):
row_iterator = self._make_one_from_data((("name", "STRING"),), (("foo",),))
row_iterator.client = mock_client

df = row_iterator.to_dataframe(progress_bar_type="tqdm", timeout=5.0)

self.assertIsInstance(df, pandas.DataFrame)
self.assertEqual(
mock_client_info.user_agent,
"gl-python/3.10.0",
)

def test_to_dataframe_delegated_when_user_agent_update_fails_logs_debug(self):
pytest.importorskip("db_dtypes")
pandas = pytest.importorskip("pandas")
mock_pandas_gbq = mock.Mock()
mock_pandas_gbq.__version__ = "1.0.0"

class ReadOnlyClientInfo:
@property
def user_agent(self):
return "gl-python/3.10.0"

@user_agent.setter
def user_agent(self, value):
raise AttributeError("user_agent is read-only")

mock_client = _mock_client()
mock_client._connection = mock.Mock(_client_info=ReadOnlyClientInfo())

with (
mock.patch(
"google.cloud.bigquery._versions_helpers.PandasGBQVersions.is_delegation_supported",
new_callable=mock.PropertyMock,
return_value=True,
),
mock.patch(
"google.cloud.bigquery._versions_helpers.SUPPORTS_RANGE_PYARROW",
False,
),
mock.patch.dict(sys.modules, {"pandas_gbq": mock_pandas_gbq}),
mock.patch("google.cloud.bigquery.table._LOGGER.debug") as mock_log,
):
row_iterator = self._make_one_from_data((("name", "STRING"),), (("foo",),))
row_iterator.client = mock_client

df = row_iterator.to_dataframe(progress_bar_type="tqdm", timeout=5.0)

self.assertIsInstance(df, pandas.DataFrame)
mock_log.assert_called_once()
self.assertIn(
"Failed to update telemetry user-agent", mock_log.call_args[0][0]
)

def test_to_geodataframe_updates_user_agent(self):
pytest.importorskip("geopandas")
pyarrow = pytest.importorskip("pyarrow")
row_iterator = self._make_one_from_data(
(("name", "STRING"), ("geog", "GEOGRAPHY")),
(("foo", "Point(0 0)"),),
)
mock_client_info = mock.Mock(user_agent="test-agent")
row_iterator.client._connection = mock.Mock(_client_info=mock_client_info)
batch = pyarrow.RecordBatch.from_arrays(
[pyarrow.array(["foo"]), pyarrow.array(["Point(0 0)"])],
names=["name", "geog"],
)

with (
mock.patch(
"google.cloud.bigquery._versions_helpers.PandasGBQVersions.is_delegation_supported",
new_callable=mock.PropertyMock,
return_value=True,
),
mock.patch(
"google.cloud.bigquery._versions_helpers.PandasGBQVersions.installed_version",
new_callable=mock.PropertyMock,
return_value="1.0.0",
),
mock.patch.object(row_iterator, "to_arrow", return_value=batch),
):
Comment thread
shuoweil marked this conversation as resolved.
_ = row_iterator.to_geodataframe(create_bqstorage_client=False)

self.assertIn("pandas-gbq/", mock_client_info.user_agent)

def test_to_geodataframe_delegated_false_does_not_update_user_agent(self):
pytest.importorskip("geopandas")
pyarrow = pytest.importorskip("pyarrow")
row_iterator = self._make_one_from_data(
(("name", "STRING"), ("geog", "GEOGRAPHY")),
(("foo", "Point(0 0)"),),
)
mock_client_info = mock.Mock(user_agent="test-agent")
row_iterator.client._connection = mock.Mock(_client_info=mock_client_info)
batch = pyarrow.RecordBatch.from_arrays(
[pyarrow.array(["foo"]), pyarrow.array(["Point(0 0)"])],
names=["name", "geog"],
)

with (
mock.patch(
"google.cloud.bigquery._versions_helpers.PandasGBQVersions.is_delegation_supported",
new_callable=mock.PropertyMock,
return_value=False,
),
mock.patch.object(row_iterator, "to_arrow", return_value=batch),
):
_ = row_iterator.to_geodataframe(create_bqstorage_client=False)

self.assertEqual(mock_client_info.user_agent, "test-agent")


class TestPartitionRange(unittest.TestCase):
def _get_target_class(self):
Expand Down
Loading