From 5bbef60a54877dff0c4a4d4a9509be3cf6b26c1c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 2 Feb 2026 15:35:55 -0800 Subject: [PATCH 1/5] feat: respect RetryInfo metadata from retryable exceptions --- google/cloud/bigtable/data/_helpers.py | 16 ++++++++++++++-- .../bigtable/data/_metrics/tracked_retry.py | 12 +++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigtable/data/_helpers.py b/google/cloud/bigtable/data/_helpers.py index 13bcfcc29..81f355273 100644 --- a/google/cloud/bigtable/data/_helpers.py +++ b/google/cloud/bigtable/data/_helpers.py @@ -25,6 +25,7 @@ from google.api_core import exceptions as core_exceptions from google.api_core.retry import exponential_sleep_generator from google.api_core.retry import RetryFailureReason +from google.rpc.error_details_pb2 import RetryInfo from google.cloud.bigtable.data.exceptions import RetryExceptionGroup if TYPE_CHECKING: @@ -288,11 +289,10 @@ def set_next(self, next_value: float): self._next_override = next_value def __next__(self) -> float: + next_backoff = next(self.subgenerator) if self._next_override is not None: next_backoff = self._next_override self._next_override = None - else: - next_backoff = next(self.subgenerator) self.history.append(next_backoff) return next_backoff @@ -308,3 +308,15 @@ def get_attempt_backoff(self, attempt_idx) -> float: if attempt_idx < 0: raise IndexError("received negative attempt number") return self.history[attempt_idx] + + def set_from_exception_info(self, retry_info: RetryInfo): + """ + Use a RetryInfo object to set the next sleep time. + + If a problem is encountered, this method does nothing. + """ + try: + retry_seconds = retry_info.retry_delay.ToTimedelta().total_seconds() + self.set_next(retry_seconds) + except Exception: + pass diff --git a/google/cloud/bigtable/data/_metrics/tracked_retry.py b/google/cloud/bigtable/data/_metrics/tracked_retry.py index 94d2e5dcb..1e1279390 100644 --- a/google/cloud/bigtable/data/_metrics/tracked_retry.py +++ b/google/cloud/bigtable/data/_metrics/tracked_retry.py @@ -26,6 +26,7 @@ from grpc import StatusCode from google.api_core.exceptions import GoogleAPICallError from google.api_core.retry import RetryFailureReason +from google.rpc.error_details_pb2 import RetryInfo from google.cloud.bigtable.data.exceptions import _MutateRowsIncomplete from google.cloud.bigtable.data._helpers import _retry_exception_factory from google.cloud.bigtable.data._metrics import ActiveOperationMetric @@ -43,10 +44,14 @@ def _track_retryable_error( operation: ActiveOperationMetric, + backoff_generator: TrackedBackoffGenerator, ) -> Callable[[Exception], None]: """ Used as input to api_core.Retry classes, to track when retryable errors are encountered + If an excemption is encountered with Retryinfo set, it will inform the backoff generator + to give it a chance to override the next backoff value + Should be passed as on_error callback """ @@ -59,6 +64,11 @@ def wrapper(exc: Exception) -> None: rpc_error.initial_metadata() ) operation.add_response_metadata({k: v for k, v in metadata}) + # check for RetryInfo: + if exc.details: + info_matches = [field for field in exc.details if isinstance(field, RetryInfo)] + if info_matches: + backoff_generator.set_from_exception_info(info_matches[0]) except Exception: # ignore errors in metadata collection pass @@ -127,7 +137,7 @@ def tracked_retry( kwargs.pop("sleep_generator", None) return retry_fn( sleep_generator=operation.backoff_generator, - on_error=_track_retryable_error(operation), + on_error=_track_retryable_error(operation, operation.backoff_generator), exception_factory=_track_terminal_error(operation, in_exception_factory), **kwargs, ) From cd65fc64987b533fa1712a8256a4c2fc1f0a1d42 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 2 Feb 2026 15:37:58 -0800 Subject: [PATCH 2/5] fix typo --- google/cloud/bigtable/data/_metrics/tracked_retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigtable/data/_metrics/tracked_retry.py b/google/cloud/bigtable/data/_metrics/tracked_retry.py index 1e1279390..1443c2829 100644 --- a/google/cloud/bigtable/data/_metrics/tracked_retry.py +++ b/google/cloud/bigtable/data/_metrics/tracked_retry.py @@ -49,7 +49,7 @@ def _track_retryable_error( """ Used as input to api_core.Retry classes, to track when retryable errors are encountered - If an excemption is encountered with Retryinfo set, it will inform the backoff generator + If an exception is encountered with Retryinfo set, it will inform the backoff generator to give it a chance to override the next backoff value Should be passed as on_error callback From fc5cada23e5502b1aea74b56c2ac66cd923f007e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 2 Feb 2026 16:55:55 -0800 Subject: [PATCH 3/5] remvoed duplicate arguments --- google/cloud/bigtable/data/_metrics/tracked_retry.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigtable/data/_metrics/tracked_retry.py b/google/cloud/bigtable/data/_metrics/tracked_retry.py index 1443c2829..0d441f989 100644 --- a/google/cloud/bigtable/data/_metrics/tracked_retry.py +++ b/google/cloud/bigtable/data/_metrics/tracked_retry.py @@ -44,7 +44,6 @@ def _track_retryable_error( operation: ActiveOperationMetric, - backoff_generator: TrackedBackoffGenerator, ) -> Callable[[Exception], None]: """ Used as input to api_core.Retry classes, to track when retryable errors are encountered @@ -68,7 +67,7 @@ def wrapper(exc: Exception) -> None: if exc.details: info_matches = [field for field in exc.details if isinstance(field, RetryInfo)] if info_matches: - backoff_generator.set_from_exception_info(info_matches[0]) + operation.backoff_generator.set_from_exception_info(info_matches[0]) except Exception: # ignore errors in metadata collection pass @@ -137,7 +136,7 @@ def tracked_retry( kwargs.pop("sleep_generator", None) return retry_fn( sleep_generator=operation.backoff_generator, - on_error=_track_retryable_error(operation, operation.backoff_generator), + on_error=_track_retryable_error(operation), exception_factory=_track_terminal_error(operation, in_exception_factory), **kwargs, ) From 0710b0cddda99d7436124bc9ee3a2d4f480629c8 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 2 Feb 2026 17:03:13 -0800 Subject: [PATCH 4/5] simplied code to grab first retry_info --- google/cloud/bigtable/data/_metrics/tracked_retry.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigtable/data/_metrics/tracked_retry.py b/google/cloud/bigtable/data/_metrics/tracked_retry.py index 0d441f989..9590d982f 100644 --- a/google/cloud/bigtable/data/_metrics/tracked_retry.py +++ b/google/cloud/bigtable/data/_metrics/tracked_retry.py @@ -65,9 +65,9 @@ def wrapper(exc: Exception) -> None: operation.add_response_metadata({k: v for k, v in metadata}) # check for RetryInfo: if exc.details: - info_matches = [field for field in exc.details if isinstance(field, RetryInfo)] - if info_matches: - operation.backoff_generator.set_from_exception_info(info_matches[0]) + info = next((field for field in exc.details if isinstance(field, RetryInfo)), None) + if info: + operation.backoff_generator.set_from_exception_info(info) except Exception: # ignore errors in metadata collection pass From 58cb9c0635e2eeb4e6061e1439925d197c688853 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 2 Feb 2026 19:59:55 -0800 Subject: [PATCH 5/5] simplified change --- google/cloud/bigtable/data/_helpers.py | 13 ------------- .../cloud/bigtable/data/_metrics/tracked_retry.py | 4 +++- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/google/cloud/bigtable/data/_helpers.py b/google/cloud/bigtable/data/_helpers.py index 81f355273..5755d35a7 100644 --- a/google/cloud/bigtable/data/_helpers.py +++ b/google/cloud/bigtable/data/_helpers.py @@ -25,7 +25,6 @@ from google.api_core import exceptions as core_exceptions from google.api_core.retry import exponential_sleep_generator from google.api_core.retry import RetryFailureReason -from google.rpc.error_details_pb2 import RetryInfo from google.cloud.bigtable.data.exceptions import RetryExceptionGroup if TYPE_CHECKING: @@ -308,15 +307,3 @@ def get_attempt_backoff(self, attempt_idx) -> float: if attempt_idx < 0: raise IndexError("received negative attempt number") return self.history[attempt_idx] - - def set_from_exception_info(self, retry_info: RetryInfo): - """ - Use a RetryInfo object to set the next sleep time. - - If a problem is encountered, this method does nothing. - """ - try: - retry_seconds = retry_info.retry_delay.ToTimedelta().total_seconds() - self.set_next(retry_seconds) - except Exception: - pass diff --git a/google/cloud/bigtable/data/_metrics/tracked_retry.py b/google/cloud/bigtable/data/_metrics/tracked_retry.py index 9590d982f..3f8fea5ff 100644 --- a/google/cloud/bigtable/data/_metrics/tracked_retry.py +++ b/google/cloud/bigtable/data/_metrics/tracked_retry.py @@ -67,7 +67,9 @@ def wrapper(exc: Exception) -> None: if exc.details: info = next((field for field in exc.details if isinstance(field, RetryInfo)), None) if info: - operation.backoff_generator.set_from_exception_info(info) + # override next backoff with server-provided value + retry_seconds = info.retry_delay.ToTimedelta().total_seconds() + operation.backoff_generator.set_next(retry_seconds) except Exception: # ignore errors in metadata collection pass