diff --git a/packages/google-cloud-storage/google/cloud/storage/_bucket_metadata_cache.py b/packages/google-cloud-storage/google/cloud/storage/_bucket_metadata_cache.py index e431d00144fb..1c3f83e372a7 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_bucket_metadata_cache.py +++ b/packages/google-cloud-storage/google/cloud/storage/_bucket_metadata_cache.py @@ -20,6 +20,7 @@ from google.api_core import exceptions as api_exceptions from google.cloud.exceptions import NotFound +from google.cloud.storage import _helpers from google.cloud.storage._lru_cache import LRUCache logger = logging.getLogger(__name__) @@ -81,9 +82,10 @@ def check_and_evict(self, bucket_name): def _verify_existence_background(self, bucket_name): try: - bucket = self._client.bucket(bucket_name) - if not bucket.exists(): - self.evict(bucket_name) + with _helpers._suppress_internal_metrics(): + bucket = self._client.bucket(bucket_name) + if not bucket.exists(): + self.evict(bucket_name) except Exception as e: logger.debug( f"Background verification for bucket existence failed for {bucket_name}: {e}" @@ -95,7 +97,8 @@ def _verify_existence_background(self, bucket_name): def _fetch_background(self, bucket_name): """Asynchronously fetch bucket metadata and update the cache.""" try: - bucket = self._client.get_bucket(bucket_name, timeout=10.0) + with _helpers._suppress_internal_metrics(): + bucket = self._client.get_bucket(bucket_name, timeout=10.0) self.update_from_bucket(bucket) except (NotFound, api_exceptions.NotFound): self.evict(bucket_name) diff --git a/packages/google-cloud-storage/google/cloud/storage/_helpers.py b/packages/google-cloud-storage/google/cloud/storage/_helpers.py index 9bea6c40845e..f555a43e659d 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_helpers.py +++ b/packages/google-cloud-storage/google/cloud/storage/_helpers.py @@ -23,7 +23,8 @@ import os import secrets import sys -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext +from contextvars import ContextVar from hashlib import md5 from urllib.parse import urlsplit, urlunsplit from uuid import uuid4 @@ -32,6 +33,7 @@ from google.auth import environment_vars from google.cloud.exceptions import NotFound +from google.cloud.storage import _opentelemetry_metrics from google.cloud.storage._opentelemetry_tracing import ( _is_bucket_metadata_disabled, ) @@ -46,6 +48,177 @@ _logger = logging.getLogger(__name__) +_SUPPRESS_INTERNAL_METRICS = ContextVar("_gcs_suppress_internal_metrics", default=False) +_SUPPRESS_BODY_SIZE = ContextVar("_gcs_suppress_body_size", default=False) +_LIST_METRICS_CONTEXT = ContextVar("_gcs_list_metrics_context", default=None) +_CLIENT_METHOD_OVERRIDE = ContextVar("_gcs_client_method_override", default=None) +_ACTIVE_METRICS_RECORDER = ContextVar("_gcs_active_metrics_recorder", default=None) + + +def _resolve_operation_client(obj, client=None): + """Best-effort resolution of a client instance without raising if unbound.""" + if client is not None: + return client + try: + return obj.client + except Exception: + return getattr(obj, "_client", None) + + +@contextmanager +def _suppress_internal_metrics(): + """Suppresses all GCS standard metrics within internal background/cleanup calls.""" + token = _SUPPRESS_INTERNAL_METRICS.set(True) + try: + yield + finally: + _SUPPRESS_INTERNAL_METRICS.reset(token) + + +@contextmanager +def _suppress_body_size(): + """Suppresses per-chunk body-size emission while keeping TTFB/attempts active.""" + token = _SUPPRESS_BODY_SIZE.set(True) + try: + yield + finally: + _SUPPRESS_BODY_SIZE.reset(token) + + +@contextmanager +def _list_metrics_context(client_method, operation): + """Propagates outermost list Veneer attribution into ``Client._list_resource``.""" + if _LIST_METRICS_CONTEXT.get() is not None: + yield + return + token = _LIST_METRICS_CONTEXT.set((client_method, operation)) + try: + yield + finally: + _LIST_METRICS_CONTEXT.reset(token) + + +@contextmanager +def _client_method_override(client_method): + """Overrides ``gcp.client.method`` for outermost composite helpers (e.g. ``transfer_manager``).""" + if _CLIENT_METHOD_OVERRIDE.get() is not None: + yield + return + token = _CLIENT_METHOD_OVERRIDE.set(client_method) + try: + yield + finally: + _CLIENT_METHOD_OVERRIDE.reset(token) + + +_NON_BATCHABLE_OPERATIONS = frozenset( + {"Batch", "ReadObject", "WriteObject", "StartResumableWrite"} +) + + +def _record_primary_exception(exc, cleanup_exc): + """Pins ``exc`` on the active root operation when ``cleanup_exc`` escapes.""" + op = _opentelemetry_metrics.get_current_operation() + if op is not None: + prev_cleanup = getattr(op, "_pinned_cleanup_exc", None) + if prev_cleanup is None or not any( + item is prev_cleanup + for item in _opentelemetry_metrics._iter_exception_chain(cleanup_exc) + ): + op._pinned_exc = exc + op._pinned_cleanup_exc = cleanup_exc + + +@contextmanager +def _start_storage_operation(client, client_method, operation): + """Starts a logical GCS standard metrics operation if enabled on ``client``.""" + if _SUPPRESS_INTERNAL_METRICS.get(): + yield None + return + + recorder = _opentelemetry_metrics.get_active_recorder(client) + if recorder is None: + yield None + return + + current_batch = getattr(client, "current_batch", None) + if ( + operation not in _NON_BATCHABLE_OPERATIONS + and getattr(current_batch, "_IS_BATCH", False) is True + ): + yield None + return + + effective_method = _CLIENT_METHOD_OVERRIDE.get() or client_method + is_root = _opentelemetry_metrics.get_current_operation() is None + rec_token = _ACTIVE_METRICS_RECORDER.set(recorder) + try: + cm = _opentelemetry_metrics.start_operation( + recorder=recorder, + client_method=effective_method, + operation=operation, + ) + state = cm.__enter__() + try: + yield state + except BaseException as exc: + metric_exc = exc + if is_root: + pinned = getattr(state, "_pinned_exc", None) + cleanup_exc = getattr(state, "_pinned_cleanup_exc", None) + if pinned is not None and any( + item is cleanup_exc + for item in _opentelemetry_metrics._iter_exception_chain(exc) + ): + metric_exc = pinned + cm.__exit__(type(metric_exc), metric_exc, metric_exc.__traceback__) + raise + else: + cm.__exit__(None, None, None) + finally: + _ACTIVE_METRICS_RECORDER.reset(rec_token) + + +@contextmanager +def _bind_suboperation_context(client, client_method, operation): + """Binds parent operation attribution without emitting a nested logical operation. + + Used by streaming file I/O (``BlobReader``/``BlobWriter``) and concurrent + ``transfer_manager`` chunk workers so physical HTTP attempts and TTFB carry + the parent ``gcp.storage.operation`` while per-chunk logical operations and + body sizes are suppressed. + """ + if _SUPPRESS_INTERNAL_METRICS.get(): + yield None + return + + recorder = _opentelemetry_metrics.get_active_recorder(client) + if recorder is None: + yield None + return + + resolved_op = _opentelemetry_metrics.normalize_logical_operation(operation) + resolved_rpc = _opentelemetry_metrics.resolve_rpc_method( + resolved_op, rpc_method=operation + ) + op_state = _opentelemetry_metrics._OperationState( + client_method=client_method, + operation=resolved_op, + rpc_method=resolved_rpc, + suppressed=False, + ) + op_token = _opentelemetry_metrics._OP_STATE.set(op_state) + rec_token = _ACTIVE_METRICS_RECORDER.set(recorder) + body_token = _SUPPRESS_BODY_SIZE.set(True) + try: + yield op_state + finally: + op_state.finished = True + _SUPPRESS_BODY_SIZE.reset(body_token) + _ACTIVE_METRICS_RECORDER.reset(rec_token) + _opentelemetry_metrics._OP_STATE.reset(op_token) + + STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST" # Despite name, includes scheme. """Environment variable defining host for Storage emulator.""" @@ -219,6 +392,10 @@ class _PropertyMixin(object): number or letter. """ + _METRICS_CLASS_NAME = None + _METRICS_GET_OP = None + _METRICS_UPDATE_OP = None + def __init__(self, name=None): self.name = name self._properties = {} @@ -377,35 +554,47 @@ def reload( See: https://cloud.google.com/storage/docs/soft-delete """ client = self._require_client(client) - query_params = self._query_params - # Pass only '?projection=noAcl' here because 'acl' and related - # are handled via custom endpoints. - query_params["projection"] = projection - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) - if soft_deleted is not None: - query_params["softDeleted"] = soft_deleted - # Soft delete reload requires a generation, even for targets - # that don't include them in default query params (buckets). - query_params["generation"] = self.generation - headers = self._encryption_headers() - _add_etag_match_headers( - headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match - ) - api_response = client._get_resource( - self.path, - query_params=query_params, - headers=headers, - timeout=timeout, - retry=retry, - _target_object=self, - ) - self._set_properties(api_response) + op_name = self._METRICS_GET_OP + if op_name is not None: + cls_name = self._METRICS_CLASS_NAME or type(self).__name__ + op_ctx = _start_storage_operation( + client, f"storage.{cls_name}.reload", op_name + ) + else: + op_ctx = nullcontext() + + with op_ctx: + query_params = self._query_params + # Pass only '?projection=noAcl' here because 'acl' and related + # are handled via custom endpoints. + query_params["projection"] = projection + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) + if soft_deleted is not None: + query_params["softDeleted"] = soft_deleted + # Soft delete reload requires a generation, even for targets + # that don't include them in default query params (buckets). + query_params["generation"] = self.generation + headers = self._encryption_headers() + _add_etag_match_headers( + headers, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + ) + api_response = client._get_resource( + self.path, + query_params=query_params, + headers=headers, + timeout=timeout, + retry=retry, + _target_object=self, + ) + self._set_properties(api_response) def _patch_property(self, name, value): """Update field of this object's properties. @@ -490,31 +679,41 @@ def patch( https://cloud.google.com/storage/docs/json_api/v1/objects/patch """ client = self._require_client(client) - query_params = self._query_params - # Pass '?projection=full' here because 'PATCH' documented not - # to work properly w/ 'noAcl'. - query_params["projection"] = "full" - if override_unlocked_retention: - query_params["overrideUnlockedRetention"] = override_unlocked_retention - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) - update_properties = {key: self._properties[key] for key in self._changes} - - # Make the API call. - api_response = client._patch_resource( - self.path, - update_properties, - query_params=query_params, - _target_object=self, - timeout=timeout, - retry=retry, - ) - self._set_properties(api_response) + op_name = self._METRICS_UPDATE_OP + if op_name is not None: + cls_name = self._METRICS_CLASS_NAME or type(self).__name__ + op_ctx = _start_storage_operation( + client, f"storage.{cls_name}.patch", op_name + ) + else: + op_ctx = nullcontext() + + with op_ctx: + query_params = self._query_params + # Pass '?projection=full' here because 'PATCH' documented not + # to work properly w/ 'noAcl'. + query_params["projection"] = "full" + if override_unlocked_retention: + query_params["overrideUnlockedRetention"] = override_unlocked_retention + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) + update_properties = {key: self._properties[key] for key in self._changes} + + # Make the API call. + api_response = client._patch_resource( + self.path, + update_properties, + query_params=query_params, + _target_object=self, + timeout=timeout, + retry=retry, + ) + self._set_properties(api_response) def update( self, @@ -571,28 +770,37 @@ def update( https://cloud.google.com/storage/docs/json_api/v1/objects/patch """ client = self._require_client(client) + op_name = self._METRICS_UPDATE_OP + if op_name is not None: + cls_name = self._METRICS_CLASS_NAME or type(self).__name__ + op_ctx = _start_storage_operation( + client, f"storage.{cls_name}.update", op_name + ) + else: + op_ctx = nullcontext() + + with op_ctx: + query_params = self._query_params + query_params["projection"] = "full" + if override_unlocked_retention: + query_params["overrideUnlockedRetention"] = override_unlocked_retention + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) - query_params = self._query_params - query_params["projection"] = "full" - if override_unlocked_retention: - query_params["overrideUnlockedRetention"] = override_unlocked_retention - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) - - api_response = client._put_resource( - self.path, - self._properties, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=self, - ) - self._set_properties(api_response) + api_response = client._put_resource( + self.path, + self._properties, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=self, + ) + self._set_properties(api_response) def _scalar_property(fieldname): diff --git a/packages/google-cloud-storage/google/cloud/storage/_http.py b/packages/google-cloud-storage/google/cloud/storage/_http.py index 72734f511b41..5b0eaa432cdd 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_http.py +++ b/packages/google-cloud-storage/google/cloud/storage/_http.py @@ -17,12 +17,14 @@ import functools import logging import re +import time +from contextvars import ContextVar from google.api_core import exceptions as api_exceptions from google.cloud.exceptions import NotFound from google.cloud import _http -from google.cloud.storage import __version__, _helpers +from google.cloud.storage import __version__, _helpers, _opentelemetry_metrics from google.cloud.storage._opentelemetry_tracing import ( HAS_OPENTELEMETRY, _is_bucket_metadata_disabled, @@ -32,6 +34,9 @@ logger = logging.getLogger(__name__) +_IN_API_REQUEST = ContextVar("_gcs_in_api_request", default=False) +_LAST_ATTEMPT_RESPONSE = ContextVar("_gcs_last_attempt_response", default=None) + class Connection(_http.JSONConnection): """A connection to Google Cloud Storage via the JSON REST API. @@ -55,6 +60,7 @@ class Connection(_http.JSONConnection): DEFAULT_API_ENDPOINT = _helpers._get_default_storage_base_url() DEFAULT_API_MTLS_ENDPOINT = "https://storage.mtls.googleapis.com" + _IS_BATCH = False def __init__(self, client, client_info=None, api_endpoint=None): super(Connection, self).__init__(client, client_info) @@ -76,6 +82,64 @@ def __init__(self, client, client_info=None, api_endpoint=None): API_URL_TEMPLATE = "{api_base_url}/storage/{api_version}{path}" """A template for the URL of a particular API call.""" + def _make_request(self, method, url, *args, **kwargs): + if ( + getattr(self, "_IS_BATCH", False) is True + or _helpers._SUPPRESS_INTERNAL_METRICS.get() + ): + return super(Connection, self)._make_request(method, url, *args, **kwargs) + + recorder = _opentelemetry_metrics.get_active_recorder(self._client) + if recorder is None: + return super(Connection, self)._make_request(method, url, *args, **kwargs) + + if _IN_API_REQUEST.get(): + req_start = time.perf_counter() + response = super(Connection, self)._make_request( + method, url, *args, **kwargs + ) + _LAST_ATTEMPT_RESPONSE.set((response, time.perf_counter() - req_start, url)) + return response + + url_template = _opentelemetry_metrics.normalize_url_template(url) + attempt_start = time.perf_counter() + try: + response = super(Connection, self)._make_request( + method, url, *args, **kwargs + ) + except BaseException as exc: + recorder.record_http_attempt( + duration=time.perf_counter() - attempt_start, + http_method=method, + url_template=url_template, + status_code=None, + error_type=_opentelemetry_metrics.classify_error_type(exc), + network_path="cloudpath", + ) + raise + + duration = time.perf_counter() - attempt_start + raw_sc = getattr(response, "status_code", None) + status_code = ( + raw_sc + if (isinstance(raw_sc, int) and not isinstance(raw_sc, bool)) + else None + ) + if status_code is not None: + recorder.record_ttfb(duration, network_path="cloudpath") + error_type = _opentelemetry_metrics.classify_error_type( + None, status_code=status_code + ) + recorder.record_http_attempt( + duration=duration, + http_method=method, + url_template=url_template, + status_code=status_code, + error_type=error_type, + network_path="cloudpath", + ) + return response + def api_request(self, *args, **kwargs): retry = kwargs.pop("retry", None) invocation_id = _helpers._get_invocation_id() @@ -103,7 +167,102 @@ def api_request(self, *args, **kwargs): except Exception as e: logger.debug(f"Failed cache.get_or_queue_fetch in api_request: {e}") - call = functools.partial(super(Connection, self).api_request, *args, **kwargs) + base_call = functools.partial( + super(Connection, self).api_request, *args, **kwargs + ) + recorder = ( + None + if ( + getattr(self, "_IS_BATCH", False) is True + or _helpers._SUPPRESS_INTERNAL_METRICS.get() + ) + else _opentelemetry_metrics.get_active_recorder(client) + ) + + if recorder is not None: + http_method = args[0] if len(args) >= 1 else kwargs.get("method", "") + raw_path = args[1] if len(args) >= 2 else kwargs.get("path", "") + fallback_template = _opentelemetry_metrics.normalize_url_template(raw_path) + + def _timed_attempt(): + _LAST_ATTEMPT_RESPONSE.set(None) + attempt_start = time.perf_counter() + try: + result = base_call() + except BaseException as exc: + duration = time.perf_counter() - attempt_start + stashed = _LAST_ATTEMPT_RESPONSE.get() + _LAST_ATTEMPT_RESPONSE.set(None) + if stashed is not None: + resp, ttfb_dur, req_url = stashed + raw_sc = getattr(resp, "status_code", None) + status_code = ( + raw_sc + if ( + isinstance(raw_sc, int) and not isinstance(raw_sc, bool) + ) + else None + ) + url_template = ( + _opentelemetry_metrics.normalize_url_template(req_url) + if req_url + else fallback_template + ) + if status_code is not None: + recorder.record_ttfb(ttfb_dur, network_path="cloudpath") + err_type = _opentelemetry_metrics.classify_error_type( + exc, status_code=status_code + ) + else: + status_code = None + url_template = fallback_template + err_type = _opentelemetry_metrics.classify_error_type(exc) + recorder.record_http_attempt( + duration=duration, + http_method=http_method, + url_template=url_template, + status_code=status_code, + error_type=err_type, + network_path="cloudpath", + ) + raise + + duration = time.perf_counter() - attempt_start + stashed = _LAST_ATTEMPT_RESPONSE.get() + _LAST_ATTEMPT_RESPONSE.set(None) + status_code = 200 + url_template = fallback_template + if stashed is not None: + resp, ttfb_dur, req_url = stashed + raw_sc = getattr(resp, "status_code", None) + status_code = ( + raw_sc + if (isinstance(raw_sc, int) and not isinstance(raw_sc, bool)) + else None + ) + if req_url: + url_template = _opentelemetry_metrics.normalize_url_template( + req_url + ) + if status_code is not None: + recorder.record_ttfb(ttfb_dur, network_path="cloudpath") + else: + recorder.record_ttfb(duration, network_path="cloudpath") + + recorder.record_http_attempt( + duration=duration, + http_method=http_method, + url_template=url_template, + status_code=status_code, + error_type="OK", + network_path="cloudpath", + ) + return result + + call = _timed_attempt + else: + call = base_call + with create_trace_span( name="Storage.Connection.api_request", attributes=span_attributes, @@ -119,6 +278,7 @@ def api_request(self, *args, **kwargs): pass if retry: call = retry(call) + in_api_token = _IN_API_REQUEST.set(True) if recorder is not None else None try: return call() except (NotFound, api_exceptions.NotFound): @@ -140,3 +300,10 @@ def api_request(self, *args, **kwargs): f"Failed cache.check_and_evict on 404 in api_request: {e}" ) raise + finally: + if in_api_token is not None: + _IN_API_REQUEST.reset(in_api_token) + _LAST_ATTEMPT_RESPONSE.set(None) + current_op = _opentelemetry_metrics.get_current_operation() + if current_op is not None: + current_op.last_failure_error_type = None diff --git a/packages/google-cloud-storage/google/cloud/storage/_media/requests/_request_helpers.py b/packages/google-cloud-storage/google/cloud/storage/_media/requests/_request_helpers.py index 604ffc31375e..099ae1db9093 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_media/requests/_request_helpers.py +++ b/packages/google-cloud-storage/google/cloud/storage/_media/requests/_request_helpers.py @@ -17,6 +17,13 @@ This utilities are explicitly catered to ``requests``-like transports. """ +import time +from contextvars import ContextVar + +from google.cloud.storage import _helpers as _storage_helpers +from google.cloud.storage import _opentelemetry_metrics +from google.cloud.storage.exceptions import InvalidResponse + _SINGLE_GET_CHUNK_SIZE = 8192 # The number of seconds to wait to establish a connection # (connect() call on socket). Avoid setting this to a multiple of 3 to not @@ -26,6 +33,81 @@ _DEFAULT_READ_TIMEOUT = 60 +class _MediaAttemptState(object): + """Tracks timing and HTTP metadata for a single physical media attempt.""" + + def __init__(self, recorder, method, fallback_url, streaming=False): + self.recorder = recorder + self.method = method + self.url = fallback_url + self.streaming = streaming + self.attempt_start = time.perf_counter() + self.header_duration = None + self.response = None + self.ttfb_recorded = False + + +_MEDIA_ATTEMPT_STATE = ContextVar("_gcs_media_attempt_state", default=None) + + +def resolve_transport_recorder(transport): + """Resolves the active ``_MetricsRecorder`` for a media ``transport``.""" + if _storage_helpers._SUPPRESS_INTERNAL_METRICS.get(): + return None + recorder = _storage_helpers._ACTIVE_METRICS_RECORDER.get() + if recorder is not None and getattr(recorder, "standard_enabled", False) is True: + return recorder + recorder = getattr(transport, "_gcs_metrics_recorder", None) + if recorder is not None and getattr(recorder, "standard_enabled", False) is True: + return recorder + return None + + +def record_streaming_ttfb(): + """Records TTFB upon receipt of the first streamed payload chunk.""" + state = _MEDIA_ATTEMPT_STATE.get() + if ( + state is not None + and not state.ttfb_recorded + and _extract_status_code(state.response) is not None + ): + state.ttfb_recorded = True + state.recorder.record_ttfb( + time.perf_counter() - state.attempt_start, + network_path="cloudpath", + ) + + +def transport_request(transport, method, url, **kwargs): + """Executes ``transport.request`` while capturing per-attempt timing metadata.""" + state = _MEDIA_ATTEMPT_STATE.get() + if state is None: + return transport.request(method, url, **kwargs) + + state.method = method + state.url = url + state.attempt_start = time.perf_counter() + response = transport.request(method, url, **kwargs) + state.header_duration = time.perf_counter() - state.attempt_start + state.response = response + if ( + not state.streaming + and not state.ttfb_recorded + and _extract_status_code(response) is not None + ): + state.ttfb_recorded = True + state.recorder.record_ttfb(state.header_duration, network_path="cloudpath") + return response + + +def _extract_status_code(response): + """Returns an integer HTTP status code from ``response`` if valid.""" + raw_sc = getattr(response, "status_code", None) + if isinstance(raw_sc, int) and not isinstance(raw_sc, bool): + return raw_sc + return None + + class RequestsMixin(object): """Mix-in class implementing ``requests``-specific behavior. @@ -68,6 +150,18 @@ def _get_body(response): Returns: bytes: The body of the ``response``. """ + if getattr(response, "_content", None) is False: + chunks = [] + for chunk in response.iter_content( + chunk_size=_SINGLE_GET_CHUNK_SIZE, decode_unicode=False + ): + record_streaming_ttfb() + chunks.append(chunk) + record_streaming_ttfb() + response._content = b"".join(chunks) + response._content_consumed = True + else: + record_streaming_ttfb() return response.content @@ -83,14 +177,28 @@ def _get_body(response): bytes: The body of the ``response``. """ if response._content is False: - response._content = b"".join( - response.raw.stream(_SINGLE_GET_CHUNK_SIZE, decode_content=False) - ) + chunks = [] + for chunk in response.raw.stream( + _SINGLE_GET_CHUNK_SIZE, decode_content=False + ): + record_streaming_ttfb() + chunks.append(chunk) + record_streaming_ttfb() + response._content = b"".join(chunks) response._content_consumed = True + else: + record_streaming_ttfb() return response._content -def wait_and_retry(func, retry_strategy): +def wait_and_retry( + func, + retry_strategy, + transport=None, + method=None, + url=None, + streaming=False, +): """Attempts to retry a call to ``func`` until success. Args: @@ -98,10 +206,90 @@ def wait_and_retry(func, retry_strategy): an HTTP response which will be checked as retry-able. retry_strategy (Optional[google.api_core.retry.Retry]): The strategy to use if the request fails and must be retried. + transport (Optional[object]): Transport object making the request. + method (Optional[str]): HTTP request method. + url (Optional[str]): Request URL or fallback URL for route normalization. + streaming (bool): Whether TTFB should be deferred to streamed body read. Returns: object: The return value of ``func``. """ - if retry_strategy: - func = retry_strategy(func) - return func() + recorder = resolve_transport_recorder(transport) + if recorder is not None: + base_func = func + + def _timed_attempt(): + state = _MediaAttemptState( + recorder=recorder, + method=method, + fallback_url=url, + streaming=streaming, + ) + token = _MEDIA_ATTEMPT_STATE.set(state) + try: + result = base_func() + except BaseException as exc: + duration = time.perf_counter() - state.attempt_start + resp = state.response + if resp is None: + resp = getattr(exc, "response", None) + status_code = _extract_status_code(resp) + if ( + status_code is not None + and state.response is not None + and not state.ttfb_recorded + and isinstance(exc, InvalidResponse) + and state.header_duration is not None + ): + state.ttfb_recorded = True + recorder.record_ttfb( + state.header_duration, network_path="cloudpath" + ) + recorder.record_http_attempt( + duration=duration, + http_method=state.method or "UNKNOWN", + url_template=_opentelemetry_metrics.normalize_url_template( + state.url + ), + status_code=status_code, + error_type=_opentelemetry_metrics.classify_error_type( + exc, status_code=status_code + ), + network_path="cloudpath", + ) + raise + finally: + _MEDIA_ATTEMPT_STATE.reset(token) + + duration = time.perf_counter() - state.attempt_start + resp = state.response if state.response is not None else result + status_code = _extract_status_code(resp) + if not state.ttfb_recorded and status_code is not None: + ttfb_dur = ( + state.header_duration + if state.header_duration is not None + else duration + ) + state.ttfb_recorded = True + recorder.record_ttfb(ttfb_dur, network_path="cloudpath") + recorder.record_http_attempt( + duration=duration, + http_method=state.method or "UNKNOWN", + url_template=_opentelemetry_metrics.normalize_url_template(state.url), + status_code=status_code, + error_type="OK", + network_path="cloudpath", + ) + return result + + func = _timed_attempt + + try: + if retry_strategy: + func = retry_strategy(func) + return func() + finally: + if recorder is not None: + current_op = _opentelemetry_metrics.get_current_operation() + if current_op is not None: + current_op.last_failure_error_type = None diff --git a/packages/google-cloud-storage/google/cloud/storage/_media/requests/download.py b/packages/google-cloud-storage/google/cloud/storage/_media/requests/download.py index c3907083af5a..33857bf43dcb 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_media/requests/download.py +++ b/packages/google-cloud-storage/google/cloud/storage/_media/requests/download.py @@ -136,7 +136,18 @@ def _write_to_stream(self, response): # This is useful for smaller files, or when the user wants to # download the entire file in one go. if self.single_shot_download: - content = response.raw.read(decode_content=True) + if _request_helpers._MEDIA_ATTEMPT_STATE.get() is not None: + first = response.raw.read( + _request_helpers._SINGLE_GET_CHUNK_SIZE, + decode_content=True, + ) + if first: + _request_helpers.record_streaming_ttfb() + rest = response.raw.read(decode_content=True) + _request_helpers.record_streaming_ttfb() + content = first + rest + else: + content = response.raw.read(decode_content=True) self._stream.write(content) self._bytes_downloaded += len(content) local_checksum_object.update(content) @@ -147,9 +158,11 @@ def _write_to_stream(self, response): decode_unicode=False, ) for chunk in body_iter: + _request_helpers.record_streaming_ttfb() self._stream.write(chunk) self._bytes_downloaded += len(chunk) local_checksum_object.update(chunk) + _request_helpers.record_streaming_ttfb() # Don't validate the checksum for partial responses. if ( @@ -250,7 +263,9 @@ def retriable_request(): query_param = {"generation": self._object_generation} url = _helpers.add_query_parameters(self.media_url, query_param) - result = transport.request(method, url, **request_kwargs) + result = _request_helpers.transport_request( + transport, method, url, **request_kwargs + ) # If a generation hasn't been specified, and this is the first response we get, let's record the # generation. In future requests we'll specify the generation query param to avoid data races. @@ -277,7 +292,14 @@ def retriable_request(): return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=self.media_url, + streaming=self._stream is not None, + ) class RawDownload(_request_helpers.RawRequestsMixin, _download.Download): @@ -359,7 +381,15 @@ def _write_to_stream(self, response): # This is useful for smaller files, or when the user wants to # download the entire file in one go. if self.single_shot_download: - content = response.raw.read() + if _request_helpers._MEDIA_ATTEMPT_STATE.get() is not None: + first = response.raw.read(_request_helpers._SINGLE_GET_CHUNK_SIZE) + if first: + _request_helpers.record_streaming_ttfb() + rest = response.raw.read() + _request_helpers.record_streaming_ttfb() + content = first + rest + else: + content = response.raw.read() self._stream.write(content) self._bytes_downloaded += len(content) checksum_object.update(content) @@ -368,9 +398,11 @@ def _write_to_stream(self, response): _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False ) for chunk in body_iter: + _request_helpers.record_streaming_ttfb() self._stream.write(chunk) self._bytes_downloaded += len(chunk) checksum_object.update(chunk) + _request_helpers.record_streaming_ttfb() response._content_consumed = True # Don't validate the checksum for partial responses. @@ -472,7 +504,9 @@ def retriable_request(): query_param = {"generation": self._object_generation} url = _helpers.add_query_parameters(self.media_url, query_param) - result = transport.request(method, url, **request_kwargs) + result = _request_helpers.transport_request( + transport, method, url, **request_kwargs + ) # If a generation hasn't been specified, and this is the first response we get, let's record the # generation. In future requests we'll specify the generation query param to avoid data races. @@ -499,7 +533,14 @@ def retriable_request(): return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=self.media_url, + streaming=self._stream is not None, + ) class ChunkedDownload(_request_helpers.RequestsMixin, _download.ChunkedDownload): @@ -566,21 +607,39 @@ def consume_next_chunk( ValueError: If the current download has finished. """ method, url, payload, headers = self._prepare_request() + metrics_active = ( + _request_helpers.resolve_transport_recorder(transport) is not None + ) + extra_req_kwargs = {"stream": True} if metrics_active else {} # Wrap the request business logic in a function to be retried. def retriable_request(): # NOTE: We assume "payload is None" but pass it along anyway. - result = transport.request( + result = _request_helpers.transport_request( + transport, method, url, data=payload, headers=headers, timeout=timeout, + **extra_req_kwargs, ) - self._process_response(result) + try: + self._process_response(result) + except BaseException: + if metrics_active: + result.close() + raise return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=url, + streaming=metrics_active, + ) class RawChunkedDownload(_request_helpers.RawRequestsMixin, _download.ChunkedDownload): @@ -651,7 +710,8 @@ def consume_next_chunk( # Wrap the request business logic in a function to be retried. def retriable_request(): # NOTE: We assume "payload is None" but pass it along anyway. - result = transport.request( + result = _request_helpers.transport_request( + transport, method, url, data=payload, @@ -662,7 +722,14 @@ def retriable_request(): self._process_response(result) return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=url, + streaming=True, + ) def _add_decoder(response_raw, checksum): diff --git a/packages/google-cloud-storage/google/cloud/storage/_media/requests/upload.py b/packages/google-cloud-storage/google/cloud/storage/_media/requests/upload.py index dd7e3d4bbb81..fe2f5365e72b 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_media/requests/upload.py +++ b/packages/google-cloud-storage/google/cloud/storage/_media/requests/upload.py @@ -70,15 +70,26 @@ def transmit( # Wrap the request business logic in a function to be retried. def retriable_request(): - result = transport.request( - method, url, data=payload, headers=headers, timeout=timeout + result = _request_helpers.transport_request( + transport, + method, + url, + data=payload, + headers=headers, + timeout=timeout, ) self._process_response(result) return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=url, + ) class MultipartUpload(_request_helpers.RequestsMixin, _upload.MultipartUpload): @@ -149,15 +160,26 @@ def transmit( # Wrap the request business logic in a function to be retried. def retriable_request(): - result = transport.request( - method, url, data=payload, headers=headers, timeout=timeout + result = _request_helpers.transport_request( + transport, + method, + url, + data=payload, + headers=headers, + timeout=timeout, ) self._process_response(result) return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=url, + ) class ResumableUpload(_request_helpers.RequestsMixin, _upload.ResumableUpload): @@ -423,15 +445,26 @@ def initiate( # Wrap the request business logic in a function to be retried. def retriable_request(): - result = transport.request( - method, url, data=payload, headers=headers, timeout=timeout + result = _request_helpers.transport_request( + transport, + method, + url, + data=payload, + headers=headers, + timeout=timeout, ) self._process_initiate_response(result) return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=url, + ) def transmit_next_chunk( self, @@ -516,15 +549,26 @@ def transmit_next_chunk( # Wrap the request business logic in a function to be retried. def retriable_request(): - result = transport.request( - method, url, data=payload, headers=headers, timeout=timeout + result = _request_helpers.transport_request( + transport, + method, + url, + data=payload, + headers=headers, + timeout=timeout, ) self._process_resumable_response(result, len(payload)) return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=url, + ) def recover(self, transport): """Recover from a failure and check the status of the current upload. @@ -554,15 +598,26 @@ def recover(self, transport): # Wrap the request business logic in a function to be retried. def retriable_request(): - result = transport.request( - method, url, data=payload, headers=headers, timeout=timeout + result = _request_helpers.transport_request( + transport, + method, + url, + data=payload, + headers=headers, + timeout=timeout, ) self._process_recover_response(result) return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=url, + ) class XMLMPUContainer(_request_helpers.RequestsMixin, _upload.XMLMPUContainer): @@ -641,15 +696,26 @@ def initiate( # Wrap the request business logic in a function to be retried. def retriable_request(): - result = transport.request( - method, url, data=payload, headers=headers, timeout=timeout + result = _request_helpers.transport_request( + transport, + method, + url, + data=payload, + headers=headers, + timeout=timeout, ) self._process_initiate_response(result) return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=url, + ) def finalize( self, @@ -679,15 +745,26 @@ def finalize( # Wrap the request business logic in a function to be retried. def retriable_request(): - result = transport.request( - method, url, data=payload, headers=headers, timeout=timeout + result = _request_helpers.transport_request( + transport, + method, + url, + data=payload, + headers=headers, + timeout=timeout, ) self._process_finalize_response(result) return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=url, + ) def cancel( self, @@ -719,15 +796,26 @@ def cancel( # Wrap the request business logic in a function to be retried. def retriable_request(): - result = transport.request( - method, url, data=payload, headers=headers, timeout=timeout + result = _request_helpers.transport_request( + transport, + method, + url, + data=payload, + headers=headers, + timeout=timeout, ) self._process_cancel_response(result) return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=url, + ) class XMLMPUPart(_request_helpers.RequestsMixin, _upload.XMLMPUPart): @@ -767,12 +855,23 @@ def upload( # Wrap the request business logic in a function to be retried. def retriable_request(): - result = transport.request( - method, url, data=payload, headers=headers, timeout=timeout + result = _request_helpers.transport_request( + transport, + method, + url, + data=payload, + headers=headers, + timeout=timeout, ) self._process_upload_response(result) return result - return _request_helpers.wait_and_retry(retriable_request, self._retry_strategy) + return _request_helpers.wait_and_retry( + retriable_request, + self._retry_strategy, + transport=transport, + method=method, + url=url, + ) diff --git a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py index 8592cc24c65d..f511eff47eea 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py +++ b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py @@ -14,9 +14,31 @@ """Manages OpenTelemetry metrics instruments and gating for GCS client.""" +import asyncio +import concurrent.futures +import contextlib +import http.client import logging import os -from typing import Any, Dict, Optional +import re +import socket +import ssl +import time +from collections import deque +from contextvars import ContextVar +from dataclasses import dataclass, field +from typing import ( + Any, + Deque, + Dict, + FrozenSet, + Iterator, + List, + Optional, + Set, + Tuple, +) +from urllib.parse import urlsplit from google.cloud.storage.version import __version__ @@ -38,14 +60,43 @@ _DEFAULT_ENABLE_METRICS = False _DEFAULT_ENABLE_DEBUG_METRICS = False +DEFAULT_SERVER_ADDRESS = "storage.googleapis.com" + +# --------------------------------------------------------------------------- +# 3. Standard Metric Instrument Names +# --------------------------------------------------------------------------- +METRIC_RPC_CLIENT_CALL_DURATION = "rpc.client.call.duration" +METRIC_HTTP_CLIENT_REQUEST_DURATION = "http.client.request.duration" +METRIC_GCP_CLIENT_REQUEST_DURATION = "gcp.client.request.duration" +METRIC_STORAGE_CLIENT_OPERATIONS = "gcp.storage.client.operations" +METRIC_STORAGE_CLIENT_ATTEMPTS = "gcp.storage.client.attempts" +METRIC_STORAGE_CLIENT_RETRIES = "gcp.storage.client.retries" +METRIC_STORAGE_REQUEST_BODY_SIZE = "gcp.storage.client.request.body.size" +METRIC_STORAGE_RESPONSE_BODY_SIZE = "gcp.storage.client.response.body.size" +METRIC_STORAGE_CLIENT_TTFB = "gcp.storage.client.ttfb" +METRIC_STORAGE_CLIENT_ERRORS = "gcp.storage.client.errors" + +_LATENCY_METRIC_NAMES = ( + METRIC_RPC_CLIENT_CALL_DURATION, + METRIC_HTTP_CLIENT_REQUEST_DURATION, + METRIC_GCP_CLIENT_REQUEST_DURATION, + METRIC_STORAGE_CLIENT_TTFB, +) + +_SIZE_METRIC_NAMES = ( + METRIC_STORAGE_REQUEST_BODY_SIZE, + METRIC_STORAGE_RESPONSE_BODY_SIZE, +) + # --------------------------------------------------------------------------- -# 3. Optional OpenTelemetry Dependency Check +# 4. Optional OpenTelemetry Dependency Check # --------------------------------------------------------------------------- try: from opentelemetry import metrics HAS_OPENTELEMETRY_METRICS = True except ImportError: + metrics = None # type: ignore[assignment] HAS_OPENTELEMETRY_METRICS = False logger.debug( "OpenTelemetry metrics package (opentelemetry-api >= 1.12.0) is not " @@ -134,12 +185,13 @@ def is_debug_metrics_enabled( # --------------------------------------------------------------------------- -# 4. Standard Common Attributes & Meter Provider +# 5. Standard Common Attributes & Meter Provider # --------------------------------------------------------------------------- +_METER_NAME = "google.cloud.storage" + _COMMON_ATTRIBUTES: Dict[str, Any] = { "gcp.client.service": "storage", "gcp.client.version": __version__, - "gcp.client.repo": "googleapis/google-cloud-python", "gcp.client.artifact": "google-cloud-storage", } @@ -168,13 +220,1192 @@ def get_meter( Optional[Any]: The OpenTelemetry Meter instance if standard or debug metrics are enabled, or None otherwise. """ - if not ( - is_metrics_enabled(enable_metrics) - or is_debug_metrics_enabled(enable_debug_metrics) + if ( + not ( + is_metrics_enabled(enable_metrics) + or is_debug_metrics_enabled(enable_debug_metrics) + ) + or metrics is None ): return None return metrics.get_meter( - "google.cloud.storage", + _METER_NAME, __version__, meter_provider=meter_provider, ) + + +# --------------------------------------------------------------------------- +# 6. Custom Histogram Bucket Boundaries & Views +# --------------------------------------------------------------------------- +def _build_latency_buckets() -> Tuple[float, ...]: + """Builds the custom latency histogram bucket boundaries in seconds. + + Uses 2ms resolution for the first 50 buckets (0.000s to 0.098s), then starts + at 0.100s with 10ms resolution and doubles the increment every 10 buckets + while boundary < 300.0s. + """ + boundaries: List[float] = [] + boundary = 0.0 + increment = 0.002 + + for _ in range(50): + boundaries.append(round(boundary, 6)) + boundary += increment + + increment = 0.010 + i = 0 + while boundary < 300.0: + if i != 0 and i % 10 == 0: + increment *= 2 + boundaries.append(round(boundary, 6)) + boundary += increment + i += 1 + + return tuple(boundaries) + + +def _build_size_buckets() -> Tuple[int, ...]: + """Builds the custom byte-size histogram bucket boundaries in bytes. + + Uses 128 KiB increments up to 4 MiB, then doubles the increment on each + subsequent bucket while ``boundary <= 16 GiB`` (48 total buckets, matching + Go and Java SDKs). + """ + kb = 1024 + mb = 1024 * kb + gb = 1024 * mb + + boundaries: List[int] = [] + boundary = 0 + increment = 128 * kb + + while boundary <= 16 * gb: + boundaries.append(boundary) + boundary += increment + if boundary >= 4 * mb: + increment *= 2 + + return tuple(boundaries) + + +LATENCY_HISTOGRAM_BOUNDARIES: Tuple[float, ...] = _build_latency_buckets() +SIZE_HISTOGRAM_BOUNDARIES: Tuple[int, ...] = _build_size_buckets() + + +def get_histogram_views() -> List[Any]: + """Returns OpenTelemetry SDK Views for custom latency and size histograms. + + If ``opentelemetry-sdk`` is not installed, returns an empty list. + """ + try: + from opentelemetry.sdk.metrics.view import ( + ExplicitBucketHistogramAggregation, + View, + ) + except ImportError: + return [] + + views: List[Any] = [] + for name in _LATENCY_METRIC_NAMES: + views.append( + View( + instrument_name=name, + meter_name=_METER_NAME, + aggregation=ExplicitBucketHistogramAggregation( + boundaries=LATENCY_HISTOGRAM_BOUNDARIES + ), + ) + ) + for name in _SIZE_METRIC_NAMES: + views.append( + View( + instrument_name=name, + meter_name=_METER_NAME, + aggregation=ExplicitBucketHistogramAggregation( + boundaries=SIZE_HISTOGRAM_BOUNDARIES + ), + ) + ) + return views + + +# --------------------------------------------------------------------------- +# 7. Canonical Error Classification (`error.type`) +# --------------------------------------------------------------------------- +_HTTP_STATUS_TO_ERROR_TYPE: Dict[int, str] = { + 400: "INVALID_ARGUMENT", + 401: "UNAUTHENTICATED", + 403: "PERMISSION_DENIED", + 404: "NOT_FOUND", + 408: "DEADLINE_EXCEEDED", + 409: "ABORTED", + 412: "FAILED_PRECONDITION", + 416: "OUT_OF_RANGE", + 429: "RESOURCE_EXHAUSTED", + 499: "CANCELLED", + 500: "INTERNAL", + 501: "UNIMPLEMENTED", + 503: "UNAVAILABLE", + 504: "DEADLINE_EXCEEDED", +} + + +def _is_valid_status_code(val: Any) -> bool: + """Returns True if ``val`` is a valid HTTP error status code (int in 400..599).""" + return isinstance(val, int) and not isinstance(val, bool) and 400 <= val < 600 + + +def _iter_exception_chain(exc: BaseException) -> Iterator[BaseException]: + """Yields ``exc`` and any nested causes, contexts, reasons, or exception args.""" + visited: Set[int] = {id(exc)} + queue: Deque[BaseException] = deque([exc]) + while queue: + current = queue.popleft() + yield current + + for attr_name in ( + "__cause__", + "__context__", + "cause", + "reason", + "original_exception", + ): + nested = getattr(current, attr_name, None) + if isinstance(nested, BaseException) and id(nested) not in visited: + visited.add(id(nested)) + queue.append(nested) + + for seq_attr in ("args", "errors"): + seq = getattr(current, seq_attr, ()) + if isinstance(seq, (tuple, list)): + for item in seq: + if isinstance(item, BaseException) and id(item) not in visited: + visited.add(id(item)) + queue.append(item) + + +def _is_subclass_by_module( + cls: type, module_prefix: str, class_names: Set[str] +) -> bool: + """Checks whether ``cls`` or any of its base classes match module/name criteria.""" + mro = getattr(cls, "__mro__", None) + if isinstance(mro, tuple): + for base in mro: + mod = getattr(base, "__module__", "") or "" + if mod == module_prefix or mod.startswith(module_prefix + "."): + if getattr(base, "__name__", None) in class_names: + return True + return False + + +def classify_error_type( + exc: Optional[BaseException] = None, + status_code: Optional[int] = None, +) -> str: + """Maps an exception or HTTP status code to a low-cardinality ``error.type``. + + Args: + exc: Optional exception raised during an operation or transport attempt. + status_code: Optional HTTP status code from the response. + + Returns: + str: Canonical low-cardinality error classification string. + """ + valid_status = _is_valid_status_code(status_code) + if exc is None: + if not valid_status or status_code is None: + return "OK" + return _HTTP_STATUS_TO_ERROR_TYPE.get(status_code, str(status_code)) + + chain = list(_iter_exception_chain(exc)) + + # 1. Checksum Mismatch (DataCorruption) + for item in chain: + if _is_subclass_by_module( + item.__class__, "google.cloud.storage", {"DataCorruption"} + ) or _is_subclass_by_module( + item.__class__, "google.resumable_media", {"DataCorruption"} + ): + return "CHECKSUM_MISMATCH" + + # 2. Authentication / Credentials Failure (google.auth.exceptions) + for item in chain: + if _is_subclass_by_module( + item.__class__, + "google.auth.exceptions", + { + "RefreshError", + "TransportError", + "DefaultCredentialsError", + "GoogleAuthError", + }, + ): + return "AUTHENTICATION_ERROR" + + # 3. Cancellation + for item in chain: + if isinstance( + item, (asyncio.CancelledError, concurrent.futures.CancelledError) + ): + return "CANCELLED" + + # 4. HTTP response status code on exception (checked before grpc_status_code so + # HTTP 409 Conflict consistently maps to ABORTED across HTTP and Veneer metrics) + for item in chain: + resp = getattr(item, "response", None) + resp_status = getattr(resp, "status_code", None) + if _is_valid_status_code(resp_status): + return _HTTP_STATUS_TO_ERROR_TYPE.get(resp_status, str(resp_status)) + + if item.__class__.__name__ == "Conflict" and _is_subclass_by_module( + item.__class__, "google.api_core.exceptions", {"Conflict"} + ): + return _HTTP_STATUS_TO_ERROR_TYPE[409] + + # 5. gRPC / GoogleAPICallError status code + for item in chain: + grpc_code = getattr(item, "grpc_status_code", None) + if grpc_code is not None: + name = getattr(grpc_code, "name", None) + if name: + return str(name) + return str(grpc_code) + + # Direct grpc.RpcError (has callable .code()) + code_attr = getattr(item, "code", None) + if callable(code_attr): + try: + rpc_status = code_attr() + name = getattr(rpc_status, "name", None) + if isinstance(name, str) and name: + return name + except Exception: # pragma: NO COVER + pass + + if _is_valid_status_code(code_attr): + return _HTTP_STATUS_TO_ERROR_TYPE.get(code_attr, str(code_attr)) + + # 6. TLS / SSL Failure + for item in chain: + if ( + isinstance(item, ssl.SSLError) + or _is_subclass_by_module( + item.__class__, + "requests.exceptions", + {"SSLError"}, + ) + or _is_subclass_by_module( + item.__class__, + "urllib3.exceptions", + {"SSLError"}, + ) + ): + return "TLS_FAILURE" + + # 7. Specific Socket / Network Failures (DNS, Connection Refused, Connection Reset) + for item in chain: + if isinstance(item, socket.gaierror) or _is_subclass_by_module( + item.__class__, + "urllib3.exceptions", + {"NameResolutionError"}, + ): + return "DNS_FAILURE" + if isinstance(item, (ConnectionRefusedError, ConnectionResetError)): + return "CONNECTION_ERROR" + + # 8. Timeouts (Note: urllib3.exceptions.NewConnectionError subclasses ConnectTimeoutError + # even for non-timeout connection failures, so exclude NewConnectionError here.) + for item in chain: + if item.__class__.__name__ != "NewConnectionError": + if isinstance( + item, + ( + socket.timeout, + TimeoutError, + asyncio.TimeoutError, + concurrent.futures.TimeoutError, + ), + ): + return "TIMEOUT" + if _is_subclass_by_module( + item.__class__, + "requests.exceptions", + {"Timeout", "ConnectTimeout", "ReadTimeout"}, + ) or _is_subclass_by_module( + item.__class__, + "urllib3.exceptions", + {"TimeoutError", "ReadTimeoutError", "ConnectTimeoutError"}, + ): + return "TIMEOUT" + + # 9. Generic Connection / Network Transport Failure + for item in chain: + if ( + _is_subclass_by_module( + item.__class__, + "requests.exceptions", + {"ConnectionError", "ChunkedEncodingError", "RequestException"}, + ) + or _is_subclass_by_module( + item.__class__, + "urllib3.exceptions", + {"HTTPError"}, + ) + or isinstance(item, (OSError, http.client.HTTPException)) + ): + return "CONNECTION_ERROR" + + # 10. Explicit HTTP status_code fallback + if valid_status and status_code is not None: + return _HTTP_STATUS_TO_ERROR_TYPE.get(status_code, str(status_code)) + + # 11. Client Validation Errors + for item in chain: + if ( + isinstance(item, (ValueError, TypeError)) + or _is_subclass_by_module( + item.__class__, "google.cloud.storage", {"InvalidResponse"} + ) + or _is_subclass_by_module( + item.__class__, "google.resumable_media", {"InvalidResponse"} + ) + ): + return "CLIENT_VALIDATION_ERROR" + + return "UNKNOWN" + + +# --------------------------------------------------------------------------- +# 8. HTTP `url.template` Route Normalization +# --------------------------------------------------------------------------- +_URL_TEMPLATE_PATTERNS: Tuple[Tuple[re.Pattern[str], str], ...] = ( + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/rewriteTo/b/[^/]+/o/.+$"), + "/storage/v1/b/{bucket}/o/{object}/rewriteTo/b/{bucket}/o/{object}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/copyTo/b/[^/]+/o/.+$"), + "/storage/v1/b/{bucket}/o/{object}/copyTo/b/{bucket}/o/{object}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/moveTo/o/.+$"), + "/storage/v1/b/{bucket}/o/{object}/moveTo/o/{object}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/compose$"), + "/storage/v1/b/{bucket}/o/{object}/compose", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/restore$"), + "/storage/v1/b/{bucket}/o/{object}/restore", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/acl/[^/]+$"), + "/storage/v1/b/{bucket}/o/{object}/acl/{entity}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/acl$"), + "/storage/v1/b/{bucket}/o/{object}/acl", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/iam(?:/.*)?$"), + "/storage/v1/b/{bucket}/o/{object}/iam", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/.+$"), + "/storage/v1/b/{bucket}/o/{object}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o$"), + "/storage/v1/b/{bucket}/o", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/acl/[^/]+$"), + "/storage/v1/b/{bucket}/acl/{entity}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/acl$"), + "/storage/v1/b/{bucket}/acl", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/defaultObjectAcl/[^/]+$"), + "/storage/v1/b/{bucket}/defaultObjectAcl/{entity}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/defaultObjectAcl$"), + "/storage/v1/b/{bucket}/defaultObjectAcl", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/iam(?:/.*)?$"), + "/storage/v1/b/{bucket}/iam", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/notificationConfigs/[^/]+$"), + "/storage/v1/b/{bucket}/notificationConfigs/{notification}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/notificationConfigs$"), + "/storage/v1/b/{bucket}/notificationConfigs", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/lockRetentionPolicy$"), + "/storage/v1/b/{bucket}/lockRetentionPolicy", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/restore$"), + "/storage/v1/b/{bucket}/restore", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+$"), + "/storage/v1/b/{bucket}", + ), + ( + re.compile(r"^/storage/v1/b$"), + "/storage/v1/b", + ), + ( + re.compile(r"^/storage/v1/projects/[^/]+/serviceAccount$"), + "/storage/v1/projects/{project}/serviceAccount", + ), + ( + re.compile(r"^/storage/v1/projects/[^/]+/hmacKeys/[^/]+$"), + "/storage/v1/projects/{project}/hmacKeys/{access_id}", + ), + ( + re.compile(r"^/storage/v1/projects/[^/]+/hmacKeys$"), + "/storage/v1/projects/{project}/hmacKeys", + ), + ( + re.compile(r"^/upload/storage/v1/b/[^/]+/o(?:/[^/]+)?$"), + "/upload/storage/v1/b/{bucket}/o", + ), + ( + re.compile(r"^/download/storage/v1/b/[^/]+/o/.+$"), + "/download/storage/v1/b/{bucket}/o/{object}", + ), + ( + re.compile(r"^/batch/storage/v1$"), + "/batch/storage/v1", + ), +) + +_XML_MPU_RESERVED_PREFIXES = {"storage", "upload", "download", "batch", "b", "projects"} +_XML_MPU_QUERY_KEYS = {"uploads", "uploadId", "partNumber"} +_XML_MULTI_SEGMENT_RE = re.compile(r"^/([^/]+)/(.+)$") + + +def normalize_url_template(url_or_path: Optional[str]) -> str: + """Normalizes a raw HTTP URL or path into a bounded ``url.template`` string. + + Bucket names, object keys, project IDs, and query strings are stripped to + prevent cardinality explosion. Unrecognized paths fall back to ``"UNKNOWN"``. + + Args: + url_or_path: Full request URL or path string. + + Returns: + str: Low-cardinality ``url.template`` value. + """ + if not url_or_path or not isinstance(url_or_path, str): + return "UNKNOWN" + + try: + parsed = urlsplit(url_or_path) + except ValueError: + return "UNKNOWN" + path = parsed.path or "" + query = parsed.query or "" + + path = "/" + path.strip("/") + if path == "/": + return "UNKNOWN" + + # Normalize relative paths passed to Connection.api_request before /storage/v1 prefixing + if path == "/b" or path.startswith(("/b/", "/projects/")): + path = "/storage/v1" + path + + for pattern, template in _URL_TEMPLATE_PATTERNS: + if pattern.match(path): + return template + + # XML API Multipart Upload routes: /{bucket}/{object} with XML MPU query params + if query: + query_keys = {param.partition("=")[0] for param in query.split("&") if param} + if query_keys & _XML_MPU_QUERY_KEYS: + match_multi = _XML_MULTI_SEGMENT_RE.match(path) + if match_multi and match_multi.group(1) not in _XML_MPU_RESERVED_PREFIXES: + return "/{bucket}/{object}" + + return "UNKNOWN" + + +# --------------------------------------------------------------------------- +# 9. Logical Operation Normalization, Context & Suppression (`ContextVar`) +# --------------------------------------------------------------------------- +_GRPC_LOGICAL_OPERATION_OVERRIDES: Dict[str, str] = { + "BidiReadObject": "ReadObject", + "BidiWriteObject": "WriteObject", +} + +_RPC_V2_METHODS: FrozenSet[str] = frozenset( + { + "BidiReadObject", + "BidiWriteObject", + "CancelResumableWrite", + "ComposeObject", + "CreateBucket", + "DeleteBucket", + "DeleteObject", + "GetBucket", + "GetIamPolicy", + "GetObject", + "ListBuckets", + "ListObjects", + "LockBucketRetentionPolicy", + "MoveObject", + "QueryWriteStatus", + "ReadObject", + "RestoreObject", + "RewriteObject", + "SetIamPolicy", + "StartResumableWrite", + "TestIamPermissions", + "UpdateBucket", + "UpdateObject", + "WriteObject", + } +) + + +def normalize_logical_operation(operation: Optional[str]) -> str: + """Normalizes an RPC path or method name into a short logical ``gcp.storage.operation``.""" + if not operation or not isinstance(operation, str): + return "UNKNOWN" + cleaned = operation.strip().strip("/") + if not cleaned: + return "UNKNOWN" + short_name = cleaned.rsplit("/", 1)[-1] if "/" in cleaned else cleaned + return _GRPC_LOGICAL_OPERATION_OVERRIDES.get(short_name, short_name) + + +def resolve_rpc_method( + operation: Optional[str], rpc_method: Optional[str] = None +) -> str: + """Resolves the full RPC path (e.g. ``google.storage.v2.Storage/ReadObject``) for ``rpc.method``.""" + raw = rpc_method if (rpc_method and isinstance(rpc_method, str)) else operation + if not raw or not isinstance(raw, str): + return "UNKNOWN" + cleaned = raw.strip().strip("/") + if not cleaned or cleaned == "UNKNOWN": + return "UNKNOWN" + if "/" in cleaned: + return cleaned + if cleaned in _RPC_V2_METHODS: + return f"google.storage.v2.Storage/{cleaned}" + if "." in cleaned: + return cleaned + return f"google.storage.v1.Storage/{cleaned}" + + +@dataclass +class _OperationState: + """Represents the active logical GCS Veneer operation in the current context.""" + + client_method: str + operation: str + rpc_method: str + suppressed: bool = False + start_time: float = field(default_factory=time.perf_counter) + finished: bool = False + last_failure_error_type: Optional[str] = None + recorded_attempts: int = 0 + + def mark_attempt_failed(self, error_type: str) -> None: + """Records the error type of a failed attempt for a subsequent retry.""" + if error_type and error_type != "OK": + self.last_failure_error_type = error_type + + def consume_retry_error(self) -> Optional[str]: + """Consumes and returns the prior failed attempt's error type, if any.""" + prior = self.last_failure_error_type + self.last_failure_error_type = None + return prior + + +_OP_STATE: ContextVar[Optional[_OperationState]] = ContextVar( + "_gcs_otel_op_state", default=None +) + + +def get_current_operation() -> Optional[_OperationState]: + """Returns the current logical `_OperationState` in this execution context, if any.""" + op = _OP_STATE.get() + if op is not None and op.finished: + return None + return op + + +@contextlib.contextmanager +def start_operation( + recorder: Optional["_MetricsRecorder"], + client_method: str, + operation: str, + suppress: bool = False, + rpc_method: Optional[str] = None, +) -> Iterator[Optional[_OperationState]]: + """Context manager tracking a logical user-facing GCS operation. + + If a parent logical operation is already active in the current context, + nested calls become a no-op so only the outermost user-invoked method + emits ``gcp.client.request.duration`` and ``gcp.storage.client.operations``. + + Args: + recorder: Optional `_MetricsRecorder` configured on the client. + client_method: Fully qualified Python method name + (e.g. ``"storage.Blob.download_to_filename"``). + operation: Short logical GCS operation name + (e.g. ``"ReadObject"``, ``"GetObject"``). + suppress: If True, suppresses logical operation and payload size + emission for this context and any nested calls (used by + ``transfer_manager`` workers and ``BlobReader`` prefetches). + rpc_method: Optional full RPC path + (e.g. ``"google.storage.v2.Storage/BidiReadObject"``). If omitted, + inferred from ``operation``. + """ + existing = get_current_operation() + if existing is not None: + if suppress and not existing.suppressed: + suppressed_child = _OperationState( + client_method=existing.client_method, + operation=existing.operation, + rpc_method=existing.rpc_method, + suppressed=True, + start_time=existing.start_time, + ) + child_token = _OP_STATE.set(suppressed_child) + try: + yield suppressed_child + finally: + suppressed_child.finished = True + _OP_STATE.reset(child_token) + else: + yield existing + return + + recorder_active = ( + recorder is not None and getattr(recorder, "standard_enabled", False) is True + ) + if not recorder_active and not suppress: + yield None + return + + resolved_operation = normalize_logical_operation(operation) + resolved_rpc_method = resolve_rpc_method( + resolved_operation, rpc_method=rpc_method or operation + ) + start_time = time.perf_counter() + state = _OperationState( + client_method=client_method, + operation=resolved_operation, + rpc_method=resolved_rpc_method, + suppressed=suppress, + start_time=start_time, + ) + token = _OP_STATE.set(state) + error_type = "OK" + try: + yield state + except BaseException as exc: + error_type = classify_error_type(exc) + raise + finally: + state.finished = True + duration = max(0.0, time.perf_counter() - start_time) + _OP_STATE.reset(token) + if recorder_active and not suppress and recorder is not None: + recorder.record_logical_operation( + duration=duration, + client_method=client_method, + operation=resolved_operation, + error_type=error_type, + rpc_method=resolved_rpc_method, + ) + + +def record_pending_retry( + recorder: Optional["_MetricsRecorder"], + operation: Optional[str] = None, + network_path: Optional[str] = None, +) -> None: + """Emits a pending ``gcp.storage.client.retries`` increment if a prior attempt failed.""" + if recorder is None or not getattr(recorder, "standard_enabled", False): + return + current_op = get_current_operation() + if current_op is None: + return + prior_error = current_op.consume_retry_error() + if prior_error is not None: + recorder.record_retry( + operation=operation or current_op.operation, + error_type=prior_error, + network_path=network_path, + ) + + +def handle_unrecorded_retryable_failure( + recorder: Optional["_MetricsRecorder"], + exc: BaseException, + attempts_before: int, + will_retry: bool, + network_path: Optional[str] = None, +) -> None: + """Tracks retry state when a failed attempt bypassed transport-level attempt recording.""" + if recorder is None or not getattr(recorder, "standard_enabled", False): + return + current_op = get_current_operation() + if current_op is None: + return + if current_op.recorded_attempts > attempts_before: + return + current_op.recorded_attempts += 1 + prior_error = current_op.consume_retry_error() + if prior_error is not None: + recorder.record_retry( + operation=current_op.operation, + error_type=prior_error, + network_path=network_path, + ) + if will_retry: + current_op.mark_attempt_failed(classify_error_type(exc)) + + +def get_active_recorder(obj: Any) -> Optional["_MetricsRecorder"]: + """Returns ``obj._metrics_recorder`` if it is an active ``_MetricsRecorder``.""" + recorder = getattr(obj, "_metrics_recorder", None) + if recorder is not None and getattr(recorder, "standard_enabled", False) is True: + return recorder + return None + + +# --------------------------------------------------------------------------- +# 10. Server Address Parsing & `_MetricsRecorder` +# --------------------------------------------------------------------------- +def _extract_server_address(api_endpoint: Optional[str]) -> str: + """Extracts the low-cardinality hostname from ``api_endpoint``.""" + if not api_endpoint or not isinstance(api_endpoint, str): + return DEFAULT_SERVER_ADDRESS + endpoint = api_endpoint.strip() + if not endpoint: + return DEFAULT_SERVER_ADDRESS + for grpc_prefix in ( + "dns:", + "google-c2p:", + "google-c2p-experimental:", + "xds:", + "ipv4:", + "ipv6:", + ): + if endpoint.startswith(grpc_prefix): + remainder = endpoint[len(grpc_prefix) :] + if remainder.startswith("//"): + remainder = remainder[2:] + remainder = ( + remainder.split("/", 1)[1] if "/" in remainder else remainder + ) + endpoint = remainder.lstrip("/") + break + if "://" not in endpoint: + endpoint = "https://" + endpoint.lstrip("/") + try: + parsed = urlsplit(endpoint) + return parsed.hostname or DEFAULT_SERVER_ADDRESS + except ValueError: + return DEFAULT_SERVER_ADDRESS + + +class _MetricsRecorder: + """Holds OpenTelemetry metric instruments and records standard GCS metrics.""" + + def __init__( + self, + enable_metrics: Optional[bool] = None, + enable_debug_metrics: Optional[bool] = None, + meter_provider: Optional[Any] = None, + api_endpoint: Optional[str] = None, + rpc_system: str = "http", + ) -> None: + self.standard_enabled: bool = is_metrics_enabled(enable_metrics) + self.debug_enabled: bool = is_debug_metrics_enabled(enable_debug_metrics) + self.enabled: bool = self.standard_enabled or self.debug_enabled + self._meter_provider: Optional[Any] = meter_provider + self.rpc_system: str = rpc_system + self.server_address: str = _extract_server_address(api_endpoint) + self.meter: Optional[Any] = get_meter( + enable_metrics=enable_metrics, + enable_debug_metrics=enable_debug_metrics, + meter_provider=meter_provider, + ) + + self._base_attributes: Dict[str, Any] = { + **_COMMON_ATTRIBUTES, + "server.address": self.server_address, + "rpc.system.name": self.rpc_system, + } + + if self.standard_enabled and self.meter is not None: + self.rpc_call_duration = self.meter.create_histogram( + METRIC_RPC_CLIENT_CALL_DURATION, + unit="s", + description="Duration of an individual gRPC attempt in seconds (excluding retries).", + ) + self.http_request_duration = self.meter.create_histogram( + METRIC_HTTP_CLIENT_REQUEST_DURATION, + unit="s", + description="Duration of an individual HTTP request attempt in seconds (excluding retries).", + ) + self.client_request_duration = self.meter.create_histogram( + METRIC_GCP_CLIENT_REQUEST_DURATION, + unit="s", + description="End-to-end latency of a logical user-facing Veneer operation in seconds.", + ) + self.operations_counter = self.meter.create_counter( + METRIC_STORAGE_CLIENT_OPERATIONS, + unit="{operation}", + description="Number of logical GCS client operations.", + ) + self.attempts_counter = self.meter.create_counter( + METRIC_STORAGE_CLIENT_ATTEMPTS, + unit="{attempt}", + description="Number of physical network transport attempts executed.", + ) + self.retries_counter = self.meter.create_counter( + METRIC_STORAGE_CLIENT_RETRIES, + unit="{retry}", + description="Number of transport retries executed after a failed attempt.", + ) + self.request_body_size = self.meter.create_histogram( + METRIC_STORAGE_REQUEST_BODY_SIZE, + unit="By", + description="Application payload bytes submitted by the caller for an upload or write operation.", + ) + self.response_body_size = self.meter.create_histogram( + METRIC_STORAGE_RESPONSE_BODY_SIZE, + unit="By", + description="Application payload bytes delivered to the caller.", + ) + self.ttfb = self.meter.create_histogram( + METRIC_STORAGE_CLIENT_TTFB, + unit="s", + description="Time elapsed from attempt initiation to receipt of the first response byte.", + ) + self.operation_ttfb = self.ttfb + self.errors_counter = self.meter.create_counter( + METRIC_STORAGE_CLIENT_ERRORS, + unit="{error}", + description="Number of physical network transport attempts terminating with a non-OK status.", + ) + else: + self.rpc_call_duration = None + self.http_request_duration = None + self.client_request_duration = None + self.operations_counter = None + self.attempts_counter = None + self.retries_counter = None + self.request_body_size = None + self.response_body_size = None + self.ttfb = None + self.operation_ttfb = None + self.errors_counter = None + + def get_base_attributes(self) -> Dict[str, Any]: + """Returns a copy of the base attributes for this recorder instance.""" + return self._base_attributes.copy() + + def _build_operation_attributes( + self, + operation: Optional[str] = None, + client_method: Optional[str] = None, + current_op: Optional[_OperationState] = None, + ) -> Dict[str, Any]: + """Builds base attributes enriched with current or explicit operation metadata.""" + attrs = self._base_attributes.copy() + if current_op is None: + current_op = get_current_operation() + raw_op = operation or (current_op.operation if current_op else None) + resolved_method = client_method or ( + current_op.client_method if current_op else None + ) + if raw_op is not None: + attrs["gcp.storage.operation"] = normalize_logical_operation(raw_op) + if resolved_method is not None: + attrs["gcp.client.method"] = resolved_method + return attrs + + def record_logical_operation( + self, + duration: float, + client_method: str, + operation: str, + error_type: str = "OK", + rpc_method: Optional[str] = None, + ) -> None: + """Records end-to-end logical operation duration and count.""" + if not self.standard_enabled or self.meter is None: + return + logical_op = normalize_logical_operation(operation) + full_rpc_method = resolve_rpc_method( + logical_op, rpc_method=rpc_method or operation + ) + op_attrs = { + **self._base_attributes, + "gcp.storage.operation": logical_op, + "gcp.client.method": client_method, + "error.type": error_type, + } + duration_attrs = { + **op_attrs, + "rpc.method": full_rpc_method, + } + self.client_request_duration.record( + max(0.0, duration), attributes=duration_attrs + ) + self.operations_counter.add(1, attributes=op_attrs) + + def record_retry( + self, + operation: Optional[str] = None, + error_type: str = "UNKNOWN", + network_path: Optional[str] = None, + ) -> None: + """Records a transport retry attempt with the prior failed attempt's error type.""" + if not self.standard_enabled or self.meter is None: + return + current_op = get_current_operation() + raw_op = operation or (current_op.operation if current_op else None) + retry_attrs: Dict[str, Any] = { + **self._base_attributes, + "error.type": error_type, + } + if raw_op is not None: + retry_attrs["gcp.storage.operation"] = normalize_logical_operation(raw_op) + if network_path is not None: + retry_attrs["network.path"] = network_path + self.retries_counter.add(1, attributes=retry_attrs) + + def record_http_attempt( + self, + duration: float, + http_method: str, + url_template: str, + status_code: Optional[int] = None, + error_type: str = "OK", + operation: Optional[str] = None, + client_method: Optional[str] = None, + network_path: Optional[str] = None, + ) -> None: + """Records a single physical HTTP request attempt.""" + if not self.standard_enabled or self.meter is None: + return + del client_method + current_op = get_current_operation() + raw_op = operation or (current_op.operation if current_op else None) + resolved_op = ( + normalize_logical_operation(raw_op) if raw_op is not None else None + ) + if current_op is not None: + current_op.recorded_attempts += 1 + prior_error = current_op.consume_retry_error() + if prior_error is not None: + self.record_retry( + operation=resolved_op, + error_type=prior_error, + network_path=network_path, + ) + if error_type != "OK": + current_op.mark_attempt_failed(error_type) + + valid_code = ( + isinstance(status_code, int) + and not isinstance(status_code, bool) + and 100 <= status_code < 600 + ) + + # 1. http.client.request.duration attributes + http_attrs: Dict[str, Any] = { + **self._base_attributes, + "http.request.method": http_method.upper() if http_method else "UNKNOWN", + "url.template": url_template, + "error.type": error_type, + } + if valid_code: + http_attrs["http.response.status_code"] = status_code + if network_path is not None: + http_attrs["network.path"] = network_path + + # 2. gcp.storage.client.attempts attributes + attempt_attrs: Dict[str, Any] = { + **self._base_attributes, + "error.type": error_type, + } + if resolved_op is not None: + attempt_attrs["gcp.storage.operation"] = resolved_op + if valid_code: + attempt_attrs["http.response.status_code"] = status_code + if network_path is not None: + attempt_attrs["network.path"] = network_path + + self.http_request_duration.record(max(0.0, duration), attributes=http_attrs) + self.attempts_counter.add(1, attributes=attempt_attrs) + + # 3. gcp.storage.client.errors attributes + if error_type != "OK": + error_attrs: Dict[str, Any] = { + **self._base_attributes, + "error.type": error_type, + } + if resolved_op is not None: + error_attrs["gcp.storage.operation"] = resolved_op + if network_path is not None: + error_attrs["network.path"] = network_path + self.errors_counter.add(1, attributes=error_attrs) + + def record_rpc_attempt( + self, + duration: float, + operation: Optional[str] = None, + status_code: Optional[str] = None, + error_type: str = "OK", + rpc_method: Optional[str] = None, + network_path: Optional[str] = None, + op_state: Optional[_OperationState] = None, + ) -> None: + """Records a single physical gRPC call attempt.""" + if not self.standard_enabled or self.meter is None: + return + active_op = op_state if op_state is not None else get_current_operation() + raw_rpc = ( + rpc_method or operation or (active_op.rpc_method if active_op else None) + ) + resolved_rpc_method = resolve_rpc_method(operation, rpc_method=raw_rpc) + raw_op = operation or rpc_method or (active_op.operation if active_op else None) + resolved_op = normalize_logical_operation(raw_op) + + if active_op is not None: + active_op.recorded_attempts += 1 + prior_error = active_op.consume_retry_error() + if prior_error is not None: + self.record_retry( + operation=resolved_op, + error_type=prior_error, + network_path=network_path, + ) + if error_type != "OK": + active_op.mark_attempt_failed(error_type) + + # 1. rpc.client.call.duration attributes + rpc_attrs: Dict[str, Any] = { + **self._base_attributes, + "rpc.method": resolved_rpc_method, + "error.type": error_type, + } + if status_code is not None: + rpc_attrs["rpc.response.status_code"] = status_code + if network_path is not None: + rpc_attrs["network.path"] = network_path + + # 2. gcp.storage.client.attempts attributes + attempt_attrs: Dict[str, Any] = { + **self._base_attributes, + "gcp.storage.operation": resolved_op, + "error.type": error_type, + } + if status_code is not None: + attempt_attrs["rpc.response.status_code"] = status_code + if network_path is not None: + attempt_attrs["network.path"] = network_path + + self.rpc_call_duration.record(max(0.0, duration), attributes=rpc_attrs) + self.attempts_counter.add(1, attributes=attempt_attrs) + + # 3. gcp.storage.client.errors attributes + if error_type != "OK": + error_attrs: Dict[str, Any] = { + **self._base_attributes, + "gcp.storage.operation": resolved_op, + "error.type": error_type, + } + if network_path is not None: + error_attrs["network.path"] = network_path + self.errors_counter.add(1, attributes=error_attrs) + + def record_request_body_size( + self, + size: int, + operation: Optional[str] = None, + client_method: Optional[str] = None, + ) -> None: + """Records the payload byte size of an upload or write operation.""" + if not self.standard_enabled or self.meter is None: + return + if not isinstance(size, int) or isinstance(size, bool): + return + current_op = get_current_operation() + if current_op is not None and current_op.suppressed: + return + attrs = self._build_operation_attributes( + operation=operation, client_method=client_method, current_op=current_op + ) + self.request_body_size.record(max(0, size), attributes=attrs) + + def record_response_body_size( + self, + size: int, + operation: Optional[str] = None, + client_method: Optional[str] = None, + ) -> None: + """Records the payload byte size of a download or read operation.""" + if not self.standard_enabled or self.meter is None: + return + if not isinstance(size, int) or isinstance(size, bool): + return + current_op = get_current_operation() + if current_op is not None and current_op.suppressed: + return + attrs = self._build_operation_attributes( + operation=operation, client_method=client_method, current_op=current_op + ) + self.response_body_size.record(max(0, size), attributes=attrs) + + def record_ttfb( + self, + duration: float, + operation: Optional[str] = None, + network_path: Optional[str] = None, + ) -> None: + """Records time to first response byte (TTFB) in seconds.""" + if not self.standard_enabled or self.meter is None: + return + current_op = get_current_operation() + if current_op is not None and current_op.suppressed: + return + raw_op = operation or (current_op.operation if current_op else None) + attrs: Dict[str, Any] = self._base_attributes.copy() + if raw_op is not None: + attrs["gcp.storage.operation"] = normalize_logical_operation(raw_op) + if network_path is not None: + attrs["network.path"] = network_path + self.ttfb.record(max(0.0, duration), attributes=attrs) + + def shutdown(self) -> None: + """Flushes pending metrics on a custom MeterProvider if configured.""" + if self._meter_provider is not None and hasattr( + self._meter_provider, "force_flush" + ): + try: + self._meter_provider.force_flush() + except Exception as exc: + logger.debug( + "Failed to force_flush custom MeterProvider during client shutdown: %s", + exc, + ) diff --git a/packages/google-cloud-storage/google/cloud/storage/acl.py b/packages/google-cloud-storage/google/cloud/storage/acl.py index 038294b8f6fc..b5cab6ad9151 100644 --- a/packages/google-cloud-storage/google/cloud/storage/acl.py +++ b/packages/google-cloud-storage/google/cloud/storage/acl.py @@ -14,7 +14,14 @@ """Manage access to objects and buckets.""" -from google.cloud.storage._helpers import _add_generation_match_parameters +from contextlib import nullcontext + +from google.cloud.storage._helpers import ( + _add_generation_match_parameters, + _client_method_override, + _resolve_operation_client, + _start_storage_operation, +) from google.cloud.storage._opentelemetry_tracing import create_trace_span from google.cloud.storage.constants import _DEFAULT_TIMEOUT from google.cloud.storage.retry import ( @@ -143,6 +150,9 @@ class ACL(object): reload_path = None save_path = None user_project = None + _METRICS_CLASS_NAME = "ACL" + _METRICS_RELOAD_OP = "ListBucketACLs" + _METRICS_SAVE_OP = "UpdateBucketACL" def __init__(self): self.entities = {} @@ -180,8 +190,16 @@ def reset(self): self.entities.clear() self.loaded = False + def _lazy_method_context(self, method_name): + if self.loaded: + return nullcontext() + return _client_method_override( + f"storage.{self._METRICS_CLASS_NAME}.{method_name}" + ) + def __iter__(self): - self._ensure_loaded() + with self._lazy_method_context("__iter__"): + self._ensure_loaded() for entity in self.entities.values(): for role in entity.get_roles(): @@ -201,24 +219,25 @@ def entity_from_dict(self, entity_dict): :rtype: :class:`_ACLEntity` :returns: An Entity constructed from the dictionary. """ - entity = entity_dict["entity"] - role = entity_dict["role"] + with self._lazy_method_context("entity_from_dict"): + entity = entity_dict["entity"] + role = entity_dict["role"] - if entity == "allUsers": - entity = self.all() + if entity == "allUsers": + entity = self.all() - elif entity == "allAuthenticatedUsers": - entity = self.all_authenticated() + elif entity == "allAuthenticatedUsers": + entity = self.all_authenticated() - elif "-" in entity: - entity_type, identifier = entity.split("-", 1) - entity = self.entity(entity_type=entity_type, identifier=identifier) + elif "-" in entity: + entity_type, identifier = entity.split("-", 1) + entity = self.entity(entity_type=entity_type, identifier=identifier) - if not isinstance(entity, _ACLEntity): - raise ValueError(f"Invalid dictionary: {entity_dict}") + if not isinstance(entity, _ACLEntity): + raise ValueError(f"Invalid dictionary: {entity_dict}") - entity.grant(role) - return entity + entity.grant(role) + return entity def has_entity(self, entity): """Returns whether or not this ACL has any entries for an entity. @@ -229,8 +248,9 @@ def has_entity(self, entity): :rtype: bool :returns: True of the entity exists in the ACL. """ - self._ensure_loaded() - return str(entity) in self.entities + with self._lazy_method_context("has_entity"): + self._ensure_loaded() + return str(entity) in self.entities def get_entity(self, entity, default=None): """Gets an entity object from the ACL. @@ -246,8 +266,9 @@ def get_entity(self, entity, default=None): :returns: The corresponding entity or the value provided to ``default``. """ - self._ensure_loaded() - return self.entities.get(str(entity), default) + with self._lazy_method_context("get_entity"): + self._ensure_loaded() + return self.entities.get(str(entity), default) def add_entity(self, entity): """Add an entity to the ACL. @@ -255,8 +276,9 @@ def add_entity(self, entity): :type entity: :class:`_ACLEntity` :param entity: The entity to add to this ACL. """ - self._ensure_loaded() - self.entities[str(entity)] = entity + with self._lazy_method_context("add_entity"): + self._ensure_loaded() + self.entities[str(entity)] = entity def entity(self, entity_type, identifier=None): """Factory method for creating an Entity. @@ -277,12 +299,13 @@ def entity(self, entity_type, identifier=None): :rtype: :class:`_ACLEntity` :returns: A new Entity or a reference to an existing identical entity. """ - entity = _ACLEntity(entity_type=entity_type, identifier=identifier) - if self.has_entity(entity): - entity = self.get_entity(entity) - else: - self.add_entity(entity) - return entity + with self._lazy_method_context("entity"): + entity = _ACLEntity(entity_type=entity_type, identifier=identifier) + if self.has_entity(entity): + entity = self.get_entity(entity) + else: + self.add_entity(entity) + return entity def user(self, identifier): """Factory method for a user Entity. @@ -293,7 +316,8 @@ def user(self, identifier): :rtype: :class:`_ACLEntity` :returns: An Entity corresponding to this user. """ - return self.entity("user", identifier=identifier) + with self._lazy_method_context("user"): + return self.entity("user", identifier=identifier) def group(self, identifier): """Factory method for a group Entity. @@ -304,7 +328,8 @@ def group(self, identifier): :rtype: :class:`_ACLEntity` :returns: An Entity corresponding to this group. """ - return self.entity("group", identifier=identifier) + with self._lazy_method_context("group"): + return self.entity("group", identifier=identifier) def domain(self, domain): """Factory method for a domain Entity. @@ -315,7 +340,8 @@ def domain(self, domain): :rtype: :class:`_ACLEntity` :returns: An entity corresponding to this domain. """ - return self.entity("domain", identifier=domain) + with self._lazy_method_context("domain"): + return self.entity("domain", identifier=domain) def all(self): """Factory method for an Entity representing all users. @@ -323,7 +349,8 @@ def all(self): :rtype: :class:`_ACLEntity` :returns: An entity representing all users. """ - return self.entity("allUsers") + with self._lazy_method_context("all"): + return self.entity("allUsers") def all_authenticated(self): """Factory method for an Entity representing all authenticated users. @@ -331,7 +358,8 @@ def all_authenticated(self): :rtype: :class:`_ACLEntity` :returns: An entity representing all authenticated users. """ - return self.entity("allAuthenticatedUsers") + with self._lazy_method_context("all_authenticated"): + return self.entity("allAuthenticatedUsers") def get_entities(self): """Get a list of all Entity objects. @@ -339,8 +367,9 @@ def get_entities(self): :rtype: list of :class:`_ACLEntity` objects :returns: A list of all Entity objects. """ - self._ensure_loaded() - return list(self.entities.values()) + with self._lazy_method_context("get_entities"): + self._ensure_loaded() + return list(self.entities.values()) @property def client(self): @@ -381,25 +410,30 @@ def reload(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ with create_trace_span(name="Storage.ACL.reload"): - path = self.reload_path - client = self._require_client(client) - query_params = {} - - if self.user_project is not None: - query_params["userProject"] = self.user_project - - self.entities.clear() - - found = client._get_resource( - path, - query_params=query_params, - timeout=timeout, - retry=retry, - ) - self.loaded = True + with _start_storage_operation( + _resolve_operation_client(self, client), + f"storage.{self._METRICS_CLASS_NAME}.reload", + self._METRICS_RELOAD_OP, + ): + path = self.reload_path + client = self._require_client(client) + query_params = {} + + if self.user_project is not None: + query_params["userProject"] = self.user_project + + self.entities.clear() + + found = client._get_resource( + path, + query_params=query_params, + timeout=timeout, + retry=retry, + ) + self.loaded = True - for entry in found.get("items", ()): - self.add_entity(self.entity_from_dict(entry)) + for entry in found.get("items", ()): + self.add_entity(self.entity_from_dict(entry)) def _save( self, @@ -545,17 +579,22 @@ def save( save_to_backend = True if save_to_backend: - self._save( - acl, - None, - client, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - retry=retry, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + f"storage.{self._METRICS_CLASS_NAME}.save", + self._METRICS_SAVE_OP, + ): + self._save( + acl, + None, + client, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + retry=retry, + ) def save_predefined( self, @@ -610,18 +649,23 @@ def save_predefined( (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ with create_trace_span(name="Storage.ACL.savePredefined"): - predefined = self.validate_predefined(predefined) - self._save( - None, - predefined, - client, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - retry=retry, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + f"storage.{self._METRICS_CLASS_NAME}.save_predefined", + self._METRICS_SAVE_OP, + ): + predefined = self.validate_predefined(predefined) + self._save( + None, + predefined, + client, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + retry=retry, + ) def clear( self, @@ -673,16 +717,21 @@ def clear( (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ with create_trace_span(name="Storage.ACL.clear"): - self.save( - [], - client=client, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - retry=retry, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + f"storage.{self._METRICS_CLASS_NAME}.clear", + self._METRICS_SAVE_OP, + ): + self.save( + [], + client=client, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + retry=retry, + ) class BucketACL(ACL): @@ -692,6 +741,8 @@ class BucketACL(ACL): :param bucket: The bucket to which this ACL relates. """ + _METRICS_CLASS_NAME = "BucketACL" + def __init__(self, bucket): super(BucketACL, self).__init__() self.bucket = bucket @@ -722,6 +773,9 @@ class DefaultObjectACL(BucketACL): _URL_PATH_ELEM = "defaultObjectAcl" _PREDEFINED_QUERY_PARAM = "predefinedDefaultObjectAcl" + _METRICS_CLASS_NAME = "DefaultObjectACL" + _METRICS_RELOAD_OP = "ListDefaultObjectACLs" + _METRICS_SAVE_OP = "UpdateDefaultObjectACL" class ObjectACL(ACL): @@ -731,6 +785,10 @@ class ObjectACL(ACL): :param blob: The blob that this ACL corresponds to. """ + _METRICS_CLASS_NAME = "ObjectACL" + _METRICS_RELOAD_OP = "ListObjectACLs" + _METRICS_SAVE_OP = "UpdateObjectACL" + def __init__(self, blob): super(ObjectACL, self).__init__() self.blob = blob diff --git a/packages/google-cloud-storage/google/cloud/storage/batch.py b/packages/google-cloud-storage/google/cloud/storage/batch.py index aed3703eb03d..304aa0cc6e88 100644 --- a/packages/google-cloud-storage/google/cloud/storage/batch.py +++ b/packages/google-cloud-storage/google/cloud/storage/batch.py @@ -41,6 +41,7 @@ import requests from google.cloud import _helpers, exceptions +from google.cloud.storage._helpers import _start_storage_operation from google.cloud.storage._http import Connection from google.cloud.storage.constants import _DEFAULT_TIMEOUT @@ -163,6 +164,7 @@ class Batch(Connection): """ _MAX_BATCH_SIZE = 1000 + _IS_BATCH = True def __init__(self, client, raise_exception=True): api_endpoint = client._connection.API_BASE_URL @@ -299,25 +301,26 @@ def finish(self, raise_exception=True): :rtype: list of tuples :returns: one ``(headers, payload)`` tuple per deferred request. """ - headers, body, timeout = self._prepare_batch_request() + with _start_storage_operation(self._client, "storage.Batch.finish", "Batch"): + headers, body, timeout = self._prepare_batch_request() - url = f"{self.API_BASE_URL}/batch/storage/v1" + url = f"{self.API_BASE_URL}/batch/storage/v1" - # Use the private ``_base_connection`` rather than the property - # ``_connection``, since the property may be this - # current batch. - response = self._client._base_connection._make_request( - "POST", url, data=body, headers=headers, timeout=timeout - ) + # Use the private ``_base_connection`` rather than the property + # ``_connection``, since the property may be this + # current batch. + response = self._client._base_connection._make_request( + "POST", url, data=body, headers=headers, timeout=timeout + ) - # Raise exception if the top-level batch request fails - if not 200 <= response.status_code < 300: - raise exceptions.from_http_response(response) + # Raise exception if the top-level batch request fails + if not 200 <= response.status_code < 300: + raise exceptions.from_http_response(response) - responses = list(_unpack_batch_response(response)) - self._finish_futures(responses, raise_exception=raise_exception) - self._responses = responses - return responses + responses = list(_unpack_batch_response(response)) + self._finish_futures(responses, raise_exception=raise_exception) + self._responses = responses + return responses def current(self): """Return the topmost batch, or None.""" diff --git a/packages/google-cloud-storage/google/cloud/storage/blob.py b/packages/google-cloud-storage/google/cloud/storage/blob.py index 87493a30dfc0..e91d52c79526 100644 --- a/packages/google-cloud-storage/google/cloud/storage/blob.py +++ b/packages/google-cloud-storage/google/cloud/storage/blob.py @@ -40,14 +40,20 @@ from google.cloud import exceptions from google.cloud.storage._helpers import ( _API_VERSION, + _SUPPRESS_BODY_SIZE, _add_etag_match_headers, _add_generation_match_parameters, _bucket_bound_hostname_url, + _client_method_override, _get_default_headers, _get_default_storage_base_url, _PropertyMixin, _raise_if_more_than_one_set, + _record_primary_exception, + _resolve_operation_client, _scalar_property, + _start_storage_operation, + _suppress_internal_metrics, _virtual_hosted_style_base_url, ) from google.cloud.storage._media.requests import ( @@ -58,6 +64,9 @@ RawDownload, ResumableUpload, ) +from google.cloud.storage._media.requests import ( + _request_helpers as _media_request_helpers, +) from google.cloud.storage._opentelemetry_tracing import ( _get_opentelemetry_attributes_from_url, ) @@ -73,7 +82,11 @@ STANDARD_STORAGE_CLASS, ) from google.cloud.storage.exceptions import DataCorruption, InvalidResponse -from google.cloud.storage.fileio import BlobReader, BlobWriter +from google.cloud.storage.fileio import ( + BlobReader, + BlobWriter, + _MetricsTextIOWrapper, +) from google.cloud.storage.retry import ( DEFAULT_RETRY, DEFAULT_RETRY_IF_ETAG_IN_JSON, @@ -189,6 +202,10 @@ class Blob(_PropertyMixin): _CHUNK_SIZE_MULTIPLE = 256 * 1024 """Number (256 KB, in bytes) that must divide the chunk size.""" + _METRICS_CLASS_NAME = "Blob" + _METRICS_GET_OP = "GetObject" + _METRICS_UPDATE_OP = "UpdateObject" + STORAGE_CLASSES = ( STANDARD_STORAGE_CLASS, NEARLINE_STORAGE_CLASS, @@ -745,45 +762,46 @@ def exists( """ with self._create_trace_span(name="Storage.Blob.exists"): client = self._require_client(client) - # We only need the status code (200 or not) so we seek to - # minimize the returned payload. - query_params = self._query_params - query_params["fields"] = "name" - if soft_deleted is not None: - query_params["softDeleted"] = soft_deleted - - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) - - headers = {} - _add_etag_match_headers( - headers, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - ) + with _start_storage_operation(client, "storage.Blob.exists", "GetObject"): + # We only need the status code (200 or not) so we seek to + # minimize the returned payload. + query_params = self._query_params + query_params["fields"] = "name" + if soft_deleted is not None: + query_params["softDeleted"] = soft_deleted + + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) - try: - # We intentionally pass `_target_object=None` since fields=name - # would limit the local properties. - client._get_resource( - self.path, - query_params=query_params, - headers=headers, - timeout=timeout, - retry=retry, - _target_object=None, + headers = {} + _add_etag_match_headers( + headers, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, ) - except NotFound: - # NOTE: This will not fail immediately in a batch. However, when - # Batch.finish() is called, the resulting `NotFound` will be - # raised. - return False - return True + + try: + # We intentionally pass `_target_object=None` since fields=name + # would limit the local properties. + client._get_resource( + self.path, + query_params=query_params, + headers=headers, + timeout=timeout, + retry=retry, + _target_object=None, + ) + except NotFound: + # NOTE: This will not fail immediately in a batch. However, when + # Batch.finish() is called, the resulting `NotFound` will be + # raised. + return False + return True def delete( self, @@ -848,17 +866,22 @@ def delete( :meth:`google.cloud.storage.bucket.Bucket.delete_blob`). """ with self._create_trace_span(name="Storage.Blob.delete"): - self.bucket.delete_blob( - self.name, - client=client, - generation=self.generation, - timeout=timeout, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.delete", + "DeleteObject", + ): + self.bucket.delete_blob( + self.name, + client=client, + generation=self.generation, + timeout=timeout, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + ) def _get_transport(self, client): """Return the client's transport. @@ -1145,6 +1168,13 @@ def _do_download( self.name, ) + if not _SUPPRESS_BODY_SIZE.get(): + recorder = _media_request_helpers.resolve_transport_recorder(transport) + if recorder is not None: + received = getattr(download, "_bytes_downloaded", 0) + if isinstance(received, int) and not isinstance(received, bool): + recorder.record_response_body_size(received, operation="ReadObject") + def download_to_file( self, file_obj, @@ -1266,23 +1296,28 @@ def download_to_file( :raises: :class:`google.cloud.exceptions.NotFound` """ with self._create_trace_span(name="Storage.Blob.downloadToFile"): - self._prep_and_do_download( - file_obj, - client=client, - start=start, - end=end, - raw_download=raw_download, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - single_shot_download=single_shot_download, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.download_to_file", + "ReadObject", + ): + self._prep_and_do_download( + file_obj, + client=client, + start=start, + end=end, + raw_download=raw_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + single_shot_download=single_shot_download, + ) def _handle_filename_and_download(self, filename, *args, **kwargs): """Download the contents of this blob into a named file. @@ -1292,24 +1327,33 @@ def _handle_filename_and_download(self, filename, *args, **kwargs): For *args and **kwargs, refer to the documentation for download_to_filename() for more information. """ + default_method = ( + "storage.transfer_manager.download_many" + if kwargs.get("command") == "tm.download_many" + else "storage.Blob.download_to_filename" + ) + with _start_storage_operation( + _resolve_operation_client(self, kwargs.get("client")), + default_method, + "ReadObject", + ): + try: + with open(filename, "wb") as file_obj: + self._prep_and_do_download( + file_obj, + *args, + **kwargs, + ) - try: - with open(filename, "wb") as file_obj: - self._prep_and_do_download( - file_obj, - *args, - **kwargs, - ) - - except (DataCorruption, NotFound): - # Delete the corrupt or empty downloaded file. - os.remove(filename) - raise + except (DataCorruption, NotFound): + # Delete the corrupt or empty downloaded file. + os.remove(filename) + raise - updated = self.updated - if updated is not None: - mtime = updated.timestamp() - os.utime(file_obj.name, (mtime, mtime)) + updated = self.updated + if updated is not None: + mtime = updated.timestamp() + os.utime(file_obj.name, (mtime, mtime)) def download_to_filename( self, @@ -1422,23 +1466,28 @@ def download_to_filename( :raises: :class:`google.cloud.exceptions.NotFound` """ with self._create_trace_span(name="Storage.Blob.downloadToFilename"): - self._handle_filename_and_download( - filename, - client=client, - start=start, - end=end, - raw_download=raw_download, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - single_shot_download=single_shot_download, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.download_to_filename", + "ReadObject", + ): + self._handle_filename_and_download( + filename, + client=client, + start=start, + end=end, + raw_download=raw_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + single_shot_download=single_shot_download, + ) def download_as_bytes( self, @@ -1547,26 +1596,31 @@ def download_as_bytes( :raises: :class:`google.cloud.exceptions.NotFound` """ with self._create_trace_span(name="Storage.Blob.downloadAsBytes"): - string_buffer = BytesIO() + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.download_as_bytes", + "ReadObject", + ): + string_buffer = BytesIO() - self._prep_and_do_download( - string_buffer, - client=client, - start=start, - end=end, - raw_download=raw_download, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - single_shot_download=single_shot_download, - ) - return string_buffer.getvalue() + self._prep_and_do_download( + string_buffer, + client=client, + start=start, + end=end, + raw_download=raw_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + single_shot_download=single_shot_download, + ) + return string_buffer.getvalue() def download_as_string( self, @@ -1670,21 +1724,26 @@ def download_as_string( stacklevel=2, ) with self._create_trace_span(name="Storage.Blob.downloadAsString"): - return self.download_as_bytes( - client=client, - start=start, - end=end, - raw_download=raw_download, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - retry=retry, - single_shot_download=single_shot_download, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.download_as_string", + "ReadObject", + ): + return self.download_as_bytes( + client=client, + start=start, + end=end, + raw_download=raw_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + retry=retry, + single_shot_download=single_shot_download, + ) def download_as_text( self, @@ -1784,32 +1843,37 @@ def download_as_text( :returns: The data stored in this blob, decoded to text. """ with self._create_trace_span(name="Storage.Blob.downloadAsText"): - data = self.download_as_bytes( - client=client, - start=start, - end=end, - raw_download=raw_download, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - retry=retry, - single_shot_download=single_shot_download, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.download_as_text", + "ReadObject", + ): + data = self.download_as_bytes( + client=client, + start=start, + end=end, + raw_download=raw_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + retry=retry, + single_shot_download=single_shot_download, + ) - if encoding is not None: - return data.decode(encoding) + if encoding is not None: + return data.decode(encoding) - if self.content_type is not None: - msg = HeaderParser().parsestr("Content-Type: " + self.content_type) - params = dict(msg.get_params()[1:]) - if "charset" in params: - return data.decode(params["charset"]) + if self.content_type is not None: + msg = HeaderParser().parsestr("Content-Type: " + self.content_type) + params = dict(msg.get_params()[1:]) + if "charset" in params: + return data.decode(params["charset"]) - return data.decode("utf-8") + return data.decode("utf-8") def _get_content_type(self, content_type, filename=None): """Determine the content type from the current object. @@ -2083,6 +2147,12 @@ def _do_multipart_upload( response = upload.transmit( transport, data, object_metadata, content_type, timeout=timeout ) + if not _SUPPRESS_BODY_SIZE.get(): + recorder = _media_request_helpers.resolve_transport_recorder(transport) + if recorder is not None: + recorder.record_request_body_size( + len(data), operation="WriteObject" + ) return response @@ -2483,10 +2553,23 @@ def _do_resumable_upload( while not upload.finished: try: response = upload.transmit_next_chunk(transport, timeout=timeout) - except DataCorruption: + except DataCorruption as exc: # Attempt to delete the corrupted object. - self.delete() + try: + with _suppress_internal_metrics(): + self.delete() + except BaseException as cleanup_exc: + _record_primary_exception(exc, cleanup_exc) + raise raise + if not _SUPPRESS_BODY_SIZE.get(): + recorder = _media_request_helpers.resolve_transport_recorder(transport) + if recorder is not None: + uploaded = getattr(upload, "bytes_uploaded", 0) + if isinstance(uploaded, int) and not isinstance(uploaded, bool): + recorder.record_request_body_size( + uploaded, operation="WriteObject" + ) return response def _do_upload( @@ -2850,29 +2933,39 @@ def _prep_and_do_upload( :raises: :class:`~google.cloud.exceptions.GoogleCloudError` if the upload response returns an error status. """ - _maybe_rewind(file_obj, rewind=rewind) - predefined_acl = ACL.validate_predefined(predefined_acl) + default_method = ( + "storage.transfer_manager.upload_many" + if command == "tm.upload_many" + else "storage.Blob.upload_from_file" + ) + with _start_storage_operation( + _resolve_operation_client(self, client), + default_method, + "WriteObject", + ): + _maybe_rewind(file_obj, rewind=rewind) + predefined_acl = ACL.validate_predefined(predefined_acl) - try: - created_json = self._do_upload( - client, - file_obj, - content_type, - size, - predefined_acl, - if_generation_match, - if_generation_not_match, - if_metageneration_match, - if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - command=command, - crc32c_checksum_value=crc32c_checksum_value, - ) - self._set_properties(created_json) - except InvalidResponse as exc: - _raise_from_invalid_response(exc) + try: + created_json = self._do_upload( + client, + file_obj, + content_type, + size, + predefined_acl, + if_generation_match, + if_generation_not_match, + if_metageneration_match, + if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + command=command, + crc32c_checksum_value=crc32c_checksum_value, + ) + self._set_properties(created_json) + except InvalidResponse as exc: + _raise_from_invalid_response(exc) def upload_from_file( self, @@ -3034,22 +3127,27 @@ def upload_from_file( if the upload response returns an error status. """ with self._create_trace_span(name="Storage.Blob.uploadFromFile"): - self._prep_and_do_upload( - file_obj, - rewind=rewind, - size=size, - content_type=content_type, - client=client, - predefined_acl=predefined_acl, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - crc32c_checksum_value=crc32c_checksum_value, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.upload_from_file", + "WriteObject", + ): + self._prep_and_do_upload( + file_obj, + rewind=rewind, + size=size, + content_type=content_type, + client=client, + predefined_acl=predefined_acl, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + crc32c_checksum_value=crc32c_checksum_value, + ) def _handle_filename_and_upload(self, filename, content_type=None, *args, **kwargs): """Upload this blob's contents from the content of a named file. @@ -3062,18 +3160,27 @@ def _handle_filename_and_upload(self, filename, content_type=None, *args, **kwar For *args and **kwargs, refer to the documentation for upload_from_filename() for more information. """ + default_method = ( + "storage.transfer_manager.upload_many" + if kwargs.get("command") == "tm.upload_many" + else "storage.Blob.upload_from_filename" + ) + with _start_storage_operation( + _resolve_operation_client(self, kwargs.get("client")), + default_method, + "WriteObject", + ): + content_type = self._get_content_type(content_type, filename=filename) - content_type = self._get_content_type(content_type, filename=filename) - - with open(filename, "rb") as file_obj: - total_bytes = os.fstat(file_obj.fileno()).st_size - self._prep_and_do_upload( - file_obj, - content_type=content_type, - size=total_bytes, - *args, - **kwargs, - ) + with open(filename, "rb") as file_obj: + total_bytes = os.fstat(file_obj.fileno()).st_size + self._prep_and_do_upload( + file_obj, + content_type=content_type, + size=total_bytes, + *args, + **kwargs, + ) def upload_from_filename( self, @@ -3217,20 +3324,25 @@ def upload_from_filename( base64: https://datatracker.ietf.org/doc/html/rfc4648#section-4 """ with self._create_trace_span(name="Storage.Blob.uploadFromFilename"): - self._handle_filename_and_upload( - filename, - content_type=content_type, - client=client, - predefined_acl=predefined_acl, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - crc32c_checksum_value=crc32c_checksum_value, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.upload_from_filename", + "WriteObject", + ): + self._handle_filename_and_upload( + filename, + content_type=content_type, + client=client, + predefined_acl=predefined_acl, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + crc32c_checksum_value=crc32c_checksum_value, + ) def upload_from_string( self, @@ -3366,25 +3478,30 @@ def upload_from_string( base64: https://datatracker.ietf.org/doc/html/rfc4648#section-4 """ with self._create_trace_span(name="Storage.Blob.uploadFromString"): - data = _to_bytes(data, encoding="utf-8") - string_buffer = BytesIO(data) - self.upload_from_file( - file_obj=string_buffer, - size=len(data), - content_type=content_type, - client=client, - predefined_acl=predefined_acl, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - crc32c_checksum_value=crc32c_checksum_value, - ) - - def create_resumable_upload_session( + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.upload_from_string", + "WriteObject", + ): + data = _to_bytes(data, encoding="utf-8") + string_buffer = BytesIO(data) + self.upload_from_file( + file_obj=string_buffer, + size=len(data), + content_type=content_type, + client=client, + predefined_acl=predefined_acl, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + crc32c_checksum_value=crc32c_checksum_value, + ) + + def create_resumable_upload_session( self, content_type=None, size=None, @@ -3518,51 +3635,56 @@ def create_resumable_upload_session( if the session creation response returns an error status. """ with self._create_trace_span(name="Storage.Blob.createResumableUploadSession"): - # Handle ConditionalRetryPolicy. - if isinstance(retry, ConditionalRetryPolicy): - # Conditional retries are designed for non-media calls, which change - # arguments into query_params dictionaries. Media operations work - # differently, so here we make a "fake" query_params to feed to the - # ConditionalRetryPolicy. - query_params = { - "ifGenerationMatch": if_generation_match, - "ifMetagenerationMatch": if_metageneration_match, - } - retry = retry.get_retry_policy_if_conditions_met( - query_params=query_params - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.create_resumable_upload_session", + "StartResumableWrite", + ): + # Handle ConditionalRetryPolicy. + if isinstance(retry, ConditionalRetryPolicy): + # Conditional retries are designed for non-media calls, which change + # arguments into query_params dictionaries. Media operations work + # differently, so here we make a "fake" query_params to feed to the + # ConditionalRetryPolicy. + query_params = { + "ifGenerationMatch": if_generation_match, + "ifMetagenerationMatch": if_metageneration_match, + } + retry = retry.get_retry_policy_if_conditions_met( + query_params=query_params + ) - extra_headers = {} - if origin is not None: - # This header is specifically for client-side uploads, it - # determines the origins allowed for CORS. - extra_headers["Origin"] = origin + extra_headers = {} + if origin is not None: + # This header is specifically for client-side uploads, it + # determines the origins allowed for CORS. + extra_headers["Origin"] = origin - try: - fake_stream = BytesIO(b"") - # Send a fake the chunk size which we **know** will be acceptable - # to the `ResumableUpload` constructor. The chunk size only - # matters when **sending** bytes to an upload. - upload, _ = self._initiate_resumable_upload( - client, - fake_stream, - content_type, - size, - predefined_acl=predefined_acl, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - extra_headers=extra_headers, - chunk_size=self._CHUNK_SIZE_MULTIPLE, - timeout=timeout, - checksum=checksum, - retry=retry, - ) + try: + fake_stream = BytesIO(b"") + # Send a fake the chunk size which we **know** will be acceptable + # to the `ResumableUpload` constructor. The chunk size only + # matters when **sending** bytes to an upload. + upload, _ = self._initiate_resumable_upload( + client, + fake_stream, + content_type, + size, + predefined_acl=predefined_acl, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + extra_headers=extra_headers, + chunk_size=self._CHUNK_SIZE_MULTIPLE, + timeout=timeout, + checksum=checksum, + retry=retry, + ) - return upload.resumable_url - except InvalidResponse as exc: - _raise_from_invalid_response(exc) + return upload.resumable_url + except InvalidResponse as exc: + _raise_from_invalid_response(exc) def get_iam_policy( self, @@ -3615,23 +3737,27 @@ def get_iam_policy( """ with self._create_trace_span(name="Storage.Blob.getIamPolicy"): client = self._require_client(client) + with _start_storage_operation( + client, "storage.Blob.get_iam_policy", "GetIamPolicy" + ): + query_params = {} - query_params = {} - - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - if requested_policy_version is not None: - query_params["optionsRequestedPolicyVersion"] = requested_policy_version + if requested_policy_version is not None: + query_params["optionsRequestedPolicyVersion"] = ( + requested_policy_version + ) - info = client._get_resource( - f"{self.path}/iam", - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) - return Policy.from_api_repr(info) + info = client._get_resource( + f"{self.path}/iam", + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) + return Policy.from_api_repr(info) def set_iam_policy( self, @@ -3676,24 +3802,26 @@ def set_iam_policy( """ with self._create_trace_span(name="Storage.Blob.setIamPolicy"): client = self._require_client(client) + with _start_storage_operation( + client, "storage.Blob.set_iam_policy", "SetIamPolicy" + ): + query_params = {} - query_params = {} - - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - path = f"{self.path}/iam" - resource = policy.to_api_repr() - resource["resourceId"] = self.path - info = client._put_resource( - path, - resource, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) - return Policy.from_api_repr(info) + path = f"{self.path}/iam" + resource = policy.to_api_repr() + resource["resourceId"] = self.path + info = client._put_resource( + path, + resource, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) + return Policy.from_api_repr(info) def test_iam_permissions( self, @@ -3738,21 +3866,26 @@ def test_iam_permissions( """ with self._create_trace_span(name="Storage.Blob.testIamPermissions"): client = self._require_client(client) - query_params = {"permissions": permissions} + with _start_storage_operation( + client, + "storage.Blob.test_iam_permissions", + "TestIamPermissions", + ): + query_params = {"permissions": permissions} - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - path = f"{self.path}/iam/testPermissions" - resp = client._get_resource( - path, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) + path = f"{self.path}/iam/testPermissions" + resp = client._get_resource( + path, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) - return resp.get("permissions", []) + return resp.get("permissions", []) def make_public( self, @@ -3797,16 +3930,21 @@ def make_public( (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ with self._create_trace_span(name="Storage.Blob.makePublic"): - self.acl.all().grant_read() - self.acl.save( - client=client, - timeout=timeout, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.make_public", + "UpdateObjectACL", + ): + self.acl.all().grant_read() + self.acl.save( + client=client, + timeout=timeout, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + ) def make_private( self, @@ -3851,16 +3989,21 @@ def make_private( (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ with self._create_trace_span(name="Storage.Blob.makePrivate"): - self.acl.all().revoke_read() - self.acl.save( - client=client, - timeout=timeout, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.make_private", + "UpdateObjectACL", + ): + self.acl.all().revoke_read() + self.acl.save( + client=client, + timeout=timeout, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + ) def compose( self, @@ -3944,89 +4087,96 @@ def compose( successful composition. """ with self._create_trace_span(name="Storage.Blob.compose"): - sources_len = len(sources) - client = self._require_client(client) - query_params = {} - - if isinstance(if_generation_match, list): - warnings.warn( - _COMPOSE_IF_GENERATION_LIST_DEPRECATED, - DeprecationWarning, - stacklevel=2, - ) - - if if_source_generation_match is not None: - raise ValueError( - _COMPOSE_IF_GENERATION_LIST_AND_IF_SOURCE_GENERATION_ERROR + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.compose", + "ComposeObject", + ): + sources_len = len(sources) + client = self._require_client(client) + query_params = {} + + if isinstance(if_generation_match, list): + warnings.warn( + _COMPOSE_IF_GENERATION_LIST_DEPRECATED, + DeprecationWarning, + stacklevel=2, ) - if_source_generation_match = if_generation_match - if_generation_match = None - - if isinstance(if_metageneration_match, list): - warnings.warn( - _COMPOSE_IF_METAGENERATION_LIST_DEPRECATED, - DeprecationWarning, - stacklevel=2, - ) - - if_metageneration_match = None - - if if_source_generation_match is None: - if_source_generation_match = [None] * sources_len - if len(if_source_generation_match) != sources_len: - raise ValueError(_COMPOSE_IF_SOURCE_GENERATION_MISMATCH_ERROR) - - source_objects = [] - for source, source_generation in zip(sources, if_source_generation_match): - source_object = { - "name": source.name, - "generation": source.generation, - } + if if_source_generation_match is not None: + raise ValueError( + _COMPOSE_IF_GENERATION_LIST_AND_IF_SOURCE_GENERATION_ERROR + ) - preconditions = {} - if source_generation is not None: - preconditions["ifGenerationMatch"] = source_generation + if_source_generation_match = if_generation_match + if_generation_match = None - if preconditions: - source_object["objectPreconditions"] = preconditions + if isinstance(if_metageneration_match, list): + warnings.warn( + _COMPOSE_IF_METAGENERATION_LIST_DEPRECATED, + DeprecationWarning, + stacklevel=2, + ) - source_objects.append(source_object) + if_metageneration_match = None - if destination_contexts is not None: - if isinstance(destination_contexts, ObjectContexts): - self.contexts = destination_contexts - else: - raise ValueError( - "destination_contexts must be an ObjectContexts object" - ) + if if_source_generation_match is None: + if_source_generation_match = [None] * sources_len + if len(if_source_generation_match) != sources_len: + raise ValueError(_COMPOSE_IF_SOURCE_GENERATION_MISMATCH_ERROR) - request = { - "sourceObjects": source_objects, - "destination": self._properties.copy(), - } + source_objects = [] + for source, source_generation in zip( + sources, if_source_generation_match + ): + source_object = { + "name": source.name, + "generation": source.generation, + } + + preconditions = {} + if source_generation is not None: + preconditions["ifGenerationMatch"] = source_generation + + if preconditions: + source_object["objectPreconditions"] = preconditions + + source_objects.append(source_object) + + if destination_contexts is not None: + if isinstance(destination_contexts, ObjectContexts): + self.contexts = destination_contexts + else: + raise ValueError( + "destination_contexts must be an ObjectContexts object" + ) + + request = { + "sourceObjects": source_objects, + "destination": self._properties.copy(), + } - if delete_source_objects is not None: - request["deleteSourceObjects"] = delete_source_objects + if delete_source_objects is not None: + request["deleteSourceObjects"] = delete_source_objects - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_metageneration_match=if_metageneration_match, - ) + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_metageneration_match=if_metageneration_match, + ) - api_response = client._post_resource( - f"{self.path}/compose", - request, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=self, - ) - self._set_properties(api_response) + api_response = client._post_resource( + f"{self.path}/compose", + request, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=self, + ) + self._set_properties(api_response) def rewrite( self, @@ -4141,71 +4291,76 @@ def rewrite( """ with self._create_trace_span(name="Storage.Blob.rewrite"): client = self._require_client(client) - headers = _get_encryption_headers(self._encryption_key) - headers.update(_get_encryption_headers(source._encryption_key, source=True)) - - query_params = self._query_params - if "generation" in query_params: - del query_params["generation"] - - if token: - query_params["rewriteToken"] = token - - if source.generation: - query_params["sourceGeneration"] = source.generation - - # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object - # at rest, object resource metadata will store the version of the Key Management - # Service cryptographic material. If a Blob instance with KMS Key metadata set is - # used to rewrite the object, then the existing kmsKeyName version - # value can't be used in the rewrite request and the client instead ignores it. - if ( - self.kms_key_name is not None - and "cryptoKeyVersions" not in self.kms_key_name + with _start_storage_operation( + client, "storage.Blob.rewrite", "RewriteObject" ): - query_params["destinationKmsKeyName"] = self.kms_key_name + headers = _get_encryption_headers(self._encryption_key) + headers.update( + _get_encryption_headers(source._encryption_key, source=True) + ) - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - if_source_generation_match=if_source_generation_match, - if_source_generation_not_match=if_source_generation_not_match, - if_source_metageneration_match=if_source_metageneration_match, - if_source_metageneration_not_match=if_source_metageneration_not_match, - ) + query_params = self._query_params + if "generation" in query_params: + del query_params["generation"] - if destination_contexts is not None: - if isinstance(destination_contexts, ObjectContexts): - self.contexts = destination_contexts - else: - raise ValueError( - "destination_contexts must be an ObjectContexts object" - ) + if token: + query_params["rewriteToken"] = token - path = f"{source.path}/rewriteTo{self.path}" - api_response = client._post_resource( - path, - self._properties, - query_params=query_params, - headers=headers, - timeout=timeout, - retry=retry, - _target_object=self, - ) - rewritten = int(api_response["totalBytesRewritten"]) - size = int(api_response["objectSize"]) + if source.generation: + query_params["sourceGeneration"] = source.generation + + # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object + # at rest, object resource metadata will store the version of the Key Management + # Service cryptographic material. If a Blob instance with KMS Key metadata set is + # used to rewrite the object, then the existing kmsKeyName version + # value can't be used in the rewrite request and the client instead ignores it. + if ( + self.kms_key_name is not None + and "cryptoKeyVersions" not in self.kms_key_name + ): + query_params["destinationKmsKeyName"] = self.kms_key_name + + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, + ) + + if destination_contexts is not None: + if isinstance(destination_contexts, ObjectContexts): + self.contexts = destination_contexts + else: + raise ValueError( + "destination_contexts must be an ObjectContexts object" + ) + + path = f"{source.path}/rewriteTo{self.path}" + api_response = client._post_resource( + path, + self._properties, + query_params=query_params, + headers=headers, + timeout=timeout, + retry=retry, + _target_object=self, + ) + rewritten = int(api_response["totalBytesRewritten"]) + size = int(api_response["objectSize"]) - # The resource key is set if and only if the API response is - # completely done. Additionally, there is no rewrite token to return - # in this case. - if api_response["done"]: - self._set_properties(api_response["resource"]) - return None, rewritten, size + # The resource key is set if and only if the API response is + # completely done. Additionally, there is no rewrite token to return + # in this case. + if api_response["done"]: + self._set_properties(api_response["resource"]) + return None, rewritten, size - return api_response["rewriteToken"], rewritten, size + return api_response["rewriteToken"], rewritten, size def update_storage_class( self, @@ -4308,27 +4463,18 @@ def update_storage_class( See [Configuring Retries](https://cloud.google.com/python/docs/reference/storage/latest/retry_timeout). """ with self._create_trace_span(name="Storage.Blob.updateStorageClass"): - # Update current blob's storage class prior to rewrite - self._patch_property("storageClass", new_class) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Blob.update_storage_class", + "RewriteObject", + ): + # Update current blob's storage class prior to rewrite + self._patch_property("storageClass", new_class) + extra_rewrite_kw = {"client": client} if client is not None else {} - # Execute consecutive rewrite operations until operation is done - token, _, _ = self.rewrite( - self, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - if_source_generation_match=if_source_generation_match, - if_source_generation_not_match=if_source_generation_not_match, - if_source_metageneration_match=if_source_metageneration_match, - if_source_metageneration_not_match=if_source_metageneration_not_match, - timeout=timeout, - retry=retry, - ) - while token is not None: + # Execute consecutive rewrite operations until operation is done token, _, _ = self.rewrite( self, - token=token, if_generation_match=if_generation_match, if_generation_not_match=if_generation_not_match, if_metageneration_match=if_metageneration_match, @@ -4339,7 +4485,24 @@ def update_storage_class( if_source_metageneration_not_match=if_source_metageneration_not_match, timeout=timeout, retry=retry, + **extra_rewrite_kw, ) + while token is not None: + token, _, _ = self.rewrite( + self, + token=token, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, + timeout=timeout, + retry=retry, + **extra_rewrite_kw, + ) def open( self, @@ -4452,56 +4615,96 @@ def open( of those classes, depending on the 'mode' argument. """ with self._create_trace_span(name="Storage.Blob.open"): - if mode == "rb": - if encoding or errors or newline: - raise ValueError( - "encoding, errors and newline arguments are for text mode only" + op_client = _resolve_operation_client(self) + if isinstance(mode, str) and mode.startswith("r"): + op_name = "ReadObject" + elif isinstance(mode, str) and mode.startswith("w"): + op_name = "WriteObject" + else: + op_name = "UNKNOWN" + + try: + if mode == "rb": + if encoding or errors or newline: + raise ValueError( + "encoding, errors and newline arguments are for text mode only" + ) + if ignore_flush: + raise ValueError( + "ignore_flush argument is for non-text write mode only" + ) + elif mode == "wb": + if encoding or errors or newline: + raise ValueError( + "encoding, errors and newline arguments are for text mode only" + ) + elif mode in ("r", "rt"): + if ignore_flush: + raise ValueError( + "ignore_flush argument is for non-text write mode only" + ) + elif mode in ("w", "wt"): + if ignore_flush is False: + raise ValueError( + "ignore_flush is required for text mode writing and " + "cannot be set to False" + ) + else: + raise NotImplementedError( + "Supported modes strings are 'r', 'rb', 'rt', 'w', 'wb', and 'wt' only." ) - if ignore_flush: - raise ValueError( - "ignore_flush argument is for non-text write mode only" + except BaseException: + with _start_storage_operation(op_client, "storage.Blob.open", op_name): + raise + + with _client_method_override("storage.Blob.open"): + if mode == "rb": + return BlobReader(self, chunk_size=chunk_size, **kwargs) + elif mode == "wb": + return BlobWriter( + self, + chunk_size=chunk_size, + ignore_flush=ignore_flush, + **kwargs, ) - return BlobReader(self, chunk_size=chunk_size, **kwargs) - elif mode == "wb": - if encoding or errors or newline: - raise ValueError( - "encoding, errors and newline arguments are for text mode only" + elif mode in ("r", "rt"): + raw_reader = BlobReader(self, chunk_size=chunk_size, **kwargs) + wrapper_cls = ( + _MetricsTextIOWrapper + if raw_reader._metrics_recorder is not None + else TextIOWrapper ) - return BlobWriter( - self, - chunk_size=chunk_size, - ignore_flush=ignore_flush, - **kwargs, - ) - elif mode in ("r", "rt"): - if ignore_flush: - raise ValueError( - "ignore_flush argument is for non-text write mode only" + try: + return wrapper_cls( + raw_reader, + encoding=encoding, + errors=errors, + newline=newline, + ) + except BaseException as exc: + raw_reader._buffer.close() + raw_reader._finalize_metrics(exc=exc, record_body=False) + raise + else: + raw_writer = BlobWriter( + self, chunk_size=chunk_size, ignore_flush=True, **kwargs ) - return TextIOWrapper( - BlobReader(self, chunk_size=chunk_size, **kwargs), - encoding=encoding, - errors=errors, - newline=newline, - ) - elif mode in ("w", "wt"): - if ignore_flush is False: - raise ValueError( - "ignore_flush is required for text mode writing and " - "cannot be set to False" + wrapper_cls = ( + _MetricsTextIOWrapper + if raw_writer._metrics_recorder is not None + else TextIOWrapper ) - return TextIOWrapper( - BlobWriter( - self, chunk_size=chunk_size, ignore_flush=True, **kwargs - ), - encoding=encoding, - errors=errors, - newline=newline, - ) - else: - raise NotImplementedError( - "Supported modes strings are 'r', 'rb', 'rt', 'w', 'wb', and 'wt' only." - ) + try: + return wrapper_cls( + raw_writer, + encoding=encoding, + errors=errors, + newline=newline, + ) + except BaseException as exc: + raw_writer._buffer.close() + raw_writer._finalize_metrics(exc=exc, record_body=False) + raise cache_control = _scalar_property("cacheControl") """HTTP 'Cache-Control' header for this object. @@ -4677,59 +4880,71 @@ def _prep_and_do_download( to be included in the X-Goog-API-Client header. Please leave as None unless otherwise directed. """ - # Handle ConditionalRetryPolicy. - if isinstance(retry, ConditionalRetryPolicy): - # Conditional retries are designed for non-media calls, which change - # arguments into query_params dictionaries. Media operations work - # differently, so here we make a "fake" query_params to feed to the - # ConditionalRetryPolicy. - query_params = { - "ifGenerationMatch": if_generation_match, - "ifMetagenerationMatch": if_metageneration_match, - } - retry = retry.get_retry_policy_if_conditions_met(query_params=query_params) - - client = self._require_client(client) - - download_url = self._get_download_url( - client, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) - headers = _get_encryption_headers(self._encryption_key) - headers["accept-encoding"] = "gzip" - _add_etag_match_headers( - headers, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, + default_method = ( + "storage.transfer_manager.download_many" + if command == "tm.download_many" + else "storage.Blob.download_to_file" ) - # Add any client attached custom headers to be sent with the request. - headers = { - **_get_default_headers(client._connection.user_agent, command=command), - **headers, - **getattr(client, "_extra_headers", {}), - } + with _start_storage_operation( + _resolve_operation_client(self, client), + default_method, + "ReadObject", + ): + # Handle ConditionalRetryPolicy. + if isinstance(retry, ConditionalRetryPolicy): + # Conditional retries are designed for non-media calls, which change + # arguments into query_params dictionaries. Media operations work + # differently, so here we make a "fake" query_params to feed to the + # ConditionalRetryPolicy. + query_params = { + "ifGenerationMatch": if_generation_match, + "ifMetagenerationMatch": if_metageneration_match, + } + retry = retry.get_retry_policy_if_conditions_met( + query_params=query_params + ) - transport = client._http + client = self._require_client(client) - try: - self._do_download( - transport, - file_obj, - download_url, + download_url = self._get_download_url( + client, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) + headers = _get_encryption_headers(self._encryption_key) + headers["accept-encoding"] = "gzip" + _add_etag_match_headers( headers, - start, - end, - raw_download, - timeout=timeout, - checksum=checksum, - retry=retry, - single_shot_download=single_shot_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, ) - except InvalidResponse as exc: - _raise_from_invalid_response(exc) + # Add any client attached custom headers to be sent with the request. + headers = { + **_get_default_headers(client._connection.user_agent, command=command), + **headers, + **getattr(client, "_extra_headers", {}), + } + + transport = client._http + + try: + self._do_download( + transport, + file_obj, + download_url, + headers, + start, + end, + raw_download, + timeout=timeout, + checksum=checksum, + retry=retry, + single_shot_download=single_shot_download, + ) + except InvalidResponse as exc: + _raise_from_invalid_response(exc) @property def component_count(self): diff --git a/packages/google-cloud-storage/google/cloud/storage/bucket.py b/packages/google-cloud-storage/google/cloud/storage/bucket.py index 7954e70e08f9..c8d52f1aa2ae 100644 --- a/packages/google-cloud-storage/google/cloud/storage/bucket.py +++ b/packages/google-cloud-storage/google/cloud/storage/bucket.py @@ -34,8 +34,11 @@ _add_etag_match_headers, _add_generation_match_parameters, _bucket_bound_hostname_url, + _list_metrics_context, _PropertyMixin, + _resolve_operation_client, _scalar_property, + _start_storage_operation, _validate_name, _virtual_hosted_style_base_url, ) @@ -648,6 +651,10 @@ class Bucket(_PropertyMixin): This is used in Bucket.delete() and Bucket.make_public(). """ + _METRICS_CLASS_NAME = "Bucket" + _METRICS_GET_OP = "GetBucket" + _METRICS_UPDATE_OP = "UpdateBucket" + STORAGE_CLASSES = ( STANDARD_STORAGE_CLASS, NEARLINE_STORAGE_CLASS, @@ -974,43 +981,44 @@ def exists( """ with self._create_trace_span(name="Storage.Bucket.exists"): client = self._require_client(client) - # We only need the status code (200 or not) so we seek to - # minimize the returned payload. - query_params = {"fields": "name"} - - if self.user_project is not None: - query_params["userProject"] = self.user_project + with _start_storage_operation(client, "storage.Bucket.exists", "GetBucket"): + # We only need the status code (200 or not) so we seek to + # minimize the returned payload. + query_params = {"fields": "name"} - _add_generation_match_parameters( - query_params, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) + if self.user_project is not None: + query_params["userProject"] = self.user_project - headers = {} - _add_etag_match_headers( - headers, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - ) + _add_generation_match_parameters( + query_params, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) - try: - # We intentionally pass `_target_object=None` since fields=name - # would limit the local properties. - client._get_resource( - self.path, - query_params=query_params, - headers=headers, - timeout=timeout, - retry=retry, - _target_object=None, + headers = {} + _add_etag_match_headers( + headers, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, ) - except NotFound: - # NOTE: This will not fail immediately in a batch. However, when - # Batch.finish() is called, the resulting `NotFound` will be - # raised. - return False - return True + + try: + # We intentionally pass `_target_object=None` since fields=name + # would limit the local properties. + client._get_resource( + self.path, + query_params=query_params, + headers=headers, + timeout=timeout, + retry=retry, + _target_object=None, + ) + except NotFound: + # NOTE: This will not fail immediately in a batch. However, when + # Batch.finish() is called, the resulting `NotFound` will be + # raised. + return False + return True def create( self, @@ -1075,17 +1083,20 @@ def create( """ with self._create_trace_span(name="Storage.Bucket.create"): client = self._require_client(client) - client.create_bucket( - bucket_or_name=self, - project=project, - user_project=self.user_project, - location=location, - predefined_acl=predefined_acl, - predefined_default_object_acl=predefined_default_object_acl, - enable_object_retention=enable_object_retention, - timeout=timeout, - retry=retry, - ) + with _start_storage_operation( + client, "storage.Bucket.create", "CreateBucket" + ): + client.create_bucket( + bucket_or_name=self, + project=project, + user_project=self.user_project, + location=location, + predefined_acl=predefined_acl, + predefined_default_object_acl=predefined_default_object_acl, + enable_object_retention=enable_object_retention, + timeout=timeout, + retry=retry, + ) def update( self, @@ -1124,13 +1135,18 @@ def update( (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ with self._create_trace_span(name="Storage.Bucket.update"): - super(Bucket, self).update( - client=client, - timeout=timeout, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Bucket.update", + "UpdateBucket", + ): + super(Bucket, self).update( + client=client, + timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + ) def reload( self, @@ -1191,39 +1207,44 @@ def reload( See: https://cloud.google.com/storage/docs/soft-delete """ with self._create_trace_span(name="Storage.Bucket.reload"): - try: - super(Bucket, self).reload( - client=client, - projection=projection, - timeout=timeout, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - soft_deleted=soft_deleted, - ) - except api_exceptions.Forbidden: + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Bucket.reload", + "GetBucket", + ): + try: + super(Bucket, self).reload( + client=client, + projection=projection, + timeout=timeout, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + soft_deleted=soft_deleted, + ) + except api_exceptions.Forbidden: + active_client = client or self.client + cache = getattr(active_client, "_bucket_metadata_cache", None) + if cache: + try: + cache.update_cache( + self.name, + f"//storage.googleapis.com/projects/_/buckets/{self.name}", + "global", + ) + except Exception: + pass + raise + active_client = client or self.client cache = getattr(active_client, "_bucket_metadata_cache", None) if cache: try: - cache.update_cache( - self.name, - f"//storage.googleapis.com/projects/_/buckets/{self.name}", - "global", - ) + cache.update_from_bucket(self) except Exception: pass - raise - - active_client = client or self.client - cache = getattr(active_client, "_bucket_metadata_cache", None) - if cache: - try: - cache.update_from_bucket(self) - except Exception: - pass def patch( self, @@ -1262,22 +1283,27 @@ def patch( (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ with self._create_trace_span(name="Storage.Bucket.patch"): - # Special case: For buckets, it is possible that labels are being - # removed; this requires special handling. - if self._label_removals: - self._changes.add("labels") - self._properties.setdefault("labels", {}) - for removed_label in self._label_removals: - self._properties["labels"][removed_label] = None - - # Call the superclass method. - super(Bucket, self).patch( - client=client, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - retry=retry, - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Bucket.patch", + "UpdateBucket", + ): + # Special case: For buckets, it is possible that labels are being + # removed; this requires special handling. + if self._label_removals: + self._changes.add("labels") + self._properties.setdefault("labels", {}) + for removed_label in self._label_removals: + self._properties["labels"][removed_label] = None + + # Call the superclass method. + super(Bucket, self).patch( + client=client, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + retry=retry, + ) @property def acl(self): @@ -1398,33 +1424,38 @@ def get_blob( :returns: The blob object if it exists, otherwise None. """ with self._create_trace_span(name="Storage.Bucket.getBlob"): - blob = Blob( - bucket=self, - name=blob_name, - encryption_key=encryption_key, - generation=generation, - **kwargs, - ) - try: - # NOTE: This will not fail immediately in a batch. However, when - # Batch.finish() is called, the resulting `NotFound` will be - # raised. - blob.reload( - client=client, - timeout=timeout, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - soft_deleted=soft_deleted, + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Bucket.get_blob", + "GetObject", + ): + blob = Blob( + bucket=self, + name=blob_name, + encryption_key=encryption_key, + generation=generation, + **kwargs, ) - except NotFound: - return None - else: - return blob + try: + # NOTE: This will not fail immediately in a batch. However, when + # Batch.finish() is called, the resulting `NotFound` will be + # raised. + blob.reload( + client=client, + timeout=timeout, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + soft_deleted=soft_deleted, + ) + except NotFound: + return None + else: + return blob def list_blobs( self, @@ -1555,26 +1586,27 @@ def list_blobs( """ with self._create_trace_span(name="Storage.Bucket.listBlobs"): client = self._require_client(client) - return client.list_blobs( - self, - max_results=max_results, - page_token=page_token, - prefix=prefix, - delimiter=delimiter, - start_offset=start_offset, - end_offset=end_offset, - include_trailing_delimiter=include_trailing_delimiter, - versions=versions, - projection=projection, - fields=fields, - page_size=page_size, - timeout=timeout, - retry=retry, - match_glob=match_glob, - include_folders_as_prefixes=include_folders_as_prefixes, - soft_deleted=soft_deleted, - filter_=filter_, - ) + with _list_metrics_context("storage.Bucket.list_blobs", "ListObjects"): + return client.list_blobs( + self, + max_results=max_results, + page_token=page_token, + prefix=prefix, + delimiter=delimiter, + start_offset=start_offset, + end_offset=end_offset, + include_trailing_delimiter=include_trailing_delimiter, + versions=versions, + projection=projection, + fields=fields, + page_size=page_size, + timeout=timeout, + retry=retry, + match_glob=match_glob, + include_folders_as_prefixes=include_folders_as_prefixes, + soft_deleted=soft_deleted, + filter_=filter_, + ) def list_notifications( self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY @@ -1605,12 +1637,15 @@ def list_notifications( with self._create_trace_span(name="Storage.Bucket.listNotifications"): client = self._require_client(client) path = self.path + "/notificationConfigs" - iterator = client._list_resource( - path, - _item_to_notification, - timeout=timeout, - retry=retry, - ) + with _list_metrics_context( + "storage.Bucket.list_notifications", "ListNotifications" + ): + iterator = client._list_resource( + path, + _item_to_notification, + timeout=timeout, + retry=retry, + ) iterator.bucket = self return iterator @@ -1648,9 +1683,14 @@ def get_notification( :returns: notification instance. """ with self._create_trace_span(name="Storage.Bucket.getNotification"): - notification = self.notification(notification_id=notification_id) - notification.reload(client=client, timeout=timeout, retry=retry) - return notification + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Bucket.get_notification", + "GetNotification", + ): + notification = self.notification(notification_id=notification_id) + notification.reload(client=client, timeout=timeout, retry=retry) + return notification def delete( self, @@ -1709,61 +1749,64 @@ def delete( """ with self._create_trace_span(name="Storage.Bucket.delete"): client = self._require_client(client) - query_params = {} + with _start_storage_operation( + client, "storage.Bucket.delete", "DeleteBucket" + ): + query_params = {} - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - _add_generation_match_parameters( - query_params, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) - if force: - blobs = list( - self.list_blobs( - max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, + _add_generation_match_parameters( + query_params, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) + if force: + blobs = list( + self.list_blobs( + max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, + client=client, + timeout=timeout, + retry=retry, + versions=True, + ) + ) + if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: + message = ( + "Refusing to delete bucket with more than " + "%d objects. If you actually want to delete " + "this bucket, please delete the objects " + "yourself before calling Bucket.delete()." + ) % (self._MAX_OBJECTS_FOR_ITERATION,) + raise ValueError(message) + + # Ignore 404 errors on delete. + self.delete_blobs( + blobs, + on_error=lambda blob: None, client=client, timeout=timeout, retry=retry, - versions=True, + preserve_generation=True, ) - ) - if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: - message = ( - "Refusing to delete bucket with more than " - "%d objects. If you actually want to delete " - "this bucket, please delete the objects " - "yourself before calling Bucket.delete()." - ) % (self._MAX_OBJECTS_FOR_ITERATION,) - raise ValueError(message) - - # Ignore 404 errors on delete. - self.delete_blobs( - blobs, - on_error=lambda blob: None, - client=client, + + # We intentionally pass `_target_object=None` since a DELETE + # request has no response value (whether in a standard request or + # in a batch request). + client._delete_resource( + self.path, + query_params=query_params, timeout=timeout, retry=retry, - preserve_generation=True, + _target_object=None, ) - - # We intentionally pass `_target_object=None` since a DELETE - # request has no response value (whether in a standard request or - # in a batch request). - client._delete_resource( - self.path, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) - cache = getattr(client, "_bucket_metadata_cache", None) - if cache: - try: - cache.evict(self.name) - except Exception: - pass + cache = getattr(client, "_bucket_metadata_cache", None) + if cache: + try: + cache.evict(self.name) + except Exception: + pass def delete_blob( self, @@ -1838,26 +1881,29 @@ def delete_blob( """ with self._create_trace_span(name="Storage.Bucket.deleteBlob"): client = self._require_client(client) - blob = Blob(blob_name, bucket=self, generation=generation) - - query_params = copy.deepcopy(blob._query_params) - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) - # We intentionally pass `_target_object=None` since a DELETE - # request has no response value (whether in a standard request or - # in a batch request). - client._delete_resource( - blob.path, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) + with _start_storage_operation( + client, "storage.Bucket.delete_blob", "DeleteObject" + ): + blob = Blob(blob_name, bucket=self, generation=generation) + + query_params = copy.deepcopy(blob._query_params) + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) + # We intentionally pass `_target_object=None` since a DELETE + # request has no response value (whether in a standard request or + # in a batch request). + client._delete_resource( + blob.path, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) def delete_blobs( self, @@ -1950,44 +1996,51 @@ def delete_blobs( `on_error` is not passed). """ with self._create_trace_span(name="Storage.Bucket.deleteBlobs"): - _raise_if_len_differs( - len(blobs), - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) - if_generation_match = iter(if_generation_match or []) - if_generation_not_match = iter(if_generation_not_match or []) - if_metageneration_match = iter(if_metageneration_match or []) - if_metageneration_not_match = iter(if_metageneration_not_match or []) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Bucket.delete_blobs", + "DeleteObject", + ): + _raise_if_len_differs( + len(blobs), + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) + if_generation_match = iter(if_generation_match or []) + if_generation_not_match = iter(if_generation_not_match or []) + if_metageneration_match = iter(if_metageneration_match or []) + if_metageneration_not_match = iter(if_metageneration_not_match or []) - for blob in blobs: - try: - blob_name = blob - generation = None - if not isinstance(blob_name, str): - blob_name = blob.name - generation = blob.generation if preserve_generation else None - - self.delete_blob( - blob_name, - client=client, - generation=generation, - if_generation_match=next(if_generation_match, None), - if_generation_not_match=next(if_generation_not_match, None), - if_metageneration_match=next(if_metageneration_match, None), - if_metageneration_not_match=next( - if_metageneration_not_match, None - ), - timeout=timeout, - retry=retry, - ) - except NotFound: - if on_error is not None: - on_error(blob) - else: - raise + for blob in blobs: + try: + blob_name = blob + generation = None + if not isinstance(blob_name, str): + blob_name = blob.name + generation = ( + blob.generation if preserve_generation else None + ) + + self.delete_blob( + blob_name, + client=client, + generation=generation, + if_generation_match=next(if_generation_match, None), + if_generation_not_match=next(if_generation_not_match, None), + if_metageneration_match=next(if_metageneration_match, None), + if_metageneration_not_match=next( + if_metageneration_not_match, None + ), + timeout=timeout, + retry=retry, + ) + except NotFound: + if on_error is not None: + on_error(blob) + else: + raise def copy_blob( self, @@ -2110,57 +2163,60 @@ def copy_blob( """ with self._create_trace_span(name="Storage.Bucket.copyBlob"): client = self._require_client(client) - query_params = {} - - if self.user_project is not None: - query_params["userProject"] = self.user_project - - if source_generation is not None: - query_params["sourceGeneration"] = source_generation - - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - if_source_generation_match=if_source_generation_match, - if_source_generation_not_match=if_source_generation_not_match, - if_source_metageneration_match=if_source_metageneration_match, - if_source_metageneration_not_match=if_source_metageneration_not_match, - ) + with _start_storage_operation( + client, "storage.Bucket.copy_blob", "RewriteObject" + ): + query_params = {} - if new_name is None: - new_name = blob.name + if self.user_project is not None: + query_params["userProject"] = self.user_project - new_blob = Blob(bucket=destination_bucket, name=new_name) + if source_generation is not None: + query_params["sourceGeneration"] = source_generation - if destination_contexts is not None: - if isinstance(destination_contexts, ObjectContexts): - new_blob.contexts = destination_contexts + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, + ) + + if new_name is None: + new_name = blob.name + + new_blob = Blob(bucket=destination_bucket, name=new_name) + + if destination_contexts is not None: + if isinstance(destination_contexts, ObjectContexts): + new_blob.contexts = destination_contexts + else: + raise ValueError( + "destination_contexts must be an ObjectContexts object" + ) + request_body = new_blob._properties.copy() else: - raise ValueError( - "destination_contexts must be an ObjectContexts object" - ) - request_body = new_blob._properties.copy() - else: - request_body = None - - api_path = blob.path + "/copyTo" + new_blob.path - copy_result = client._post_resource( - api_path, - request_body, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=new_blob, - ) + request_body = None - if not preserve_acl: - new_blob.acl.save(acl={}, client=client, timeout=timeout) + api_path = blob.path + "/copyTo" + new_blob.path + copy_result = client._post_resource( + api_path, + request_body, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=new_blob, + ) - new_blob._set_properties(copy_result) - return new_blob + if not preserve_acl: + new_blob.acl.save(acl={}, client=client, timeout=timeout) + + new_blob._set_properties(copy_result) + return new_blob def rename_blob( self, @@ -2275,36 +2331,41 @@ def rename_blob( :returns: The newly-renamed blob. """ with self._create_trace_span(name="Storage.Bucket.renameBlob"): - same_name = blob.name == new_name - - new_blob = self.copy_blob( - blob, - self, - new_name, - client=client, - timeout=timeout, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - if_source_generation_match=if_source_generation_match, - if_source_generation_not_match=if_source_generation_not_match, - if_source_metageneration_match=if_source_metageneration_match, - if_source_metageneration_not_match=if_source_metageneration_not_match, - retry=retry, - ) - - if not same_name: - blob.delete( + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Bucket.rename_blob", + "RewriteObject", + ): + same_name = blob.name == new_name + + new_blob = self.copy_blob( + blob, + self, + new_name, client=client, timeout=timeout, - if_generation_match=if_source_generation_match, - if_generation_not_match=if_source_generation_not_match, - if_metageneration_match=if_source_metageneration_match, - if_metageneration_not_match=if_source_metageneration_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, retry=retry, ) - return new_blob + + if not same_name: + blob.delete( + client=client, + timeout=timeout, + if_generation_match=if_source_generation_match, + if_generation_not_match=if_source_generation_not_match, + if_metageneration_match=if_source_metageneration_match, + if_metageneration_not_match=if_source_metageneration_not_match, + retry=retry, + ) + return new_blob def move_blob( self, @@ -2396,39 +2457,42 @@ def move_blob( """ with self._create_trace_span(name="Storage.Bucket.moveBlob"): client = self._require_client(client) - query_params = {} - - if self.user_project is not None: - query_params["userProject"] = self.user_project - - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - if_source_generation_match=if_source_generation_match, - if_source_generation_not_match=if_source_generation_not_match, - if_source_metageneration_match=if_source_metageneration_match, - if_source_metageneration_not_match=if_source_metageneration_not_match, - ) + with _start_storage_operation( + client, "storage.Bucket.move_blob", "MoveObject" + ): + query_params = {} - new_blob = Blob(bucket=self, name=new_name) - api_path = "{blob_path}/moveTo/o/{new_name}".format( - blob_path=blob.path, new_name=_quote(new_blob.name) - ) + if self.user_project is not None: + query_params["userProject"] = self.user_project - move_result = client._post_resource( - api_path, - None, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=new_blob, - ) + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, + ) - new_blob._set_properties(move_result) - return new_blob + new_blob = Blob(bucket=self, name=new_name) + api_path = "{blob_path}/moveTo/o/{new_name}".format( + blob_path=blob.path, new_name=_quote(new_blob.name) + ) + + move_result = client._post_resource( + api_path, + None, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=new_blob, + ) + + new_blob._set_properties(move_result) + return new_blob def restore_blob( self, @@ -2505,35 +2569,38 @@ def restore_blob( """ with self._create_trace_span(name="Storage.Bucket.restore_blob"): client = self._require_client(client) - query_params = {} - - if self.user_project is not None: - query_params["userProject"] = self.user_project - if generation is not None: - query_params["generation"] = generation - if copy_source_acl is not None: - query_params["copySourceAcl"] = copy_source_acl - if projection is not None: - query_params["projection"] = projection - - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) + with _start_storage_operation( + client, "storage.Bucket.restore_blob", "RestoreObject" + ): + query_params = {} + + if self.user_project is not None: + query_params["userProject"] = self.user_project + if generation is not None: + query_params["generation"] = generation + if copy_source_acl is not None: + query_params["copySourceAcl"] = copy_source_acl + if projection is not None: + query_params["projection"] = projection + + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) - blob = Blob(bucket=self, name=blob_name) - api_response = client._post_resource( - f"{blob.path}/restore", - None, - query_params=query_params, - timeout=timeout, - retry=retry, - ) - blob._set_properties(api_response) - return blob + blob = Blob(bucket=self, name=blob_name) + api_response = client._post_resource( + f"{blob.path}/restore", + None, + query_params=query_params, + timeout=timeout, + retry=retry, + ) + blob._set_properties(api_response) + return blob @property def cors(self): @@ -3400,22 +3467,27 @@ def get_iam_policy( """ with self._create_trace_span(name="Storage.Bucket.getIamPolicy"): client = self._require_client(client) - query_params = {} + with _start_storage_operation( + client, "storage.Bucket.get_iam_policy", "GetIamPolicy" + ): + query_params = {} - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - if requested_policy_version is not None: - query_params["optionsRequestedPolicyVersion"] = requested_policy_version + if requested_policy_version is not None: + query_params["optionsRequestedPolicyVersion"] = ( + requested_policy_version + ) - info = client._get_resource( - f"{self.path}/iam", - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) - return Policy.from_api_repr(info) + info = client._get_resource( + f"{self.path}/iam", + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) + return Policy.from_api_repr(info) def set_iam_policy( self, @@ -3454,25 +3526,28 @@ def set_iam_policy( """ with self._create_trace_span(name="Storage.Bucket.setIamPolicy"): client = self._require_client(client) - query_params = {} - - if self.user_project is not None: - query_params["userProject"] = self.user_project - - path = f"{self.path}/iam" - resource = policy.to_api_repr() - resource["resourceId"] = self.path - - info = client._put_resource( - path, - resource, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) + with _start_storage_operation( + client, "storage.Bucket.set_iam_policy", "SetIamPolicy" + ): + query_params = {} + + if self.user_project is not None: + query_params["userProject"] = self.user_project - return Policy.from_api_repr(info) + path = f"{self.path}/iam" + resource = policy.to_api_repr() + resource["resourceId"] = self.path + + info = client._put_resource( + path, + resource, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) + + return Policy.from_api_repr(info) def test_iam_permissions( self, @@ -3511,20 +3586,25 @@ def test_iam_permissions( """ with self._create_trace_span(name="Storage.Bucket.testIamPermissions"): client = self._require_client(client) - query_params = {"permissions": permissions} - - if self.user_project is not None: - query_params["userProject"] = self.user_project - - path = f"{self.path}/iam/testPermissions" - resp = client._get_resource( - path, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) - return resp.get("permissions", []) + with _start_storage_operation( + client, + "storage.Bucket.test_iam_permissions", + "TestIamPermissions", + ): + query_params = {"permissions": permissions} + + if self.user_project is not None: + query_params["userProject"] = self.user_project + + path = f"{self.path}/iam/testPermissions" + resp = client._get_resource( + path, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) + return resp.get("permissions", []) def make_public( self, @@ -3576,21 +3656,13 @@ def make_public( for each blob. """ with self._create_trace_span(name="Storage.Bucket.makePublic"): - self.acl.all().grant_read() - self.acl.save( - client=client, - timeout=timeout, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) - - if future: - doa = self.default_object_acl - if not doa.loaded: - doa.reload(client=client, timeout=timeout) - doa.all().grant_read() - doa.save( + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Bucket.make_public", + "UpdateBucketACL", + ): + self.acl.all().grant_read() + self.acl.save( client=client, timeout=timeout, if_metageneration_match=if_metageneration_match, @@ -3598,31 +3670,44 @@ def make_public( retry=retry, ) - if recursive: - blobs = list( - self.list_blobs( - projection="full", - max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, + if future: + doa = self.default_object_acl + if not doa.loaded: + doa.reload(client=client, timeout=timeout) + doa.all().grant_read() + doa.save( client=client, timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, ) - ) - if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: - message = ( - "Refusing to make public recursively with more than " - "%d objects. If you actually want to make every object " - "in this bucket public, iterate through the blobs " - "returned by 'Bucket.list_blobs()' and call " - "'make_public' on each one." - ) % (self._MAX_OBJECTS_FOR_ITERATION,) - raise ValueError(message) - for blob in blobs: - blob.acl.all().grant_read() - blob.acl.save( - client=client, - timeout=timeout, + if recursive: + blobs = list( + self.list_blobs( + projection="full", + max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, + client=client, + timeout=timeout, + ) ) + if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: + message = ( + "Refusing to make public recursively with more than " + "%d objects. If you actually want to make every object " + "in this bucket public, iterate through the blobs " + "returned by 'Bucket.list_blobs()' and call " + "'make_public' on each one." + ) % (self._MAX_OBJECTS_FOR_ITERATION,) + raise ValueError(message) + + for blob in blobs: + blob.acl.all().grant_read() + blob.acl.save( + client=client, + timeout=timeout, + ) def make_private( self, @@ -3673,21 +3758,13 @@ def make_private( for each blob. """ with self._create_trace_span(name="Storage.Bucket.makePrivate"): - self.acl.all().revoke_read() - self.acl.save( - client=client, - timeout=timeout, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) - - if future: - doa = self.default_object_acl - if not doa.loaded: - doa.reload(client=client, timeout=timeout) - doa.all().revoke_read() - doa.save( + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Bucket.make_private", + "UpdateBucketACL", + ): + self.acl.all().revoke_read() + self.acl.save( client=client, timeout=timeout, if_metageneration_match=if_metageneration_match, @@ -3695,28 +3772,41 @@ def make_private( retry=retry, ) - if recursive: - blobs = list( - self.list_blobs( - projection="full", - max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, + if future: + doa = self.default_object_acl + if not doa.loaded: + doa.reload(client=client, timeout=timeout) + doa.all().revoke_read() + doa.save( client=client, timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, ) - ) - if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: - message = ( - "Refusing to make private recursively with more than " - "%d objects. If you actually want to make every object " - "in this bucket private, iterate through the blobs " - "returned by 'Bucket.list_blobs()' and call " - "'make_private' on each one." - ) % (self._MAX_OBJECTS_FOR_ITERATION,) - raise ValueError(message) - for blob in blobs: - blob.acl.all().revoke_read() - blob.acl.save(client=client, timeout=timeout) + if recursive: + blobs = list( + self.list_blobs( + projection="full", + max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, + client=client, + timeout=timeout, + ) + ) + if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: + message = ( + "Refusing to make private recursively with more than " + "%d objects. If you actually want to make every object " + "in this bucket private, iterate through the blobs " + "returned by 'Bucket.list_blobs()' and call " + "'make_private' on each one." + ) % (self._MAX_OBJECTS_FOR_ITERATION,) + raise ValueError(message) + + for blob in blobs: + blob.acl.all().revoke_read() + blob.acl.save(client=client, timeout=timeout) def generate_upload_policy(self, conditions, expiration=None, client=None): """Create a signed upload policy for uploading objects. @@ -3797,38 +3887,43 @@ def lock_retention_policy( if the bucket's retention policy is already locked. """ with self._create_trace_span(name="Storage.Bucket.lockRetentionPolicy"): - if "metageneration" not in self._properties: - raise ValueError( - "Bucket has no retention policy assigned: try 'reload'?" - ) + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.Bucket.lock_retention_policy", + "LockBucketRetentionPolicy", + ): + if "metageneration" not in self._properties: + raise ValueError( + "Bucket has no retention policy assigned: try 'reload'?" + ) - policy = self._properties.get("retentionPolicy") + policy = self._properties.get("retentionPolicy") - if policy is None: - raise ValueError( - "Bucket has no retention policy assigned: try 'reload'?" - ) + if policy is None: + raise ValueError( + "Bucket has no retention policy assigned: try 'reload'?" + ) - if policy.get("isLocked"): - raise ValueError("Bucket's retention policy is already locked.") + if policy.get("isLocked"): + raise ValueError("Bucket's retention policy is already locked.") - client = self._require_client(client) + client = self._require_client(client) - query_params = {"ifMetagenerationMatch": self.metageneration} + query_params = {"ifMetagenerationMatch": self.metageneration} - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - path = f"/b/{self.name}/lockRetentionPolicy" - api_response = client._post_resource( - path, - None, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=self, - ) - self._set_properties(api_response) + path = f"/b/{self.name}/lockRetentionPolicy" + api_response = client._post_resource( + path, + None, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=self, + ) + self._set_properties(api_response) def generate_signed_url( self, diff --git a/packages/google-cloud-storage/google/cloud/storage/client.py b/packages/google-cloud-storage/google/cloud/storage/client.py index f9d354a4d269..052122ee073a 100644 --- a/packages/google-cloud-storage/google/cloud/storage/client.py +++ b/packages/google-cloud-storage/google/cloud/storage/client.py @@ -37,6 +37,7 @@ from google.cloud.storage._helpers import ( _DEFAULT_SCHEME, _DEFAULT_UNIVERSE_DOMAIN, + _LIST_METRICS_CONTEXT, _NOW, _STORAGE_HOST_TEMPLATE, _UTC, @@ -45,6 +46,8 @@ _get_api_endpoint_override, _get_environ_project, _get_storage_emulator_override, + _list_metrics_context, + _start_storage_operation, _virtual_hosted_style_base_url, create_trace_span_helper, ) @@ -140,6 +143,14 @@ class Client(ClientWithProject): metrics. If ``None``, falls back to the ``GCP_STORAGE_PYTHON_ENABLE_OTEL_DEBUG_METRICS`` environment variable, or ``False`` if unset. + + :type meter_provider: :class:`opentelemetry.metrics.MeterProvider` or None + :param meter_provider: + (Optional, Experimental) Custom OpenTelemetry ``MeterProvider`` for + recording client metrics. If ``None``, falls back to the global + OpenTelemetry ``MeterProvider``. Note that custom ``MeterProvider`` + instances are not serialized across ``ProcessPoolExecutor`` workers in + :mod:`~google.cloud.storage.transfer_manager`. """ SCOPE = ( @@ -162,6 +173,7 @@ def __init__( api_key=None, enable_metrics=None, enable_debug_metrics=None, + meter_provider=None, ): self._base_connection = None @@ -181,12 +193,7 @@ def __init__( self._extra_headers = extra_headers self._enable_metrics = enable_metrics self._enable_debug_metrics = enable_debug_metrics - self._metrics_enabled = _opentelemetry_metrics.is_metrics_enabled( - enable_metrics - ) - self._debug_metrics_enabled = _opentelemetry_metrics.is_debug_metrics_enabled( - enable_debug_metrics - ) + self._meter_provider = meter_provider connection_kw_args = {"client_info": client_info} @@ -317,19 +324,50 @@ def __init__( self._connection = connection self._batch_stack = _LocalStack() self._bucket_metadata_cache = BucketMetadataCache(self) + self._metrics_recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=enable_metrics, + enable_debug_metrics=enable_debug_metrics, + meter_provider=meter_provider, + api_endpoint=self.api_endpoint, + rpc_system="http", + ) + if getattr(self, "_http_internal", None) is not None: + try: + self._http_internal._gcs_metrics_recorder = self._metrics_recorder + except Exception: + pass + + @property + def _http(self): + http = super(Client, self)._http + recorder = getattr(self, "_metrics_recorder", None) + if ( + http is not None + and recorder is not None + and getattr(http, "_gcs_metrics_recorder", None) is not recorder + ): + try: + http._gcs_metrics_recorder = recorder + except Exception: + pass + return http @property def metrics_enabled(self) -> bool: """(Experimental) Returns ``True`` if OpenTelemetry metrics recording is active for this client.""" - return self._metrics_enabled + recorder = getattr(self, "_metrics_recorder", None) + return bool(recorder is not None and recorder.standard_enabled) @property def debug_metrics_enabled(self) -> bool: """(Experimental) Returns ``True`` if debug metrics recording is active for this client.""" - return self._debug_metrics_enabled + recorder = getattr(self, "_metrics_recorder", None) + return bool(recorder is not None and recorder.debug_enabled) def close(self): """Close the client and clear any cached metadata or active connections.""" + if hasattr(self, "_metrics_recorder") and self._metrics_recorder is not None: + self._metrics_recorder.shutdown() if hasattr(self, "_bucket_metadata_cache") and self._bucket_metadata_cache: self._bucket_metadata_cache.clear() if hasattr(self._http, "close"): @@ -460,12 +498,17 @@ def get_service_account_email( :returns: service account email address """ with create_trace_span(name="Storage.Client.getServiceAccountEmail"): - if project is None: - project = self.project + with _start_storage_operation( + self, + "storage.Client.get_service_account_email", + "GetServiceAccount", + ): + if project is None: + project = self.project - path = f"/projects/{project}/serviceAccount" - api_response = self._get_resource(path, timeout=timeout, retry=retry) - return api_response["email_address"] + path = f"/projects/{project}/serviceAccount" + api_response = self._get_resource(path, timeout=timeout, retry=retry) + return api_response["email_address"] def bucket(self, bucket_name, user_project=None, generation=None): """Factory constructor for bucket object. @@ -603,9 +646,21 @@ def _list_resource( api_request=kwargs, retry=retry, ): - api_request = functools.partial( + raw_api_request = functools.partial( self._connection.api_request, timeout=timeout, retry=retry ) + list_ctx = _LIST_METRICS_CONTEXT.get() + if ( + list_ctx is not None + and _opentelemetry_metrics.get_active_recorder(self) is not None + ): + client_method, operation = list_ctx + + def api_request(*args, **kw): + with _start_storage_operation(self, client_method, operation): + return raw_api_request(*args, **kw) + else: + api_request = raw_api_request return page_iterator.HTTPIterator( client=self, api_request=api_request, @@ -1007,37 +1062,45 @@ def get_bucket( with create_trace_span_helper( self, bucket_name, name="Storage.Client.getBucket" ): - bucket = self._bucket_arg_to_bucket(bucket_or_name, generation=generation) - try: - bucket.reload( - client=self, - timeout=timeout, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - soft_deleted=soft_deleted, + with _start_storage_operation( + self, "storage.Client.get_bucket", "GetBucket" + ): + bucket = self._bucket_arg_to_bucket( + bucket_or_name, generation=generation ) - except api_exceptions.Forbidden: + try: + bucket.reload( + client=self, + timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + soft_deleted=soft_deleted, + ) + except api_exceptions.Forbidden: + if ( + hasattr(self, "_bucket_metadata_cache") + and self._bucket_metadata_cache + ): + try: + self._bucket_metadata_cache.update_cache( + bucket_name, + f"//storage.googleapis.com/projects/_/buckets/{bucket_name}", + "global", + ) + except Exception: + pass + raise + if ( hasattr(self, "_bucket_metadata_cache") and self._bucket_metadata_cache ): try: - self._bucket_metadata_cache.update_cache( - bucket_name, - f"//storage.googleapis.com/projects/_/buckets/{bucket_name}", - "global", - ) + self._bucket_metadata_cache.update_from_bucket(bucket) except Exception: pass - raise - - if hasattr(self, "_bucket_metadata_cache") and self._bucket_metadata_cache: - try: - self._bucket_metadata_cache.update_from_bucket(bucket) - except Exception: - pass - return bucket + return bucket def lookup_bucket( self, @@ -1078,39 +1141,42 @@ def lookup_bucket( with create_trace_span_helper( self, bucket_name, name="Storage.Client.lookupBucket" ): - try: - bucket = self.get_bucket( - bucket_name, - timeout=timeout, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) - if ( - hasattr(self, "_bucket_metadata_cache") - and self._bucket_metadata_cache - ): - try: - self._bucket_metadata_cache.update_from_bucket(bucket) - except Exception: - pass - return bucket - except api_exceptions.Forbidden: - if ( - hasattr(self, "_bucket_metadata_cache") - and self._bucket_metadata_cache - ): - try: - self._bucket_metadata_cache.update_cache( - bucket_name, - f"//storage.googleapis.com/projects/_/buckets/{bucket_name}", - "global", - ) - except Exception: - pass - raise - except NotFound: - return None + with _start_storage_operation( + self, "storage.Client.lookup_bucket", "GetBucket" + ): + try: + bucket = self.get_bucket( + bucket_name, + timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + ) + if ( + hasattr(self, "_bucket_metadata_cache") + and self._bucket_metadata_cache + ): + try: + self._bucket_metadata_cache.update_from_bucket(bucket) + except Exception: + pass + return bucket + except api_exceptions.Forbidden: + if ( + hasattr(self, "_bucket_metadata_cache") + and self._bucket_metadata_cache + ): + try: + self._bucket_metadata_cache.update_cache( + bucket_name, + f"//storage.googleapis.com/projects/_/buckets/{bucket_name}", + "global", + ) + except Exception: + pass + raise + except NotFound: + return None def create_bucket( self, @@ -1193,75 +1259,85 @@ def create_bucket( If the bucket already exists. """ with create_trace_span(name="Storage.Client.createBucket"): - bucket = self._bucket_arg_to_bucket(bucket_or_name) - query_params = {} + with _start_storage_operation( + self, "storage.Client.create_bucket", "CreateBucket" + ): + bucket = self._bucket_arg_to_bucket(bucket_or_name) + query_params = {} - if project is None: - project = self.project - - # Use no project if STORAGE_EMULATOR_HOST is set - if self._is_emulator_set: - if project is None: - project = _get_environ_project() if project is None: - project = "" - - # Only include the project parameter if a project is set. - # If a project is not set, falls back to API validation (BadRequest). - if project is not None: - query_params = {"project": project} - - if requester_pays is not None: - warnings.warn( - "requester_pays arg is deprecated. Use Bucket().requester_pays instead.", - PendingDeprecationWarning, - stacklevel=1, - ) - bucket.requester_pays = requester_pays + project = self.project + + # Use no project if STORAGE_EMULATOR_HOST is set + if self._is_emulator_set: + if project is None: + project = _get_environ_project() + if project is None: + project = "" + + # Only include the project parameter if a project is set. + # If a project is not set, falls back to API validation (BadRequest). + if project is not None: + query_params = {"project": project} + + if requester_pays is not None: + warnings.warn( + "requester_pays arg is deprecated. Use Bucket().requester_pays instead.", + PendingDeprecationWarning, + stacklevel=1, + ) + bucket.requester_pays = requester_pays + + if predefined_acl is not None: + predefined_acl = BucketACL.validate_predefined(predefined_acl) + query_params["predefinedAcl"] = predefined_acl + + if predefined_default_object_acl is not None: + predefined_default_object_acl = ( + DefaultObjectACL.validate_predefined( + predefined_default_object_acl + ) + ) + query_params["predefinedDefaultObjectAcl"] = ( + predefined_default_object_acl + ) - if predefined_acl is not None: - predefined_acl = BucketACL.validate_predefined(predefined_acl) - query_params["predefinedAcl"] = predefined_acl + if user_project is not None: + query_params["userProject"] = user_project - if predefined_default_object_acl is not None: - predefined_default_object_acl = DefaultObjectACL.validate_predefined( - predefined_default_object_acl - ) - query_params["predefinedDefaultObjectAcl"] = ( - predefined_default_object_acl - ) - - if user_project is not None: - query_params["userProject"] = user_project + if enable_object_retention: + query_params["enableObjectRetention"] = enable_object_retention - if enable_object_retention: - query_params["enableObjectRetention"] = enable_object_retention + properties = {key: bucket._properties[key] for key in bucket._changes} + properties["name"] = bucket.name - properties = {key: bucket._properties[key] for key in bucket._changes} - properties["name"] = bucket.name + if location is not None: + properties["location"] = location - if location is not None: - properties["location"] = location + if data_locations is not None: + properties["customPlacementConfig"] = { + "dataLocations": data_locations + } - if data_locations is not None: - properties["customPlacementConfig"] = {"dataLocations": data_locations} - - api_response = self._post_resource( - "/b", - properties, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=bucket, - ) + api_response = self._post_resource( + "/b", + properties, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=bucket, + ) - bucket._set_properties(api_response) - if hasattr(self, "_bucket_metadata_cache") and self._bucket_metadata_cache: - try: - self._bucket_metadata_cache.update_from_bucket(bucket) - except Exception: - pass - return bucket + bucket._set_properties(api_response) + if ( + hasattr(self, "_bucket_metadata_cache") + and self._bucket_metadata_cache + ): + try: + self._bucket_metadata_cache.update_from_bucket(bucket) + except Exception: + pass + return bucket def download_blob_to_file( self, @@ -1372,26 +1448,29 @@ def download_blob_to_file( bucket_name, name="Storage.Client.downloadBlobToFile", ): - if not isinstance(blob_or_uri, Blob): - blob_or_uri = Blob.from_uri(blob_or_uri) - - blob_or_uri._prep_and_do_download( - file_obj, - client=self, - start=start, - end=end, - raw_download=raw_download, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - single_shot_download=single_shot_download, - ) + with _start_storage_operation( + self, "storage.Client.download_blob_to_file", "ReadObject" + ): + if not isinstance(blob_or_uri, Blob): + blob_or_uri = Blob.from_uri(blob_or_uri) + + blob_or_uri._prep_and_do_download( + file_obj, + client=self, + start=start, + end=end, + raw_download=raw_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + single_shot_download=single_shot_download, + ) def list_blobs( self, @@ -1581,17 +1660,18 @@ def list_blobs( extra_params["userProject"] = bucket.user_project path = bucket.path + "/o" - iterator = self._list_resource( - path, - _item_to_blob, - page_token=page_token, - max_results=max_results, - extra_params=extra_params, - page_start=_blobs_page_start, - page_size=page_size, - timeout=timeout, - retry=retry, - ) + with _list_metrics_context("storage.Client.list_blobs", "ListObjects"): + iterator = self._list_resource( + path, + _item_to_blob, + page_token=page_token, + max_results=max_results, + extra_params=extra_params, + page_start=_blobs_page_start, + page_size=page_size, + timeout=timeout, + retry=retry, + ) iterator.bucket = bucket iterator.prefixes = set() return iterator @@ -1713,17 +1793,18 @@ def list_buckets( if return_partial_success is not None: extra_params["returnPartialSuccess"] = return_partial_success - iterator = self._list_resource( - "/b", - _item_to_bucket, - page_token=page_token, - max_results=max_results, - extra_params=extra_params, - page_size=page_size, - timeout=timeout, - retry=retry, - page_start=_buckets_page_start, - ) + with _list_metrics_context("storage.Client.list_buckets", "ListBuckets"): + iterator = self._list_resource( + "/b", + _item_to_bucket, + page_token=page_token, + max_results=max_results, + extra_params=extra_params, + page_size=page_size, + timeout=timeout, + retry=retry, + page_start=_buckets_page_start, + ) return iterator def restore_bucket( @@ -1772,24 +1853,27 @@ def restore_bucket( :rtype: :class:`google.cloud.storage.bucket.Bucket` :returns: The restored Bucket. """ - query_params = {"generation": generation, "projection": projection} + with _start_storage_operation( + self, "storage.Client.restore_bucket", "RestoreBucket" + ): + query_params = {"generation": generation, "projection": projection} - _add_generation_match_parameters( - query_params, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) + _add_generation_match_parameters( + query_params, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) - bucket = self.bucket(bucket_name) - api_response = self._post_resource( - f"{bucket.path}/restore", - None, - query_params=query_params, - timeout=timeout, - retry=retry, - ) - bucket._set_properties(api_response) - return bucket + bucket = self.bucket(bucket_name) + api_response = self._post_resource( + f"{bucket.path}/restore", + None, + query_params=query_params, + timeout=timeout, + retry=retry, + ) + bucket._set_properties(api_response) + return bucket def create_hmac_key( self, @@ -1835,26 +1919,29 @@ def create_hmac_key( :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string. """ with create_trace_span(name="Storage.Client.createHmacKey"): - if project_id is None: - project_id = self.project - - path = f"/projects/{project_id}/hmacKeys" - qs_params = {"serviceAccountEmail": service_account_email} - - if user_project is not None: - qs_params["userProject"] = user_project - - api_response = self._post_resource( - path, - None, - query_params=qs_params, - timeout=timeout, - retry=retry, - ) - metadata = HMACKeyMetadata(self) - metadata._properties = api_response["metadata"] - secret = api_response["secret"] - return metadata, secret + with _start_storage_operation( + self, "storage.Client.create_hmac_key", "CreateHMACKey" + ): + if project_id is None: + project_id = self.project + + path = f"/projects/{project_id}/hmacKeys" + qs_params = {"serviceAccountEmail": service_account_email} + + if user_project is not None: + qs_params["userProject"] = user_project + + api_response = self._post_resource( + path, + None, + query_params=qs_params, + timeout=timeout, + retry=retry, + ) + metadata = HMACKeyMetadata(self) + metadata._properties = api_response["metadata"] + secret = api_response["secret"] + return metadata, secret def list_hmac_keys( self, @@ -1917,14 +2004,15 @@ def list_hmac_keys( if user_project is not None: extra_params["userProject"] = user_project - return self._list_resource( - path, - _item_to_hmac_key_metadata, - max_results=max_results, - extra_params=extra_params, - timeout=timeout, - retry=retry, - ) + with _list_metrics_context("storage.Client.list_hmac_keys", "ListHMACKeys"): + return self._list_resource( + path, + _item_to_hmac_key_metadata, + max_results=max_results, + extra_params=extra_params, + timeout=timeout, + retry=retry, + ) def get_hmac_key_metadata( self, access_id, project_id=None, user_project=None, timeout=_DEFAULT_TIMEOUT @@ -1947,9 +2035,12 @@ def get_hmac_key_metadata( :param user_project: (Optional) This parameter is currently ignored. """ with create_trace_span(name="Storage.Client.getHmacKeyMetadata"): - metadata = HMACKeyMetadata(self, access_id, project_id, user_project) - metadata.reload(timeout=timeout) # raises NotFound for missing key - return metadata + with _start_storage_operation( + self, "storage.Client.get_hmac_key_metadata", "GetHMACKey" + ): + metadata = HMACKeyMetadata(self, access_id, project_id, user_project) + metadata.reload(timeout=timeout) # raises NotFound for missing key + return metadata def generate_signed_post_policy_v4( self, diff --git a/packages/google-cloud-storage/google/cloud/storage/fileio.py b/packages/google-cloud-storage/google/cloud/storage/fileio.py index d721b6e3de4f..fc230fd1aec9 100644 --- a/packages/google-cloud-storage/google/cloud/storage/fileio.py +++ b/packages/google-cloud-storage/google/cloud/storage/fileio.py @@ -15,9 +15,11 @@ """Module for file-like access of blobs, usually invoked via Blob.open().""" import io +import time from google.api_core.exceptions import RequestRangeNotSatisfiable +from google.cloud.storage import _helpers, _opentelemetry_metrics from google.cloud.storage.retry import DEFAULT_RETRY, ConditionalRetryPolicy # Resumable uploads require a chunk size of precisely a multiple of 256 KiB. @@ -52,7 +54,82 @@ } -class BlobReader(io.BufferedIOBase): +class _StreamMetricsMixin(object): + """Tracks stream-lifetime logical operation and payload size metrics.""" + + def _init_stream_metrics(self, blob, default_client_method, operation): + self._metrics_client = _helpers._resolve_operation_client(blob) + if ( + _helpers._SUPPRESS_INTERNAL_METRICS.get() + or _opentelemetry_metrics.get_current_operation() is not None + ): + self._metrics_recorder = None + else: + self._metrics_recorder = _opentelemetry_metrics.get_active_recorder( + self._metrics_client + ) + self._metrics_client_method = ( + _helpers._CLIENT_METHOD_OVERRIDE.get() or default_client_method + ) + self._metrics_operation = operation + self._metrics_start = time.perf_counter() + self._metrics_finalized = False + self._stream_error = None + + def _stream_metrics_context(self): + return _helpers._bind_suboperation_context( + self._metrics_client, + self._metrics_client_method, + self._metrics_operation, + ) + + def _finalize_metrics( + self, + exc=None, + default_error_type="OK", + record_body=False, + body_bytes=0, + is_read=True, + ): + if self._metrics_finalized: + return + self._metrics_finalized = True + recorder = self._metrics_recorder + if recorder is None: + return + if record_body: + if is_read: + recorder.record_response_body_size( + body_bytes, + operation=self._metrics_operation, + client_method=self._metrics_client_method, + ) + else: + recorder.record_request_body_size( + body_bytes, + operation=self._metrics_operation, + client_method=self._metrics_client_method, + ) + error_type = ( + _opentelemetry_metrics.classify_error_type(exc) + if exc is not None + else default_error_type + ) + recorder.record_logical_operation( + duration=time.perf_counter() - self._metrics_start, + client_method=self._metrics_client_method, + operation=self._metrics_operation, + error_type=error_type, + ) + + def __del__(self): + self._metrics_finalized = True + buf = getattr(self, "_buffer", None) + if buf is not None: + buf.close() + + +class BlobReader(_StreamMetricsMixin, io.BufferedIOBase): """A file-like object that reads from a blob. :type blob: 'google.cloud.storage.blob.Blob' @@ -106,15 +183,22 @@ class BlobReader(io.BufferedIOBase): """ def __init__(self, blob, chunk_size=None, retry=DEFAULT_RETRY, **download_kwargs): - for kwarg in download_kwargs: - if kwarg not in VALID_DOWNLOAD_KWARGS: - raise ValueError( - f"BlobReader does not support keyword argument {kwarg}." - ) - self._blob = blob self._pos = 0 self._buffer = io.BytesIO() + self._bytes_read = 0 + self._init_stream_metrics(blob, "storage.fileio.BlobReader", "ReadObject") + try: + for kwarg in download_kwargs: + if kwarg not in VALID_DOWNLOAD_KWARGS: + raise ValueError( + f"BlobReader does not support keyword argument {kwarg}." + ) + except BaseException as exc: + self._buffer.close() + self._finalize_metrics(exc=exc, record_body=False, is_read=True) + raise + self._chunk_size = chunk_size or blob.chunk_size or DEFAULT_CHUNK_SIZE self._retry = retry self._download_kwargs = download_kwargs @@ -142,17 +226,21 @@ def read(self, size=-1): # chunked downloads, and the server only knows the checksum of the # entire file. try: - result += self._blob.download_as_bytes( - start=fetch_start, - end=fetch_end, - checksum=None, - retry=self._retry, - **self._download_kwargs, - ) + with self._stream_metrics_context(): + result += self._blob.download_as_bytes( + start=fetch_start, + end=fetch_end, + checksum=None, + retry=self._retry, + **self._download_kwargs, + ) except RequestRangeNotSatisfiable: # We've reached the end of the file. Python file objects should # return an empty response in this case, not raise an error. pass + except BaseException as exc: + self._stream_error = exc + raise # If more bytes were read than is immediately needed, buffer the # remainder and then trim the result. @@ -162,6 +250,8 @@ def read(self, size=-1): result = result[:size] # Increment relative offset by true amount read. self._pos += len(result) - read_size + self._bytes_read += len(result) + self._stream_error = None return result def read1(self, size=-1): @@ -182,7 +272,12 @@ def seek(self, pos, whence=0): for k, v in self._download_kwargs.items() if (k != "raw_download" and k != "single_shot_download") } - self._blob.reload(**reload_kwargs) + try: + with self._stream_metrics_context(): + self._blob.reload(**reload_kwargs) + except BaseException as exc: + self._stream_error = exc + raise initial_offset = self._pos + self._buffer.tell() @@ -193,7 +288,9 @@ def seek(self, pos, whence=0): elif whence == 2: target_pos = self._blob.size + pos if whence not in {0, 1, 2}: - raise ValueError("invalid whence value") + exc = ValueError("invalid whence value") + self._stream_error = exc + raise exc if target_pos > self._blob.size: target_pos = self._blob.size @@ -210,10 +307,17 @@ def seek(self, pos, whence=0): # relative offset <= target position <= size of file. difference = target_pos - initial_offset new_pos = self._pos + self._buffer.seek(difference, 1) + self._stream_error = None return new_pos def close(self): self._buffer.close() + self._finalize_metrics( + exc=self._stream_error, + record_body=(self._stream_error is None or self._bytes_read > 0), + body_bytes=self._bytes_read, + is_read=True, + ) @property def closed(self): @@ -229,7 +333,7 @@ def seekable(self): return True -class BlobWriter(io.BufferedIOBase): +class BlobWriter(_StreamMetricsMixin, io.BufferedIOBase): """A file-like object that writes to a blob. :type blob: 'google.cloud.storage.blob.Blob' @@ -302,17 +406,24 @@ def __init__( retry=DEFAULT_RETRY, **upload_kwargs, ): - for kwarg in upload_kwargs: - if kwarg not in VALID_UPLOAD_KWARGS: - raise ValueError( - f"BlobWriter does not support keyword argument {kwarg}." - ) self._blob = blob self._buffer = SlidingBuffer() self._upload_and_transport = None - # Resumable uploads require a chunk size of a multiple of 256KiB. - # self._chunk_size must not be changed after the upload is initiated. - self._chunk_size = chunk_size or blob.chunk_size or DEFAULT_CHUNK_SIZE + self._bytes_written = 0 + self._init_stream_metrics(blob, "storage.fileio.BlobWriter", "WriteObject") + try: + for kwarg in upload_kwargs: + if kwarg not in VALID_UPLOAD_KWARGS: + raise ValueError( + f"BlobWriter does not support keyword argument {kwarg}." + ) + # Resumable uploads require a chunk size of a multiple of 256KiB. + # self._chunk_size must not be changed after the upload is initiated. + self._chunk_size = chunk_size or blob.chunk_size or DEFAULT_CHUNK_SIZE + except BaseException as exc: + self._buffer.close() + self._finalize_metrics(exc=exc, record_body=False, is_read=False) + raise self._ignore_flush = ignore_flush self._retry = retry self._upload_kwargs = upload_kwargs @@ -346,11 +457,17 @@ def write(self, b): self._checkClosed() # Raises ValueError if closed. pos = self._buffer.write(b) + self._bytes_written += pos # If there is enough content, upload chunks. num_chunks = len(self._buffer) // self._chunk_size if num_chunks: - self._upload_chunks_from_buffer(num_chunks) + try: + with self._stream_metrics_context(): + self._upload_chunks_from_buffer(num_chunks) + except BaseException as exc: + self._stream_error = exc + raise return pos @@ -403,6 +520,7 @@ def _upload_chunks_from_buffer(self, num_chunks): # Wipe the buffer of chunks uploaded, preserving any remaining data. self._buffer.flush() + self._stream_error = None def tell(self): return self._buffer.tell() + len(self._buffer) @@ -419,15 +537,42 @@ def flush(self): def close(self): if not self._buffer.closed: - self._upload_chunks_from_buffer(1) + try: + with self._stream_metrics_context(): + self._upload_chunks_from_buffer(1) + except BaseException as exc: + if self._stream_error is None or not isinstance(exc, ValueError): + self._stream_error = exc + self._buffer.close() + self._finalize_metrics( + exc=self._stream_error, record_body=False, is_read=False + ) + raise + self._buffer.close() + self._finalize_metrics( + exc=None, + record_body=True, + body_bytes=self._bytes_written, + is_read=False, + ) + return self._buffer.close() def terminate(self): """Cancel the ResumableUpload.""" - if self._upload_and_transport: - upload, transport = self._upload_and_transport - transport.delete(upload.upload_url) - self._buffer.close() + try: + if self._upload_and_transport: + upload, transport = self._upload_and_transport + with _helpers._suppress_internal_metrics(): + transport.delete(upload.upload_url) + finally: + self._buffer.close() + self._finalize_metrics( + exc=self._stream_error, + default_error_type="CANCELLED", + record_body=False, + is_read=False, + ) def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: @@ -548,3 +693,63 @@ def _checkClosed(self): @property def closed(self): return self._buffer.closed + + +class _MetricsTextIOWrapper(io.TextIOWrapper): + """TextIOWrapper subclass propagating text-layer errors and context-exit aborts.""" + + def _capture_text_op(self, fn, *args, **kwargs): + try: + result = fn(*args, **kwargs) + except BaseException as exc: + raw = self.buffer + if raw is not None: + raw._stream_error = exc + raise + self.buffer._stream_error = None + return result + + def read(self, *args, **kwargs): + return self._capture_text_op(super().read, *args, **kwargs) + + def readline(self, *args, **kwargs): + return self._capture_text_op(super().readline, *args, **kwargs) + + def __next__(self): + try: + line = super().__next__() + except StopIteration: + self.buffer._stream_error = None + raise + except BaseException as exc: + raw = self.buffer + if raw is not None: + raw._stream_error = exc + raise + self.buffer._stream_error = None + return line + + def write(self, *args, **kwargs): + return self._capture_text_op(super().write, *args, **kwargs) + + def writelines(self, *args, **kwargs): + return self._capture_text_op(super().writelines, *args, **kwargs) + + def seek(self, *args, **kwargs): + return self._capture_text_op(super().seek, *args, **kwargs) + + def truncate(self, *args, **kwargs): + return self._capture_text_op(super().truncate, *args, **kwargs) + + def __exit__(self, exc_type, exc_val, exc_tb): + raw = self.buffer + if exc_type is not None and isinstance(raw, BlobWriter) and not raw.closed: + raw.terminate() + return + super().__exit__(exc_type, exc_val, exc_tb) + + def __del__(self): + raw = self.buffer + if raw is not None and not raw.closed: + raw._metrics_finalized = True + raw._buffer.close() diff --git a/packages/google-cloud-storage/google/cloud/storage/hmac_key.py b/packages/google-cloud-storage/google/cloud/storage/hmac_key.py index 0976b5e5a8e6..bdabcc19118a 100644 --- a/packages/google-cloud-storage/google/cloud/storage/hmac_key.py +++ b/packages/google-cloud-storage/google/cloud/storage/hmac_key.py @@ -20,6 +20,7 @@ from google.cloud._helpers import _rfc3339_nanos_to_datetime from google.cloud.exceptions import NotFound +from google.cloud.storage._helpers import _start_storage_operation from google.cloud.storage._opentelemetry_tracing import create_trace_span from google.cloud.storage.constants import _DEFAULT_TIMEOUT from google.cloud.storage.retry import DEFAULT_RETRY, DEFAULT_RETRY_IF_ETAG_IN_JSON @@ -203,22 +204,25 @@ def exists(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): :returns: True if the key exists in Cloud Storage. """ with create_trace_span(name="Storage.HmacKey.exists"): - try: - qs_params = {} - - if self.user_project is not None: - qs_params["userProject"] = self.user_project - - self._client._get_resource( - self.path, - query_params=qs_params, - timeout=timeout, - retry=retry, - ) - except NotFound: - return False - else: - return True + with _start_storage_operation( + self._client, "storage.HMACKeyMetadata.exists", "GetHMACKey" + ): + try: + qs_params = {} + + if self.user_project is not None: + qs_params["userProject"] = self.user_project + + self._client._get_resource( + self.path, + query_params=qs_params, + timeout=timeout, + retry=retry, + ) + except NotFound: + return False + else: + return True def reload(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): """Reload properties from Cloud Storage. @@ -236,17 +240,20 @@ def reload(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): if the key does not exist on the back-end. """ with create_trace_span(name="Storage.HmacKey.reload"): - qs_params = {} + with _start_storage_operation( + self._client, "storage.HMACKeyMetadata.reload", "GetHMACKey" + ): + qs_params = {} - if self.user_project is not None: - qs_params["userProject"] = self.user_project + if self.user_project is not None: + qs_params["userProject"] = self.user_project - self._properties = self._client._get_resource( - self.path, - query_params=qs_params, - timeout=timeout, - retry=retry, - ) + self._properties = self._client._get_resource( + self.path, + query_params=qs_params, + timeout=timeout, + retry=retry, + ) def update(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY_IF_ETAG_IN_JSON): """Save writable properties to Cloud Storage. @@ -264,18 +271,21 @@ def update(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY_IF_ETAG_IN_JSON): if the key does not exist on the back-end. """ with create_trace_span(name="Storage.HmacKey.update"): - qs_params = {} - if self.user_project is not None: - qs_params["userProject"] = self.user_project - - payload = {"state": self.state} - self._properties = self._client._put_resource( - self.path, - payload, - query_params=qs_params, - timeout=timeout, - retry=retry, - ) + with _start_storage_operation( + self._client, "storage.HMACKeyMetadata.update", "UpdateHMACKey" + ): + qs_params = {} + if self.user_project is not None: + qs_params["userProject"] = self.user_project + + payload = {"state": self.state} + self._properties = self._client._put_resource( + self.path, + payload, + query_params=qs_params, + timeout=timeout, + retry=retry, + ) def delete(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): """Delete the key from Cloud Storage. @@ -293,13 +303,16 @@ def delete(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): if the key does not exist on the back-end. """ with create_trace_span(name="Storage.HmacKey.delete"): - qs_params = {} - if self.user_project is not None: - qs_params["userProject"] = self.user_project - - self._client._delete_resource( - self.path, - query_params=qs_params, - timeout=timeout, - retry=retry, - ) + with _start_storage_operation( + self._client, "storage.HMACKeyMetadata.delete", "DeleteHMACKey" + ): + qs_params = {} + if self.user_project is not None: + qs_params["userProject"] = self.user_project + + self._client._delete_resource( + self.path, + query_params=qs_params, + timeout=timeout, + retry=retry, + ) diff --git a/packages/google-cloud-storage/google/cloud/storage/notification.py b/packages/google-cloud-storage/google/cloud/storage/notification.py index 21588f6eda4e..0d4319444a6f 100644 --- a/packages/google-cloud-storage/google/cloud/storage/notification.py +++ b/packages/google-cloud-storage/google/cloud/storage/notification.py @@ -21,6 +21,10 @@ from google.api_core.exceptions import NotFound +from google.cloud.storage._helpers import ( + _resolve_operation_client, + _start_storage_operation, +) from google.cloud.storage._opentelemetry_tracing import create_trace_span from google.cloud.storage.constants import _DEFAULT_TIMEOUT from google.cloud.storage.retry import DEFAULT_RETRY @@ -254,35 +258,40 @@ def create(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=None): :raises ValueError: if the notification already exists. """ with create_trace_span(name="Storage.BucketNotification.create"): - if self.notification_id is not None: - raise ValueError( - f"notification_id already set to {self.notification_id}; must be None to create a Notification." # noqa: E702 - ) - - client = self._require_client(client) - - query_params = {} - if self.bucket.user_project is not None: - query_params["userProject"] = self.bucket.user_project - - path = f"/b/{self.bucket.name}/notificationConfigs" - properties = self._properties.copy() - - if self.topic_name is None: - properties["topic"] = _TOPIC_REF_FMT.format(self.topic_project, "") - else: - properties["topic"] = _TOPIC_REF_FMT.format( - self.topic_project, self.topic_name + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.BucketNotification.create", + "CreateNotification", + ): + if self.notification_id is not None: + raise ValueError( + f"notification_id already set to {self.notification_id}; must be None to create a Notification." # noqa: E702 + ) + + client = self._require_client(client) + + query_params = {} + if self.bucket.user_project is not None: + query_params["userProject"] = self.bucket.user_project + + path = f"/b/{self.bucket.name}/notificationConfigs" + properties = self._properties.copy() + + if self.topic_name is None: + properties["topic"] = _TOPIC_REF_FMT.format(self.topic_project, "") + else: + properties["topic"] = _TOPIC_REF_FMT.format( + self.topic_project, self.topic_name + ) + + self._properties = client._post_resource( + path, + properties, + query_params=query_params, + timeout=timeout, + retry=retry, ) - self._properties = client._post_resource( - path, - properties, - query_params=query_params, - timeout=timeout, - retry=retry, - ) - def exists(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): """Test whether this notification exists. @@ -310,28 +319,33 @@ def exists(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): :raises ValueError: if the notification has no ID. """ with create_trace_span(name="Storage.BucketNotification.exists"): - if self.notification_id is None: - raise ValueError( - "Notification ID not set: set an explicit notification_id" - ) - - client = self._require_client(client) - - query_params = {} - if self.bucket.user_project is not None: - query_params["userProject"] = self.bucket.user_project - - try: - client._get_resource( - self.path, - query_params=query_params, - timeout=timeout, - retry=retry, - ) - except NotFound: - return False - else: - return True + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.BucketNotification.exists", + "GetNotification", + ): + if self.notification_id is None: + raise ValueError( + "Notification ID not set: set an explicit notification_id" + ) + + client = self._require_client(client) + + query_params = {} + if self.bucket.user_project is not None: + query_params["userProject"] = self.bucket.user_project + + try: + client._get_resource( + self.path, + query_params=query_params, + timeout=timeout, + retry=retry, + ) + except NotFound: + return False + else: + return True def reload(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): """Update this notification from the server configuration. @@ -359,24 +373,29 @@ def reload(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): :raises ValueError: if the notification has no ID. """ with create_trace_span(name="Storage.BucketNotification.reload"): - if self.notification_id is None: - raise ValueError( - "Notification ID not set: set an explicit notification_id" + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.BucketNotification.reload", + "GetNotification", + ): + if self.notification_id is None: + raise ValueError( + "Notification ID not set: set an explicit notification_id" + ) + + client = self._require_client(client) + + query_params = {} + if self.bucket.user_project is not None: + query_params["userProject"] = self.bucket.user_project + + response = client._get_resource( + self.path, + query_params=query_params, + timeout=timeout, + retry=retry, ) - - client = self._require_client(client) - - query_params = {} - if self.bucket.user_project is not None: - query_params["userProject"] = self.bucket.user_project - - response = client._get_resource( - self.path, - query_params=query_params, - timeout=timeout, - retry=retry, - ) - self._set_properties(response) + self._set_properties(response) def delete(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): """Delete this notification. @@ -405,24 +424,29 @@ def delete(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): :raises ValueError: if the notification has no ID. """ with create_trace_span(name="Storage.BucketNotification.delete"): - if self.notification_id is None: - raise ValueError( - "Notification ID not set: set an explicit notification_id" + with _start_storage_operation( + _resolve_operation_client(self, client), + "storage.BucketNotification.delete", + "DeleteNotification", + ): + if self.notification_id is None: + raise ValueError( + "Notification ID not set: set an explicit notification_id" + ) + + client = self._require_client(client) + + query_params = {} + if self.bucket.user_project is not None: + query_params["userProject"] = self.bucket.user_project + + client._delete_resource( + self.path, + query_params=query_params, + timeout=timeout, + retry=retry, ) - client = self._require_client(client) - - query_params = {} - if self.bucket.user_project is not None: - query_params["userProject"] = self.bucket.user_project - - client._delete_resource( - self.path, - query_params=query_params, - timeout=timeout, - retry=retry, - ) - def _parse_topic_path(topic_path): """Verify that a topic path is in the correct format. diff --git a/packages/google-cloud-storage/google/cloud/storage/transfer_manager.py b/packages/google-cloud-storage/google/cloud/storage/transfer_manager.py index b376c5138d72..12846bef1a2c 100644 --- a/packages/google-cloud-storage/google/cloud/storage/transfer_manager.py +++ b/packages/google-cloud-storage/google/cloud/storage/transfer_manager.py @@ -14,22 +14,26 @@ """Concurrent media operations.""" +import atexit import base64 import concurrent.futures +import contextvars import copyreg import functools import inspect import io +import multiprocessing.util import os import pickle import struct import warnings +from contextlib import nullcontext from pathlib import Path import google_crc32c from google.api_core import exceptions -from google.cloud.storage import Blob, Client +from google.cloud.storage import Blob, Client, _helpers, _opentelemetry_metrics from google.cloud.storage._media.requests.upload import XMLMPUContainer, XMLMPUPart from google.cloud.storage.blob import _get_host_name, _quote from google.cloud.storage.constants import _DEFAULT_TIMEOUT @@ -68,6 +72,34 @@ _cached_clients = {} +_cleanup_registered_pid = None + + +def _cleanup_cached_clients(): + if not _cached_clients: + return + clients = list(_cached_clients.values()) + _cached_clients.clear() + had_active_metrics = False + for cl in clients: + if getattr(cl, "metrics_enabled", False) or getattr( + cl, "debug_metrics_enabled", False + ): + had_active_metrics = True + try: + cl.close() + except Exception: + pass + if not had_active_metrics: + return + metrics_mod = _opentelemetry_metrics.metrics + if metrics_mod is not None: + try: + provider = metrics_mod.get_meter_provider() + if hasattr(provider, "force_flush"): + provider.force_flush() + except Exception: + pass def _deprecate_threads_param(func): @@ -174,7 +206,9 @@ def upload_many( recreated in each subprocess. The serialization of the Client object for use in subprocesses is an approximation and may not capture every detail of the Client object, especially if the Client was modified after - its initial creation or if `Client._http` was modified in any way. + its initial creation, if `Client._http` was modified in any way, or if a + custom OpenTelemetry `meter_provider` was passed to `Client` (subprocess + workers fall back to the subprocess's global `MeterProvider`). THREAD worker types are observed to be relatively efficient for operations with many small files, but not for operations with large @@ -212,33 +246,55 @@ def upload_many( upload_kwargs["command"] = "tm.upload_many" pool_class, needs_pickling = _get_pool_class_and_requirements(worker_type) + default_method = "storage.transfer_manager.upload_many" + method_override = _helpers._CLIENT_METHOD_OVERRIDE.get() or default_method + pickle_extra_kwargs = {} + if method_override != default_method: + pickle_extra_kwargs["_gcs_method_override"] = method_override + if _helpers._SUPPRESS_INTERNAL_METRICS.get(): + pickle_extra_kwargs["_gcs_suppress_metrics"] = True + + with _helpers._client_method_override(method_override): + with pool_class(max_workers=max_workers) as executor: + futures = [] + for path_or_file, blob in file_blob_pairs: + # File objects are only supported by the THREAD worker because they can't + # be pickled. + if needs_pickling and not isinstance(path_or_file, str): + raise ValueError( + "Passing in a file object is only supported by the THREAD worker type. Please either select THREAD workers, or pass in filenames only." + ) - with pool_class(max_workers=max_workers) as executor: - futures = [] - for path_or_file, blob in file_blob_pairs: - # File objects are only supported by the THREAD worker because they can't - # be pickled. - if needs_pickling and not isinstance(path_or_file, str): - raise ValueError( - "Passing in a file object is only supported by the THREAD worker type. Please either select THREAD workers, or pass in filenames only." - ) - - futures.append( - executor.submit( - _call_method_on_maybe_pickled_blob, - _pickle_client(blob) if needs_pickling else blob, - ( - "_handle_filename_and_upload" - if isinstance(path_or_file, str) - else "_prep_and_do_upload" - ), - path_or_file, - **upload_kwargs, + method_name = ( + "_handle_filename_and_upload" + if isinstance(path_or_file, str) + else "_prep_and_do_upload" ) + if needs_pickling: + futures.append( + executor.submit( + _call_method_on_maybe_pickled_blob, + _pickle_client(blob), + method_name, + path_or_file, + **upload_kwargs, + **pickle_extra_kwargs, + ) + ) + else: + futures.append( + executor.submit( + contextvars.copy_context().run, + _call_method_on_maybe_pickled_blob, + blob, + method_name, + path_or_file, + **upload_kwargs, + ) + ) + concurrent.futures.wait( + futures, timeout=deadline, return_when=concurrent.futures.ALL_COMPLETED ) - concurrent.futures.wait( - futures, timeout=deadline, return_when=concurrent.futures.ALL_COMPLETED - ) results = [] for future in futures: @@ -335,7 +391,9 @@ def download_many( recreated in each subprocess. The serialization of the Client object for use in subprocesses is an approximation and may not capture every detail of the Client object, especially if the Client was modified after - its initial creation or if `Client._http` was modified in any way. + its initial creation, if `Client._http` was modified in any way, or if a + custom OpenTelemetry `meter_provider` was passed to `Client` (subprocess + workers fall back to the subprocess's global `MeterProvider`). THREAD worker types are observed to be relatively efficient for operations with many small files, but not for operations with large @@ -375,37 +433,59 @@ def download_many( download_kwargs["command"] = "tm.download_many" pool_class, needs_pickling = _get_pool_class_and_requirements(worker_type) + default_method = "storage.transfer_manager.download_many" + method_override = _helpers._CLIENT_METHOD_OVERRIDE.get() or default_method + pickle_extra_kwargs = {} + if method_override != default_method: + pickle_extra_kwargs["_gcs_method_override"] = method_override + if _helpers._SUPPRESS_INTERNAL_METRICS.get(): + pickle_extra_kwargs["_gcs_suppress_metrics"] = True + + with _helpers._client_method_override(method_override): + with pool_class(max_workers=max_workers) as executor: + futures = [] + for blob, path_or_file in blob_file_pairs: + # File objects are only supported by the THREAD worker because they can't + # be pickled. + if needs_pickling and not isinstance(path_or_file, str): + raise ValueError( + "Passing in a file object is only supported by the THREAD worker type. Please either select THREAD workers, or pass in filenames only." + ) - with pool_class(max_workers=max_workers) as executor: - futures = [] - for blob, path_or_file in blob_file_pairs: - # File objects are only supported by the THREAD worker because they can't - # be pickled. - if needs_pickling and not isinstance(path_or_file, str): - raise ValueError( - "Passing in a file object is only supported by the THREAD worker type. Please either select THREAD workers, or pass in filenames only." - ) + if skip_if_exists and isinstance(path_or_file, str): + if os.path.isfile(path_or_file): + continue - if skip_if_exists and isinstance(path_or_file, str): - if os.path.isfile(path_or_file): - continue - - futures.append( - executor.submit( - _call_method_on_maybe_pickled_blob, - _pickle_client(blob) if needs_pickling else blob, - ( - "_handle_filename_and_download" - if isinstance(path_or_file, str) - else "_prep_and_do_download" - ), - path_or_file, - **download_kwargs, + method_name = ( + "_handle_filename_and_download" + if isinstance(path_or_file, str) + else "_prep_and_do_download" ) + if needs_pickling: + futures.append( + executor.submit( + _call_method_on_maybe_pickled_blob, + _pickle_client(blob), + method_name, + path_or_file, + **download_kwargs, + **pickle_extra_kwargs, + ) + ) + else: + futures.append( + executor.submit( + contextvars.copy_context().run, + _call_method_on_maybe_pickled_blob, + blob, + method_name, + path_or_file, + **download_kwargs, + ) + ) + concurrent.futures.wait( + futures, timeout=deadline, return_when=concurrent.futures.ALL_COMPLETED ) - concurrent.futures.wait( - futures, timeout=deadline, return_when=concurrent.futures.ALL_COMPLETED - ) results = [] for future in futures: @@ -549,7 +629,9 @@ def upload_many_from_filenames( recreated in each subprocess. The serialization of the Client object for use in subprocesses is an approximation and may not capture every detail of the Client object, especially if the Client was modified after - its initial creation or if `Client._http` was modified in any way. + its initial creation, if `Client._http` was modified in any way, or if a + custom OpenTelemetry `meter_provider` was passed to `Client` (subprocess + workers fall back to the subprocess's global `MeterProvider`). THREAD worker types are observed to be relatively efficient for operations with many small files, but not for operations with large @@ -600,15 +682,18 @@ def upload_many_from_filenames( setattr(blob, prop, value) file_blob_pairs.append((path, blob)) - return upload_many( - file_blob_pairs, - skip_if_exists=skip_if_exists, - upload_kwargs=upload_kwargs, - deadline=deadline, - raise_exception=raise_exception, - worker_type=worker_type, - max_workers=max_workers, - ) + with _helpers._client_method_override( + "storage.transfer_manager.upload_many_from_filenames" + ): + return upload_many( + file_blob_pairs, + skip_if_exists=skip_if_exists, + upload_kwargs=upload_kwargs, + deadline=deadline, + raise_exception=raise_exception, + worker_type=worker_type, + max_workers=max_workers, + ) @_deprecate_threads_param @@ -778,7 +863,9 @@ def download_many_to_path( recreated in each subprocess. The serialization of the Client object for use in subprocesses is an approximation and may not capture every detail of the Client object, especially if the Client was modified after - its initial creation or if `Client._http` was modified in any way. + its initial creation, if `Client._http` was modified in any way, or if a + custom OpenTelemetry `meter_provider` was passed to `Client` (subprocess + workers fall back to the subprocess's global `MeterProvider`). THREAD worker types are observed to be relatively efficient for operations with many small files, but not for operations with large @@ -846,15 +933,18 @@ def download_many_to_path( blob_file_pairs.append((bucket.blob(full_blob_name), resolved_path)) indices_to_process.append(i) - many_results = download_many( - blob_file_pairs, - download_kwargs=download_kwargs, - deadline=deadline, - raise_exception=raise_exception, - worker_type=worker_type, - max_workers=max_workers, - skip_if_exists=False, # skip_if_exists is handled in the loop above - ) + with _helpers._client_method_override( + "storage.transfer_manager.download_many_to_path" + ): + many_results = download_many( + blob_file_pairs, + download_kwargs=download_kwargs, + deadline=deadline, + raise_exception=raise_exception, + worker_type=worker_type, + max_workers=max_workers, + skip_if_exists=False, # skip_if_exists is handled in the loop above + ) for meta_index, result in zip(indices_to_process, many_results): results[meta_index] = result @@ -929,7 +1019,9 @@ def download_chunks_concurrently( recreated in each subprocess. The serialization of the Client object for use in subprocesses is an approximation and may not capture every detail of the Client object, especially if the Client was modified after - its initial creation or if `Client._http` was modified in any way. + its initial creation, if `Client._http` was modified in any way, or if a + custom OpenTelemetry `meter_provider` was passed to `Client` (subprocess + workers fall back to the subprocess's global `MeterProvider`). THREAD worker types are observed to be relatively efficient for operations with many small files, but not for operations with large @@ -961,88 +1053,140 @@ def download_chunks_concurrently( consistency with other download methods despite the exception originating elsewhere. """ - client = blob.client - - if download_kwargs is None: - download_kwargs = {} - if "start" in download_kwargs or "end" in download_kwargs: - raise ValueError( - "Download arguments 'start' and 'end' are not supported by download_chunks_concurrently." - ) - if "checksum" in download_kwargs: - raise ValueError( - "'checksum' is in download_kwargs, but is not supported because sliced downloads have a different checksum mechanism from regular downloads. Use the 'crc32c_checksum' argument on download_chunks_concurrently instead." - ) - - download_kwargs = download_kwargs.copy() - download_kwargs["checksum"] = None - download_kwargs["command"] = "tm.download_sharded" - - # We must know the size and the generation of the blob. - if not blob.size or not blob.generation: - blob.reload() - - pool_class, needs_pickling = _get_pool_class_and_requirements(worker_type) - # Pickle the blob ahead of time (just once, not once per chunk) if needed. - maybe_pickled_blob = _pickle_client(blob) if needs_pickling else blob - - futures = [] - - # Create and/or truncate the destination file to prepare for sparse writing. - with open(filename, "wb") as _: - pass - - with pool_class(max_workers=max_workers) as executor: - cursor = 0 - end = blob.size - while cursor < end: - start = cursor - cursor = min(cursor + chunk_size, end) - futures.append( - executor.submit( - _download_and_write_chunk_in_place, - maybe_pickled_blob, - filename, - start=start, - end=cursor - 1, - download_kwargs=download_kwargs, - crc32c_checksum=crc32c_checksum, + override_client = ( + download_kwargs.get("client") if isinstance(download_kwargs, dict) else None + ) + client = _helpers._resolve_operation_client(blob, override_client) + + with _helpers._start_storage_operation( + client, + "storage.transfer_manager.download_chunks_concurrently", + "ReadObject", + ): + with _helpers._suppress_body_size(): + if download_kwargs is None: + download_kwargs = {} + if "start" in download_kwargs or "end" in download_kwargs: + raise ValueError( + "Download arguments 'start' and 'end' are not supported by download_chunks_concurrently." ) + if "checksum" in download_kwargs: + raise ValueError( + "'checksum' is in download_kwargs, but is not supported because sliced downloads have a different checksum mechanism from regular downloads. Use the 'crc32c_checksum' argument on download_chunks_concurrently instead." + ) + + download_kwargs = download_kwargs.copy() + download_kwargs["checksum"] = None + download_kwargs["command"] = "tm.download_sharded" + + # We must know the size and the generation of the blob. + if not blob.size or not blob.generation: + if override_client is not None: + blob.reload(client=override_client) + else: + blob.reload() + + pool_class, needs_pickling = _get_pool_class_and_requirements(worker_type) + # Pickle the blob ahead of time (just once, not once per chunk) if needed. + maybe_pickled_blob = _pickle_client(blob) if needs_pickling else blob + pickle_extra_kwargs = ( + {"_gcs_suppress_metrics": True} + if _helpers._SUPPRESS_INTERNAL_METRICS.get() + else {} ) - concurrent.futures.wait( - futures, timeout=deadline, return_when=concurrent.futures.ALL_COMPLETED - ) + futures = [] + + # Create and/or truncate the destination file to prepare for sparse writing. + with open(filename, "wb") as _: + pass + + with pool_class(max_workers=max_workers) as executor: + cursor = 0 + end = blob.size + while cursor < end: + start = cursor + cursor = min(cursor + chunk_size, end) + if needs_pickling: + futures.append( + executor.submit( + _download_and_write_chunk_in_place, + maybe_pickled_blob, + filename, + start=start, + end=cursor - 1, + download_kwargs=download_kwargs, + crc32c_checksum=crc32c_checksum, + **pickle_extra_kwargs, + ) + ) + else: + futures.append( + executor.submit( + contextvars.copy_context().run, + functools.partial( + _download_and_write_chunk_in_place, + maybe_pickled_blob, + filename, + start=start, + end=cursor - 1, + download_kwargs=download_kwargs, + crc32c_checksum=crc32c_checksum, + ), + ) + ) + + concurrent.futures.wait( + futures, + timeout=deadline, + return_when=concurrent.futures.ALL_COMPLETED, + ) - # Raise any exceptions; combine checksums. - results = [] - for future in futures: - results.append(future.result()) + # Raise any exceptions; combine checksums. + results = [] + for future in futures: + results.append(future.result()) + + if crc32c_checksum and results: + crc_digest = _digest_ordered_checksum_and_size_pairs(results) + actual_checksum = base64.b64encode(crc_digest).decode("utf-8") + expected_checksum = blob.crc32c + if actual_checksum != expected_checksum: + # For consistency with other download methods we will use + # "google.cloud.storage._media.common.DataCorruption" despite the error + # not originating inside google.cloud.storage._media. + download_url = blob._get_download_url( + client, + if_generation_match=download_kwargs.get("if_generation_match"), + if_generation_not_match=download_kwargs.get( + "if_generation_not_match" + ), + if_metageneration_match=download_kwargs.get( + "if_metageneration_match" + ), + if_metageneration_not_match=download_kwargs.get( + "if_metageneration_not_match" + ), + ) + raise DataCorruption( + None, + DOWNLOAD_CRC32C_MISMATCH_TEMPLATE.format( + download_url, expected_checksum, actual_checksum + ), + ) - if crc32c_checksum and results: - crc_digest = _digest_ordered_checksum_and_size_pairs(results) - actual_checksum = base64.b64encode(crc_digest).decode("utf-8") - expected_checksum = blob.crc32c - if actual_checksum != expected_checksum: - # For consistency with other download methods we will use - # "google.cloud.storage._media.common.DataCorruption" despite the error - # not originating inside google.cloud.storage._media. - download_url = blob._get_download_url( - client, - if_generation_match=download_kwargs.get("if_generation_match"), - if_generation_not_match=download_kwargs.get("if_generation_not_match"), - if_metageneration_match=download_kwargs.get("if_metageneration_match"), - if_metageneration_not_match=download_kwargs.get( - "if_metageneration_not_match" - ), - ) - raise DataCorruption( - None, - DOWNLOAD_CRC32C_MISMATCH_TEMPLATE.format( - download_url, expected_checksum, actual_checksum - ), - ) - return None + if ( + not _helpers._SUPPRESS_BODY_SIZE.get() + and not _helpers._SUPPRESS_INTERNAL_METRICS.get() + ): + recorder = _opentelemetry_metrics.get_active_recorder(client) + if ( + recorder is not None + and isinstance(blob.size, int) + and not isinstance(blob.size, bool) + ): + recorder.record_response_body_size(blob.size, operation="ReadObject") + return None def upload_chunks_concurrently( @@ -1121,7 +1265,9 @@ def upload_chunks_concurrently( recreated in each subprocess. The serialization of the Client object for use in subprocesses is an approximation and may not capture every detail of the Client object, especially if the Client was modified after - its initial creation or if `Client._http` was modified in any way. + its initial creation, if `Client._http` was modified in any way, or if a + custom OpenTelemetry `meter_provider` was passed to `Client` (subprocess + workers fall back to the subprocess's global `MeterProvider`). THREAD worker types are observed to be relatively efficient for operations with many small files, but not for operations with large @@ -1172,79 +1318,130 @@ def upload_chunks_concurrently( """ bucket = blob.bucket - client = blob.client - transport = blob._get_transport(client) - - hostname = _get_host_name(client._connection) - url = "{hostname}/{bucket}/{blob}".format( - hostname=hostname, bucket=bucket.name, blob=_quote(blob.name) - ) - - base_headers, object_metadata, content_type = blob._get_upload_arguments( - client, content_type, filename=filename, command="tm.upload_sharded" - ) - headers = {**base_headers, **_headers_from_metadata(object_metadata)} - - if blob.user_project is not None: - headers["x-goog-user-project"] = blob.user_project - - # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object - # at rest, object resource metadata will store the version of the Key Management - # Service cryptographic material. If a Blob instance with KMS Key metadata set is - # used to upload a new version of the object then the existing kmsKeyName version - # value can't be used in the upload request and the client instead ignores it. - if blob.kms_key_name is not None and "cryptoKeyVersions" not in blob.kms_key_name: - headers["x-goog-encryption-kms-key-name"] = blob.kms_key_name - - container = XMLMPUContainer(url, filename, headers=headers, retry=retry) - - container.initiate(transport=transport, content_type=content_type) - upload_id = container.upload_id - - size = os.path.getsize(filename) - num_of_parts = -(size // -chunk_size) # Ceiling division - - pool_class, needs_pickling = _get_pool_class_and_requirements(worker_type) - # Pickle the blob ahead of time (just once, not once per chunk) if needed. - maybe_pickled_client = _pickle_client(client) if needs_pickling else client - - futures = [] - - with pool_class(max_workers=max_workers) as executor: - for part_number in range(1, num_of_parts + 1): - start = (part_number - 1) * chunk_size - end = min(part_number * chunk_size, size) - - futures.append( - executor.submit( - _upload_part, - maybe_pickled_client, - url, - upload_id, - filename, - start=start, - end=end, - part_number=part_number, - checksum=checksum, - headers=headers.copy(), - retry=retry, - ) + client = _helpers._resolve_operation_client(blob) + + with _helpers._start_storage_operation( + client, + "storage.transfer_manager.upload_chunks_concurrently", + "WriteObject", + ): + with _helpers._suppress_body_size(): + transport = blob._get_transport(client) + + hostname = _get_host_name(client._connection) + url = "{hostname}/{bucket}/{blob}".format( + hostname=hostname, bucket=bucket.name, blob=_quote(blob.name) ) - concurrent.futures.wait( - futures, timeout=deadline, return_when=concurrent.futures.ALL_COMPLETED - ) + base_headers, object_metadata, content_type = blob._get_upload_arguments( + client, content_type, filename=filename, command="tm.upload_sharded" + ) + headers = {**base_headers, **_headers_from_metadata(object_metadata)} + + if blob.user_project is not None: + headers["x-goog-user-project"] = blob.user_project + + # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object + # at rest, object resource metadata will store the version of the Key Management + # Service cryptographic material. If a Blob instance with KMS Key metadata set is + # used to upload a new version of the object then the existing kmsKeyName version + # value can't be used in the upload request and the client instead ignores it. + if ( + blob.kms_key_name is not None + and "cryptoKeyVersions" not in blob.kms_key_name + ): + headers["x-goog-encryption-kms-key-name"] = blob.kms_key_name + + container = XMLMPUContainer(url, filename, headers=headers, retry=retry) + + container.initiate(transport=transport, content_type=content_type) + upload_id = container.upload_id + + size = os.path.getsize(filename) + num_of_parts = -(size // -chunk_size) # Ceiling division + + pool_class, needs_pickling = _get_pool_class_and_requirements(worker_type) + # Pickle the blob ahead of time (just once, not once per chunk) if needed. + maybe_pickled_client = _pickle_client(client) if needs_pickling else client + pickle_extra_kwargs = ( + {"_gcs_suppress_metrics": True} + if _helpers._SUPPRESS_INTERNAL_METRICS.get() + else {} + ) - try: - # Harvest results and raise exceptions. - for future in futures: - part_number, etag = future.result() - container.register_part(part_number, etag) + futures = [] + + with pool_class(max_workers=max_workers) as executor: + for part_number in range(1, num_of_parts + 1): + start = (part_number - 1) * chunk_size + end = min(part_number * chunk_size, size) + + if needs_pickling: + futures.append( + executor.submit( + _upload_part, + maybe_pickled_client, + url, + upload_id, + filename, + start=start, + end=end, + part_number=part_number, + checksum=checksum, + headers=headers.copy(), + retry=retry, + **pickle_extra_kwargs, + ) + ) + else: + futures.append( + executor.submit( + contextvars.copy_context().run, + functools.partial( + _upload_part, + maybe_pickled_client, + url, + upload_id, + filename, + start=start, + end=end, + part_number=part_number, + checksum=checksum, + headers=headers.copy(), + retry=retry, + ), + ) + ) + + concurrent.futures.wait( + futures, + timeout=deadline, + return_when=concurrent.futures.ALL_COMPLETED, + ) - container.finalize(blob._get_transport(client)) - except Exception: - container.cancel(blob._get_transport(client)) - raise + try: + # Harvest results and raise exceptions. + for future in futures: + part_number, etag = future.result() + container.register_part(part_number, etag) + + container.finalize(blob._get_transport(client)) + except BaseException as exc: + try: + with _helpers._suppress_internal_metrics(): + container.cancel(blob._get_transport(client)) + except BaseException as cleanup_exc: + _helpers._record_primary_exception(exc, cleanup_exc) + raise + raise + + if ( + not _helpers._SUPPRESS_BODY_SIZE.get() + and not _helpers._SUPPRESS_INTERNAL_METRICS.get() + ): + recorder = _opentelemetry_metrics.get_active_recorder(client) + if recorder is not None: + recorder.record_request_body_size(size, operation="WriteObject") def _upload_part( @@ -1258,30 +1455,41 @@ def _upload_part( checksum, headers, retry, + _gcs_suppress_metrics=False, ): """Helper function that runs inside a thread or subprocess to upload a part. `maybe_pickled_client` is either a Client (for threads) or a specially pickled Client (for processes) because the default pickling mangles Client objects.""" - - if isinstance(maybe_pickled_client, Client): - client = maybe_pickled_client - else: - client = pickle.loads(maybe_pickled_client) - part = XMLMPUPart( - url, - upload_id, - filename, - start=start, - end=end, - part_number=part_number, - checksum=checksum, - headers=headers, - retry=retry, + suppress_ctx = ( + _helpers._suppress_internal_metrics() + if _gcs_suppress_metrics + else nullcontext() ) - part.upload(client._http) - return (part_number, part.etag) + with suppress_ctx: + if isinstance(maybe_pickled_client, Client): + client = maybe_pickled_client + else: + client = pickle.loads(maybe_pickled_client) + with _helpers._bind_suboperation_context( + client, + "storage.transfer_manager.upload_chunks_concurrently", + "WriteObject", + ): + part = XMLMPUPart( + url, + upload_id, + filename, + start=start, + end=end, + part_number=part_number, + checksum=checksum, + headers=headers, + retry=retry, + ) + part.upload(client._http) + return (part_number, part.etag) def _headers_from_metadata(metadata): @@ -1300,7 +1508,13 @@ def _headers_from_metadata(metadata): def _download_and_write_chunk_in_place( - maybe_pickled_blob, filename, start, end, download_kwargs, crc32c_checksum + maybe_pickled_blob, + filename, + start, + end, + download_kwargs, + crc32c_checksum, + _gcs_suppress_metrics=False, ): """Helper function that runs inside a thread or subprocess. @@ -1310,15 +1524,26 @@ def _download_and_write_chunk_in_place( Returns a crc if configured (or None) and the size written. """ + suppress_ctx = ( + _helpers._suppress_internal_metrics() + if _gcs_suppress_metrics + else nullcontext() + ) + with suppress_ctx: + if isinstance(maybe_pickled_blob, Blob): + blob = maybe_pickled_blob + else: + blob = pickle.loads(maybe_pickled_blob) - if isinstance(maybe_pickled_blob, Blob): - blob = maybe_pickled_blob - else: - blob = pickle.loads(maybe_pickled_blob) - - with _ChecksummingSparseFileWrapper(filename, start, crc32c_checksum) as f: - blob._prep_and_do_download(f, start=start, end=end, **download_kwargs) - return (f.crc, (end - start) + 1) + override_client = download_kwargs.get("client") + with _helpers._bind_suboperation_context( + _helpers._resolve_operation_client(blob, override_client), + "storage.transfer_manager.download_chunks_concurrently", + "ReadObject", + ): + with _ChecksummingSparseFileWrapper(filename, start, crc32c_checksum) as f: + blob._prep_and_do_download(f, start=start, end=end, **download_kwargs) + return (f.crc, (end - start) + 1) class _ChecksummingSparseFileWrapper: @@ -1354,6 +1579,14 @@ def __exit__(self, exc_type, exc_value, tb): self.f.close() +def _invoke_maybe_pickled_blob(maybe_pickled_blob, method_name, *args, **kwargs): + if isinstance(maybe_pickled_blob, Blob): + blob = maybe_pickled_blob + else: + blob = pickle.loads(maybe_pickled_blob) + return getattr(blob, method_name)(*args, **kwargs) + + def _call_method_on_maybe_pickled_blob( maybe_pickled_blob, method_name, *args, **kwargs ): @@ -1362,12 +1595,19 @@ def _call_method_on_maybe_pickled_blob( `maybe_pickled_blob` is either a Blob (for threads) or a specially pickled Blob (for processes) because the default pickling mangles Client objects which are attached to Blobs.""" - - if isinstance(maybe_pickled_blob, Blob): - blob = maybe_pickled_blob - else: - blob = pickle.loads(maybe_pickled_blob) - return getattr(blob, method_name)(*args, **kwargs) + method_override = kwargs.pop("_gcs_method_override", None) + suppress_metrics = kwargs.pop("_gcs_suppress_metrics", False) + if suppress_metrics: + with _helpers._suppress_internal_metrics(): + return _invoke_maybe_pickled_blob( + maybe_pickled_blob, method_name, *args, **kwargs + ) + if method_override is not None: + with _helpers._client_method_override(method_override): + return _invoke_maybe_pickled_blob( + maybe_pickled_blob, method_name, *args, **kwargs + ) + return _invoke_maybe_pickled_blob(maybe_pickled_blob, method_name, *args, **kwargs) def _reduce_client(cl): @@ -1485,4 +1725,12 @@ def __new__(cls, id, *args, **kwargs): args = args[:_REDUCE_CLIENT_POS_ARGS] cached_client = Client(*args, **kwargs) _cached_clients[id] = cached_client + global _cleanup_registered_pid + pid = os.getpid() + if _cleanup_registered_pid != pid: + atexit.register(_cleanup_cached_clients) + multiprocessing.util.Finalize( + None, _cleanup_cached_clients, exitpriority=10 + ) + _cleanup_registered_pid = pid return cached_client diff --git a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py index 966cb8a1c78d..2e10d433cbe5 100644 --- a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py +++ b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py @@ -12,13 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio +import concurrent.futures +import http.client import importlib +import socket +import ssl import sys +import types from unittest import mock import pytest +import requests.exceptions +import urllib3.exceptions +from google.api_core import exceptions as api_exceptions +from google.auth import exceptions as auth_exceptions from google.cloud.storage import _opentelemetry_metrics +from google.cloud.storage import exceptions as storage_exceptions from google.cloud.storage.version import __version__ @@ -46,6 +57,8 @@ def test_opentelemetry_import_error_on_load(monkeypatch): is False ) assert _opentelemetry_metrics.get_meter(enable_metrics=True) is None + m.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + assert _opentelemetry_metrics.get_meter(enable_metrics=True) is None finally: importlib.reload(_opentelemetry_metrics) @@ -144,7 +157,6 @@ def test_client_setting_overrides_env_var(monkeypatch): def test_non_boolean_client_setting_raises_type_error(): """Passing a non-boolean setting such as 'false' must raise TypeError.""" - # Argument validation executes before the _ENABLE_METRICS_DEV_GATE check. with pytest.raises(TypeError, match="enable_metrics must be a boolean or None"): _opentelemetry_metrics.is_metrics_enabled(enable_metrics="false") # type: ignore[arg-type] @@ -218,9 +230,14 @@ def test_otel_missing_disables_metrics(monkeypatch): def test_get_common_attributes(): """Verify common attributes conform to GCS OTel specification.""" attrs = _opentelemetry_metrics.get_common_attributes() + assert set(attrs.keys()) == { + "gcp.client.service", + "gcp.client.version", + "gcp.client.artifact", + } + assert "gcp.client.repo" not in attrs assert attrs["gcp.client.service"] == "storage" assert attrs["gcp.client.version"] == __version__ - assert attrs["gcp.client.repo"] == "googleapis/google-cloud-python" assert attrs["gcp.client.artifact"] == "google-cloud-storage" attrs["gcp.client.service"] = "mutated" @@ -287,3 +304,1302 @@ def test_get_meter(monkeypatch): __version__, meter_provider=mock_provider, ) + + +# --------------------------------------------------------------------------- +# Histogram Bucket Boundaries & Views Tests +# --------------------------------------------------------------------------- +def test_histogram_bucket_boundaries(): + """Verify custom latency and byte-size bucket boundaries match specification.""" + latency = _opentelemetry_metrics.LATENCY_HISTOGRAM_BOUNDARIES + assert len(latency) == 165 + assert latency[0] == 0.0 + assert latency[1] == 0.002 + assert latency[49] == 0.098 + assert latency[50] == 0.1 + assert latency[60] == 0.2 + assert latency[-1] < 300.0 + assert list(latency) == sorted(latency) + + sizes = _opentelemetry_metrics.SIZE_HISTOGRAM_BOUNDARIES + assert len(sizes) == 48 + assert sizes[0] == 0 + assert sizes[1] == 128 * 1024 + assert sizes[32] == 4 * 1024 * 1024 + assert sizes[33] == int(4.25 * 1024 * 1024) + assert sizes[34] == int(4.75 * 1024 * 1024) + assert sizes[-1] == 8593866752 + assert list(sizes) == sorted(sizes) + + +def test_get_histogram_views_with_and_without_sdk(monkeypatch): + """Verify get_histogram_views builds 6 Views when SDK is installed and [] otherwise.""" + fake_view_mod = types.ModuleType("opentelemetry.sdk.metrics.view") + + class FakeAggregation: + def __init__(self, boundaries): + self.boundaries = boundaries + + class FakeView: + def __init__(self, instrument_name, aggregation, meter_name=None): + self.instrument_name = instrument_name + self.aggregation = aggregation + self.meter_name = meter_name + + fake_view_mod.ExplicitBucketHistogramAggregation = FakeAggregation + fake_view_mod.View = FakeView + + with monkeypatch.context() as m: + m.setitem(sys.modules, "opentelemetry", types.ModuleType("opentelemetry")) + m.setitem( + sys.modules, "opentelemetry.sdk", types.ModuleType("opentelemetry.sdk") + ) + m.setitem( + sys.modules, + "opentelemetry.sdk.metrics", + types.ModuleType("opentelemetry.sdk.metrics"), + ) + m.setitem(sys.modules, "opentelemetry.sdk.metrics.view", fake_view_mod) + + views = _opentelemetry_metrics.get_histogram_views() + assert len(views) == 6 + assert all(v.meter_name == "google.cloud.storage" for v in views) + latency_views = [ + v + for v in views + if v.instrument_name in _opentelemetry_metrics._LATENCY_METRIC_NAMES + ] + size_views = [ + v + for v in views + if v.instrument_name in _opentelemetry_metrics._SIZE_METRIC_NAMES + ] + assert len(latency_views) == 4 + assert len(size_views) == 2 + assert ( + latency_views[0].aggregation.boundaries + == _opentelemetry_metrics.LATENCY_HISTOGRAM_BOUNDARIES + ) + assert ( + size_views[0].aggregation.boundaries + == _opentelemetry_metrics.SIZE_HISTOGRAM_BOUNDARIES + ) + + # Without opentelemetry-sdk installed + with monkeypatch.context() as m: + m.setitem(sys.modules, "opentelemetry.sdk.metrics.view", None) + assert _opentelemetry_metrics.get_histogram_views() == [] + + +# --------------------------------------------------------------------------- +# Error Classification (`classify_error_type`) Tests +# --------------------------------------------------------------------------- +def test_classify_error_type_none_and_status_codes(): + assert _opentelemetry_metrics.classify_error_type() == "OK" + assert _opentelemetry_metrics.classify_error_type(None, status_code=200) == "OK" + assert _opentelemetry_metrics.classify_error_type(None, status_code=206) == "OK" + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=True) == "OK" # type: ignore[arg-type] + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=400) + == "INVALID_ARGUMENT" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=401) + == "UNAUTHENTICATED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=403) + == "PERMISSION_DENIED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=404) == "NOT_FOUND" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=408) + == "DEADLINE_EXCEEDED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=409) == "ABORTED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=412) + == "FAILED_PRECONDITION" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=416) + == "OUT_OF_RANGE" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=429) + == "RESOURCE_EXHAUSTED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=499) == "CANCELLED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=500) == "INTERNAL" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=501) + == "UNIMPLEMENTED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=503) + == "UNAVAILABLE" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=504) + == "DEADLINE_EXCEEDED" + ) + assert _opentelemetry_metrics.classify_error_type(None, status_code=502) == "502" + assert _opentelemetry_metrics.classify_error_type(None, status_code=99999) == "OK" + + +def test_classify_error_type_exceptions(): + # 1. Checksum Mismatch (including subclasses of DataCorruption) + resp_mock = mock.Mock(status_code=200) + assert ( + _opentelemetry_metrics.classify_error_type( + storage_exceptions.DataCorruption(resp_mock, "crc mismatch") + ) + == "CHECKSUM_MISMATCH" + ) + + class CustomDataCorruption(storage_exceptions.DataCorruption): + pass + + assert ( + _opentelemetry_metrics.classify_error_type( + CustomDataCorruption(resp_mock, "subclass mismatch") + ) + == "CHECKSUM_MISMATCH" + ) + + # 2. Authentication Errors + assert ( + _opentelemetry_metrics.classify_error_type( + auth_exceptions.RefreshError("token expired") + ) + == "AUTHENTICATION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + auth_exceptions.TransportError("auth transport error") + ) + == "AUTHENTICATION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + auth_exceptions.DefaultCredentialsError("missing adc") + ) + == "AUTHENTICATION_ERROR" + ) + + # 3. Cancellation + assert ( + _opentelemetry_metrics.classify_error_type(asyncio.CancelledError()) + == "CANCELLED" + ) + + # 4. HTTP Conflict (409) consistency with status_code=409 + conflict_exc = api_exceptions.Conflict("conflict") + assert _opentelemetry_metrics.classify_error_type(conflict_exc) == "ABORTED" + + # 5. gRPC / GoogleAPICallError status codes + not_found_exc = api_exceptions.NotFound("not found") + assert _opentelemetry_metrics.classify_error_type(not_found_exc) == "NOT_FOUND" + + custom_api_exc = api_exceptions.GoogleAPICallError("custom") + custom_api_exc.grpc_status_code = "RAW_GRPC_STATUS" # type: ignore[assignment] + assert ( + _opentelemetry_metrics.classify_error_type(custom_api_exc) == "RAW_GRPC_STATUS" + ) + + # Direct grpc.RpcError-like exception with .code() method + class FakeStatusCode: + name = "DEADLINE_EXCEEDED" + + class FakeRpcError(Exception): + def code(self): + return FakeStatusCode() + + assert ( + _opentelemetry_metrics.classify_error_type(FakeRpcError()) + == "DEADLINE_EXCEEDED" + ) + + class FakeRpcErrorNoName(Exception): + def code(self): + return object() + + assert _opentelemetry_metrics.classify_error_type(FakeRpcErrorNoName()) == "UNKNOWN" + + # 6. HTTP status code on exception + http_api_exc = api_exceptions.GoogleAPICallError("bad gateway") + http_api_exc.code = 502 + http_api_exc.grpc_status_code = None + assert _opentelemetry_metrics.classify_error_type(http_api_exc) == "502" + + invalid_resp_with_code = storage_exceptions.InvalidResponse( + mock.Mock(status_code=429), "rate limited" + ) + assert ( + _opentelemetry_metrics.classify_error_type(invalid_resp_with_code) + == "RESOURCE_EXHAUSTED" + ) + + req_http_err = requests.exceptions.HTTPError(response=mock.Mock(status_code=503)) + assert _opentelemetry_metrics.classify_error_type(req_http_err) == "UNAVAILABLE" + + # 7. Timeouts + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.ConnectTimeout("connect timeout") + ) + == "TIMEOUT" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.ReadTimeout("read timeout") + ) + == "TIMEOUT" + ) + assert ( + _opentelemetry_metrics.classify_error_type(socket.timeout("timed out")) + == "TIMEOUT" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + urllib3.exceptions.ReadTimeoutError(None, "/", "read timed out") # type: ignore[arg-type] + ) + == "TIMEOUT" + ) + assert ( + _opentelemetry_metrics.classify_error_type(concurrent.futures.TimeoutError()) + == "TIMEOUT" + ) + retry_err = api_exceptions.RetryError( + "deadline exceeded", cause=socket.timeout("timed out") + ) + assert _opentelemetry_metrics.classify_error_type(retry_err) == "TIMEOUT" + + # 8. TLS / SSL Failure + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.SSLError("cert verify failed") + ) + == "TLS_FAILURE" + ) + assert ( + _opentelemetry_metrics.classify_error_type(ssl.SSLError("ssl handshake error")) + == "TLS_FAILURE" + ) + + # 9. DNS, Connection Refused, Connection Reset (direct and wrapped in requests/urllib3) + gai_err = socket.gaierror(-2, "Name or service not known") + new_conn_err = urllib3.exceptions.NewConnectionError(None, "dns failed") # type: ignore[arg-type] + new_conn_err.__cause__ = gai_err + wrapped_dns = requests.exceptions.ConnectionError( + urllib3.exceptions.MaxRetryError( + None, # type: ignore[arg-type] + "/", + reason=new_conn_err, + ) + ) + assert _opentelemetry_metrics.classify_error_type(wrapped_dns) == "DNS_FAILURE" + + refused_err = ConnectionRefusedError(111, "Connection refused") + wrapped_refused = requests.exceptions.ConnectionError(refused_err) + assert ( + _opentelemetry_metrics.classify_error_type(wrapped_refused) + == "CONNECTION_ERROR" + ) + + reset_err = ConnectionResetError(104, "Connection reset by peer") + wrapped_reset = requests.exceptions.ConnectionError(reset_err) + assert ( + _opentelemetry_metrics.classify_error_type(wrapped_reset) == "CONNECTION_ERROR" + ) + + # 10. Generic Connection Errors + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.ConnectionError("connection aborted") + ) + == "CONNECTION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.ChunkedEncodingError("incomplete read") + ) + == "CONNECTION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type(OSError("network unreachable")) + == "CONNECTION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + http.client.IncompleteRead(b"partial", 10) + ) + == "CONNECTION_ERROR" + ) + + # 11. Explicit status_code fallback when exception is generic + assert ( + _opentelemetry_metrics.classify_error_type( + RuntimeError("unexpected"), status_code=503 + ) + == "UNAVAILABLE" + ) + + # 12. Client Validation Errors + assert ( + _opentelemetry_metrics.classify_error_type(ValueError("bad arg")) + == "CLIENT_VALIDATION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type(TypeError("bad type")) + == "CLIENT_VALIDATION_ERROR" + ) + invalid_resp_no_code = storage_exceptions.InvalidResponse( + mock.Mock(status_code=None), "missing header" + ) + assert ( + _opentelemetry_metrics.classify_error_type(invalid_resp_no_code) + == "CLIENT_VALIDATION_ERROR" + ) + + # 13. Cyclic exception chain and unknown exception + e1 = RuntimeError("cycle 1") + e2 = RuntimeError("cycle 2", e1) + e1.__cause__ = e2 + assert _opentelemetry_metrics.classify_error_type(e1) == "UNKNOWN" + + +# --------------------------------------------------------------------------- +# Route Normalization (`normalize_url_template`) Tests +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "raw_url,expected", + [ + (None, "UNKNOWN"), + ("", "UNKNOWN"), + (123, "UNKNOWN"), + ("https://storage.googleapis.com", "UNKNOWN"), + ("/", "UNKNOWN"), + ("///", "UNKNOWN"), + ( + "https://storage.googleapis.com//storage/v1/b/my-bucket//", + "/storage/v1/b/{bucket}", + ), + ( + "https://storage.googleapis.com/upload/storage/v1/b/my-bucket/o/my-obj?uploadType=resumable", + "/upload/storage/v1/b/{bucket}/o", + ), + ( + "https://storage.googleapis.com/custom/path?tag=my_uploads", + "UNKNOWN", + ), + ( + "https://storage.googleapis.com/custom/path?my_uploadId=123", + "UNKNOWN", + ), + ( + "https://storage.googleapis.com/storage/v1/b?project=my-proj", + "/storage/v1/b", + ), + ("/b?project=my-proj", "/storage/v1/b"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket?projection=full", + "/storage/v1/b/{bucket}", + ), + ("/b/my-bucket/", "/storage/v1/b/{bucket}"), + ("b/my-bucket", "/storage/v1/b/{bucket}"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o?prefix=foo", + "/storage/v1/b/{bucket}/o", + ), + ("/b/my-bucket/o", "/storage/v1/b/{bucket}/o"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o/folder%2Ffile.txt?generation=1", + "/storage/v1/b/{bucket}/o/{object}", + ), + ("/b/my-bucket/o/file.txt", "/storage/v1/b/{bucket}/o/{object}"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o/dest.txt/compose", + "/storage/v1/b/{bucket}/o/{object}/compose", + ), + ( + "/b/my-bucket/o/dest.txt/compose", + "/storage/v1/b/{bucket}/o/{object}/compose", + ), + ( + "https://storage.googleapis.com/storage/v1/b/src-b/o/src-o/rewriteTo/b/dst-b/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/rewriteTo/b/{bucket}/o/{object}", + ), + ( + "/b/src-b/o/src-o/rewriteTo/b/dst-b/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/rewriteTo/b/{bucket}/o/{object}", + ), + ( + "https://storage.googleapis.com/storage/v1/b/src-b/o/src-o/copyTo/b/dst-b/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/copyTo/b/{bucket}/o/{object}", + ), + ( + "/b/src-b/o/src-o/copyTo/b/dst-b/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/copyTo/b/{bucket}/o/{object}", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o/src-o/moveTo/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/moveTo/o/{object}", + ), + ( + "/b/my-bucket/o/src-o/moveTo/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/moveTo/o/{object}", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o/my-o/restore", + "/storage/v1/b/{bucket}/o/{object}/restore", + ), + ( + "/b/my-bucket/o/my-o/restore", + "/storage/v1/b/{bucket}/o/{object}/restore", + ), + ( + "/b/my-bucket/o/my-o/acl", + "/storage/v1/b/{bucket}/o/{object}/acl", + ), + ( + "/b/my-bucket/o/my-o/acl/allUsers", + "/storage/v1/b/{bucket}/o/{object}/acl/{entity}", + ), + ( + "/b/my-bucket/o/my-o/iam", + "/storage/v1/b/{bucket}/o/{object}/iam", + ), + ( + "/b/my-bucket/o/my-o/iam/testPermissions", + "/storage/v1/b/{bucket}/o/{object}/iam", + ), + ( + "/b/my-bucket/acl", + "/storage/v1/b/{bucket}/acl", + ), + ( + "/b/my-bucket/acl/allUsers", + "/storage/v1/b/{bucket}/acl/{entity}", + ), + ( + "/b/my-bucket/defaultObjectAcl", + "/storage/v1/b/{bucket}/defaultObjectAcl", + ), + ( + "/b/my-bucket/defaultObjectAcl/allUsers", + "/storage/v1/b/{bucket}/defaultObjectAcl/{entity}", + ), + ( + "/b/my-bucket/restore", + "/storage/v1/b/{bucket}/restore", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/iam/testPermissions", + "/storage/v1/b/{bucket}/iam", + ), + ("/b/my-bucket/iam", "/storage/v1/b/{bucket}/iam"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/notificationConfigs", + "/storage/v1/b/{bucket}/notificationConfigs", + ), + ( + "/b/my-bucket/notificationConfigs", + "/storage/v1/b/{bucket}/notificationConfigs", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/notificationConfigs/42", + "/storage/v1/b/{bucket}/notificationConfigs/{notification}", + ), + ( + "/b/my-bucket/notificationConfigs/42", + "/storage/v1/b/{bucket}/notificationConfigs/{notification}", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/lockRetentionPolicy", + "/storage/v1/b/{bucket}/lockRetentionPolicy", + ), + ( + "/b/my-bucket/lockRetentionPolicy", + "/storage/v1/b/{bucket}/lockRetentionPolicy", + ), + ( + "/storage/v1/projects/my-proj/serviceAccount", + "/storage/v1/projects/{project}/serviceAccount", + ), + ( + "/projects/my-proj/serviceAccount", + "/storage/v1/projects/{project}/serviceAccount", + ), + ( + "/storage/v1/projects/my-proj/hmacKeys", + "/storage/v1/projects/{project}/hmacKeys", + ), + ("/projects/my-proj/hmacKeys", "/storage/v1/projects/{project}/hmacKeys"), + ( + "/storage/v1/projects/my-proj/hmacKeys/GOOG123", + "/storage/v1/projects/{project}/hmacKeys/{access_id}", + ), + ( + "/projects/my-proj/hmacKeys/GOOG123", + "/storage/v1/projects/{project}/hmacKeys/{access_id}", + ), + ( + "https://storage.googleapis.com/upload/storage/v1/b/my-bucket/o?uploadType=resumable", + "/upload/storage/v1/b/{bucket}/o", + ), + ( + "https://storage.googleapis.com/download/storage/v1/b/my-bucket/o/my-obj?alt=media", + "/download/storage/v1/b/{bucket}/o/{object}", + ), + ("https://storage.googleapis.com/batch/storage/v1", "/batch/storage/v1"), + ( + "https://storage.googleapis.com/my-bucket/my-object?uploads", + "/{bucket}/{object}", + ), + ( + "https://storage.googleapis.com/my-bucket/nested/dir/object.bin?uploadId=xyz&partNumber=2", + "/{bucket}/{object}", + ), + ( + "https://storage.googleapis.com/unrecognized/two-segment", + "UNKNOWN", + ), + ("https://storage.googleapis.com/storage/v2", "UNKNOWN"), + ("https://storage.googleapis.com/storage/v2/other?uploadId=xyz", "UNKNOWN"), + ("https://storage.googleapis.com/custom/unrecognized/path/here", "UNKNOWN"), + ("https://[::1/storage/v1/b", "UNKNOWN"), + ], +) +def test_normalize_url_template(raw_url, expected): + assert _opentelemetry_metrics.normalize_url_template(raw_url) == expected + + +# --------------------------------------------------------------------------- +# Logical Operation Context (`start_operation`) Tests +# --------------------------------------------------------------------------- +def test_start_operation_disabled_and_nesting(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + # Disabled recorder -> yields None and records nothing + disabled_recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=False) + with _opentelemetry_metrics.start_operation( + disabled_recorder, "storage.Blob.reload", "storage.objects.get" + ) as op: + assert op is None + assert _opentelemetry_metrics.get_current_operation() is None + + # Enabled recorder -> outermost call records, nested call is suppressed + enabled_recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) + with _opentelemetry_metrics.start_operation( + enabled_recorder, + "storage.Blob.download_to_filename", + "storage.objects.get", + ) as outer_op: + assert outer_op is not None + assert outer_op.client_method == "storage.Blob.download_to_filename" + assert outer_op.rpc_method == "storage.objects.get" + assert _opentelemetry_metrics.get_current_operation() == outer_op + + with _opentelemetry_metrics.start_operation( + enabled_recorder, + "storage.Blob.download_to_file", + "storage.objects.get", + ) as inner_op: + assert inner_op is outer_op + + assert _opentelemetry_metrics.get_current_operation() is None + enabled_recorder.client_request_duration.record.assert_called_once() + enabled_recorder.operations_counter.add.assert_called_once() + _, kwargs = enabled_recorder.operations_counter.add.call_args + assert ( + kwargs["attributes"]["gcp.client.method"] == "storage.Blob.download_to_filename" + ) + assert kwargs["attributes"]["gcp.storage.operation"] == "storage.objects.get" + assert kwargs["attributes"]["error.type"] == "OK" + + +def test_start_operation_exception_and_suppression(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) + + with pytest.raises(api_exceptions.NotFound): + with _opentelemetry_metrics.start_operation( + recorder, "storage.Bucket.get_blob", "storage.objects.get" + ): + raise api_exceptions.NotFound("missing") + + assert _opentelemetry_metrics.get_current_operation() is None + _, kwargs = recorder.operations_counter.add.call_args + assert kwargs["attributes"]["error.type"] == "NOT_FOUND" + + # Suppressed outer operation (e.g. transfer_manager worker) suppresses both + # logical operation metrics and body size recording + recorder.operations_counter.add.reset_mock() + recorder.client_request_duration.record.reset_mock() + with _opentelemetry_metrics.start_operation( + None, + "storage.transfer_manager.download_many_to_path", + "storage.objects.get", + suppress=True, + ) as suppressed_op: + assert suppressed_op is not None + assert suppressed_op.suppressed is True + with _opentelemetry_metrics.start_operation( + recorder, + "storage.Blob.download_to_filename", + "storage.objects.get", + ): + recorder.record_request_body_size(2048) + recorder.record_response_body_size(4096) + + assert _opentelemetry_metrics.get_current_operation() is None + recorder.operations_counter.add.assert_not_called() + recorder.client_request_duration.record.assert_not_called() + recorder.request_body_size.record.assert_not_called() + recorder.response_body_size.record.assert_not_called() + + # Nested suppress=True inside an active unsuppressed operation (e.g. BlobReader prefetch) + with _opentelemetry_metrics.start_operation( + recorder, + "storage.Blob.open", + "storage.objects.get", + ) as active_op: + assert active_op is not None and active_op.suppressed is False + with _opentelemetry_metrics.start_operation( + recorder, + "storage.Blob.download_as_bytes", + "storage.objects.get", + suppress=True, + ) as child_suppressed: + assert child_suppressed is not None + assert child_suppressed.suppressed is True + assert child_suppressed.client_method == "storage.Blob.open" + # Prefetch body size and TTFB are suppressed inside child_suppressed + recorder.record_response_body_size(40 * 1024 * 1024) + recorder.record_ttfb(0.05) + # Outside the suppressed prefetch block, actual user bytes are recorded + assert _opentelemetry_metrics.get_current_operation() == active_op + recorder.record_response_body_size(1024) + recorder.record_response_body_size(True) # type: ignore[arg-type] + recorder.record_request_body_size(False) # type: ignore[arg-type] + + recorder.response_body_size.record.assert_called_once() + assert recorder.response_body_size.record.call_args[0][0] == 1024 + recorder.operation_ttfb.record.assert_not_called() + recorder.operations_counter.add.assert_called_once() + + +def test_start_operation_asyncio_concurrency_isolation(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + + async def worker(method_name: str, rpc_name: str): + with _opentelemetry_metrics.start_operation(recorder, method_name, rpc_name): + await asyncio.sleep(0.005) + current = _opentelemetry_metrics.get_current_operation() + assert current is not None + assert current.client_method == method_name + assert current.rpc_method == rpc_name + + async def run_all(): + await asyncio.gather( + worker( + "storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + "google.storage.v2.Storage/BidiReadObject", + ), + worker( + "storage.asyncio.AsyncAppendableObjectWriter.append", + "google.storage.v2.Storage/BidiWriteObject", + ), + ) + + asyncio.run(run_all()) + assert recorder.operations_counter.add.call_count == 2 + + +# --------------------------------------------------------------------------- +# `_extract_server_address` & `_MetricsRecorder` Tests +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "endpoint,expected", + [ + (None, "storage.googleapis.com"), + ("", "storage.googleapis.com"), + (" ", "storage.googleapis.com"), + ("https://storage.googleapis.com:443", "storage.googleapis.com"), + ("storage.googleapis.com:443", "storage.googleapis.com"), + ("//storage.googleapis.com:443", "storage.googleapis.com"), + ("https://[::1]:8080", "::1"), + ("http://localhost:9023/storage/v1", "localhost"), + ("https://", "storage.googleapis.com"), + ("https://[::1", "storage.googleapis.com"), + ("[::1", "storage.googleapis.com"), + ("dns://8.8.8.8/custom.endpoint.com:443", "custom.endpoint.com"), + ("dns:custom.endpoint.com:443", "custom.endpoint.com"), + ("dns://custom.endpoint.com:443", "custom.endpoint.com"), + ("dns://8.8.8.8/", "storage.googleapis.com"), + ("ipv4:127.0.0.1:443", "127.0.0.1"), + ("ipv6:[::1]:443", "::1"), + ], +) +def test_extract_server_address(endpoint, expected): + assert _opentelemetry_metrics._extract_server_address(endpoint) == expected + + +def test_metrics_recorder_recording_and_shutdown(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + mock_provider = mock.Mock() + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, + meter_provider=mock_provider, + api_endpoint="https://custom.storage.googleapis.com:443", + rpc_system="http", + ) + assert recorder.standard_enabled is True + assert recorder.enabled is True + assert recorder.server_address == "custom.storage.googleapis.com" + + base_attrs = recorder.get_base_attributes() + assert base_attrs["server.address"] == "custom.storage.googleapis.com" + assert base_attrs["rpc.system.name"] == "http" + + # Verify exact instrument names, units, and descriptions + histogram_specs = { + call.args[0]: (call.kwargs["unit"], call.kwargs["description"]) + for call in mock_meter.create_histogram.call_args_list + } + assert histogram_specs == { + "rpc.client.call.duration": ( + "s", + "Duration of an individual gRPC attempt in seconds (excluding retries).", + ), + "http.client.request.duration": ( + "s", + "Duration of an individual HTTP request attempt in seconds (excluding retries).", + ), + "gcp.client.request.duration": ( + "s", + "End-to-end latency of a logical user-facing Veneer operation in seconds.", + ), + "gcp.storage.client.request.body.size": ( + "By", + "Application payload bytes submitted by the caller for an upload or write operation.", + ), + "gcp.storage.client.response.body.size": ( + "By", + "Application payload bytes delivered to the caller.", + ), + "gcp.storage.client.ttfb": ( + "s", + "Time elapsed from attempt initiation to receipt of the first response byte.", + ), + } + counter_specs = { + call.args[0]: (call.kwargs["unit"], call.kwargs["description"]) + for call in mock_meter.create_counter.call_args_list + } + assert counter_specs == { + "gcp.storage.client.operations": ( + "{operation}", + "Number of logical GCS client operations.", + ), + "gcp.storage.client.attempts": ( + "{attempt}", + "Number of physical network transport attempts executed.", + ), + "gcp.storage.client.retries": ( + "{retry}", + "Number of transport retries executed after a failed attempt.", + ), + "gcp.storage.client.errors": ( + "{error}", + "Number of physical network transport attempts terminating with a non-OK status.", + ), + } + + # Record within a logical operation context + with _opentelemetry_metrics.start_operation( + recorder, + "storage.Blob.upload_from_string", + "WriteObject", + rpc_method="google.storage.v2.Storage/WriteObject", + ): + recorder.record_http_attempt( + duration=0.05, + http_method="post", + url_template="/upload/storage/v1/b/{bucket}/o", + status_code=503, + error_type="UNAVAILABLE", + network_path="cloudpath", + ) + recorder.record_http_attempt( + duration=0.04, + http_method="POST", + url_template="/upload/storage/v1/b/{bucket}/o", + status_code=200, + error_type="OK", + network_path="cloudpath", + ) + recorder.record_request_body_size(1024) + recorder.record_response_body_size(512) + recorder.record_ttfb(0.01, network_path="cloudpath") + + # Logical operation metrics: duration has rpc.method; operations counter does not + recorder.client_request_duration.record.assert_called_once() + _, req_dur_kwargs = recorder.client_request_duration.record.call_args + assert req_dur_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert ( + req_dur_kwargs["attributes"]["rpc.method"] + == "google.storage.v2.Storage/WriteObject" + ) + assert ( + req_dur_kwargs["attributes"]["gcp.client.method"] + == "storage.Blob.upload_from_string" + ) + assert req_dur_kwargs["attributes"]["error.type"] == "OK" + + recorder.operations_counter.add.assert_called_once() + _, op_cnt_kwargs = recorder.operations_counter.add.call_args + assert op_cnt_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert "rpc.method" not in op_cnt_kwargs["attributes"] + + assert recorder.http_request_duration.record.call_count == 2 + _, http_dur_kwargs = recorder.http_request_duration.record.call_args_list[0] + assert http_dur_kwargs["attributes"]["http.request.method"] == "POST" + assert ( + http_dur_kwargs["attributes"]["url.template"] + == "/upload/storage/v1/b/{bucket}/o" + ) + assert http_dur_kwargs["attributes"]["http.response.status_code"] == 503 + assert http_dur_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert http_dur_kwargs["attributes"]["network.path"] == "cloudpath" + assert "gcp.storage.operation" not in http_dur_kwargs["attributes"] + assert "gcp.client.method" not in http_dur_kwargs["attributes"] + + assert recorder.attempts_counter.add.call_count == 2 + _, attempt_kwargs = recorder.attempts_counter.add.call_args_list[0] + assert attempt_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert attempt_kwargs["attributes"]["http.response.status_code"] == 503 + assert attempt_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert attempt_kwargs["attributes"]["network.path"] == "cloudpath" + assert "url.template" not in attempt_kwargs["attributes"] + assert "http.request.method" not in attempt_kwargs["attributes"] + assert "gcp.client.method" not in attempt_kwargs["attributes"] + + # Retry counter incremented once for the second attempt after 503 UNAVAILABLE + recorder.retries_counter.add.assert_called_once() + _, http_retry_kwargs = recorder.retries_counter.add.call_args + assert http_retry_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert http_retry_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert http_retry_kwargs["attributes"]["network.path"] == "cloudpath" + assert "gcp.client.method" not in http_retry_kwargs["attributes"] + + # Only the 503 attempt records an error (no gcp.client.method on errors) + recorder.errors_counter.add.assert_called_once() + _, err_kwargs = recorder.errors_counter.add.call_args + assert err_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert err_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert err_kwargs["attributes"]["network.path"] == "cloudpath" + assert "gcp.client.method" not in err_kwargs["attributes"] + assert "url.template" not in err_kwargs["attributes"] + assert "http.response.status_code" not in err_kwargs["attributes"] + + recorder.request_body_size.record.assert_called_once() + recorder.response_body_size.record.assert_called_once() + recorder.ttfb.record.assert_called_once() + _, ttfb_kwargs = recorder.ttfb.record.call_args + assert ttfb_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert ttfb_kwargs["attributes"]["network.path"] == "cloudpath" + assert "gcp.client.method" not in ttfb_kwargs["attributes"] + + # gRPC attempt recording (UNAVAILABLE -> OK retry sequence) + recorder.errors_counter.add.reset_mock() + recorder.attempts_counter.add.reset_mock() + recorder.retries_counter.add.reset_mock() + with _opentelemetry_metrics.start_operation( + recorder, + "storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + "ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ): + recorder.record_rpc_attempt( + duration=0.03, + operation="ReadObject", + status_code="UNAVAILABLE", + error_type="UNAVAILABLE", + rpc_method="google.storage.v2.Storage/BidiReadObject", + network_path="directpath", + ) + recorder.record_rpc_attempt( + duration=0.02, + operation="ReadObject", + status_code="OK", + error_type="OK", + rpc_method="google.storage.v2.Storage/BidiReadObject", + network_path="directpath", + ) + assert recorder.rpc_call_duration.record.call_count == 2 + _, rpc_dur_kwargs = recorder.rpc_call_duration.record.call_args_list[1] + assert ( + rpc_dur_kwargs["attributes"]["rpc.method"] + == "google.storage.v2.Storage/BidiReadObject" + ) + assert rpc_dur_kwargs["attributes"]["rpc.response.status_code"] == "OK" + assert rpc_dur_kwargs["attributes"]["error.type"] == "OK" + assert rpc_dur_kwargs["attributes"]["network.path"] == "directpath" + assert "gcp.storage.operation" not in rpc_dur_kwargs["attributes"] + assert "gcp.client.method" not in rpc_dur_kwargs["attributes"] + + assert recorder.attempts_counter.add.call_count == 2 + _, rpc_attempt_kwargs = recorder.attempts_counter.add.call_args_list[1] + assert rpc_attempt_kwargs["attributes"]["gcp.storage.operation"] == "ReadObject" + assert rpc_attempt_kwargs["attributes"]["rpc.response.status_code"] == "OK" + assert rpc_attempt_kwargs["attributes"]["error.type"] == "OK" + assert rpc_attempt_kwargs["attributes"]["network.path"] == "directpath" + assert "gcp.client.method" not in rpc_attempt_kwargs["attributes"] + assert "rpc.method" not in rpc_attempt_kwargs["attributes"] + + recorder.errors_counter.add.assert_called_once() + _, rpc_err_kwargs = recorder.errors_counter.add.call_args + assert rpc_err_kwargs["attributes"]["gcp.storage.operation"] == "ReadObject" + assert rpc_err_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert rpc_err_kwargs["attributes"]["network.path"] == "directpath" + assert "gcp.client.method" not in rpc_err_kwargs["attributes"] + assert "rpc.response.status_code" not in rpc_err_kwargs["attributes"] + + recorder.retries_counter.add.assert_called_once() + _, rpc_retry_kwargs = recorder.retries_counter.add.call_args + assert rpc_retry_kwargs["attributes"]["gcp.storage.operation"] == "ReadObject" + assert rpc_retry_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert rpc_retry_kwargs["attributes"]["network.path"] == "directpath" + + # Record outside of start_operation with status_code=None and callable .code() returning None + recorder.record_http_attempt( + duration=0.01, + http_method="GET", + url_template="/storage/v1/b", + status_code=None, + error_type="UNKNOWN", + ) + recorder.record_rpc_attempt( + duration=0.01, + operation="google.storage.v2.Storage/GetObject", + status_code=None, + error_type="UNKNOWN", + ) + recorder.record_request_body_size(64) + + class _NoneCodeErr(Exception): + def code(self): + return None + + class _NonTupleArgsErr(Exception): + args = 123 # type: ignore[assignment] + + assert _opentelemetry_metrics.classify_error_type(_NoneCodeErr()) == "UNKNOWN" + assert _opentelemetry_metrics.classify_error_type(_NonTupleArgsErr()) == "UNKNOWN" + assert ( + _opentelemetry_metrics.classify_error_type(concurrent.futures.CancelledError()) + == "CANCELLED" + ) + assert ( + _opentelemetry_metrics._is_subclass_by_module( + mock.MagicMock(__mro__=123), "google.cloud.storage", {"DataCorruption"} + ) + is False + ) + assert ( + _opentelemetry_metrics.classify_error_type( + urllib3.exceptions.NewConnectionError(None, "conn err") + ) + == "CONNECTION_ERROR" + ) + + # Shutdown flushes custom meter_provider and swallows force_flush errors gracefully + recorder.shutdown() + mock_provider.force_flush.assert_called_once() + + mock_provider.force_flush.side_effect = RuntimeError("flush failed") + recorder.shutdown() # Must not raise + + +def test_get_active_recorder_and_grpc_uri_schemes(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + + assert ( + _opentelemetry_metrics._extract_server_address("dns:///custom.endpoint.com:443") + == "custom.endpoint.com" + ) + assert ( + _opentelemetry_metrics._extract_server_address( + "google-c2p-experimental:///storage.googleapis.com" + ) + == "storage.googleapis.com" + ) + + # Unconfigured MagicMock should not be treated as an active recorder + unconfigured_mock = mock.MagicMock() + assert _opentelemetry_metrics.get_active_recorder(unconfigured_mock) is None + assert _opentelemetry_metrics.get_active_recorder(None) is None + + active_rec = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) + holder = mock.MagicMock() + holder._metrics_recorder = active_rec + assert _opentelemetry_metrics.get_active_recorder(holder) is active_rec + + disabled_rec = _opentelemetry_metrics._MetricsRecorder(enable_metrics=False) + holder._metrics_recorder = disabled_rec + assert _opentelemetry_metrics.get_active_recorder(holder) is None + + +def test_metrics_recorder_disabled_no_ops(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", False) + recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) + assert recorder.standard_enabled is False + assert recorder.enabled is False + + # All recording methods are safe no-ops when disabled + recorder.record_logical_operation(0.1, "m", "op") + recorder.record_retry("ReadObject", "UNAVAILABLE", "cloudpath") + recorder.record_http_attempt(0.1, "GET", "/storage/v1/b") + recorder.record_rpc_attempt(0.1, "google.storage.v2.Storage/ReadObject") + recorder.record_request_body_size(100) + recorder.record_response_body_size(100) + recorder.record_ttfb(0.01) + recorder.shutdown() + + +def test_normalize_logical_operation_and_resolve_rpc_method(): + assert _opentelemetry_metrics.normalize_logical_operation(None) == "UNKNOWN" + assert _opentelemetry_metrics.normalize_logical_operation("") == "UNKNOWN" + assert _opentelemetry_metrics.normalize_logical_operation(123) == "UNKNOWN" # type: ignore[arg-type] + assert _opentelemetry_metrics.normalize_logical_operation("///") == "UNKNOWN" + assert ( + _opentelemetry_metrics.normalize_logical_operation( + "google.storage.v2.Storage/BidiReadObject" + ) + == "ReadObject" + ) + assert ( + _opentelemetry_metrics.normalize_logical_operation( + "/google.storage.v2.Storage/BidiWriteObject" + ) + == "WriteObject" + ) + assert ( + _opentelemetry_metrics.normalize_logical_operation( + "google.storage.v2.Storage/GetObject" + ) + == "GetObject" + ) + assert ( + _opentelemetry_metrics.normalize_logical_operation("GetNotificationConfig") + == "GetNotificationConfig" + ) + + assert _opentelemetry_metrics.resolve_rpc_method(None, None) == "UNKNOWN" + assert _opentelemetry_metrics.resolve_rpc_method("", "") == "UNKNOWN" + assert _opentelemetry_metrics.resolve_rpc_method("///", None) == "UNKNOWN" + assert _opentelemetry_metrics.resolve_rpc_method("UNKNOWN", None) == "UNKNOWN" + assert ( + _opentelemetry_metrics.resolve_rpc_method( + "ReadObject", "google.storage.v2.Storage/BidiReadObject" + ) + == "google.storage.v2.Storage/BidiReadObject" + ) + assert ( + _opentelemetry_metrics.resolve_rpc_method("ReadObject") + == "google.storage.v2.Storage/ReadObject" + ) + assert ( + _opentelemetry_metrics.resolve_rpc_method("GetNotificationConfig") + == "google.storage.v1.Storage/GetNotificationConfig" + ) + assert ( + _opentelemetry_metrics.resolve_rpc_method("storage.objects.get") + == "storage.objects.get" + ) + + +def test_iter_exception_chain_traverses_errors_list(): + wrapped = api_exceptions.Aborted( + "aborted", + errors=[socket.timeout("nested timeout in errors")], + ) + chain = list(_opentelemetry_metrics._iter_exception_chain(wrapped)) + assert any(isinstance(item, socket.timeout) for item in chain) + + class _NonSeqErrorsErr(Exception): + errors = 123 + + assert _opentelemetry_metrics.classify_error_type(_NonSeqErrorsErr()) == "UNKNOWN" + + +def test_operation_state_and_retry_helpers(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + disabled_recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=False) + + # No-ops when recorder is None, disabled, or no active operation + _opentelemetry_metrics.record_pending_retry(None) + _opentelemetry_metrics.record_pending_retry(disabled_recorder) + _opentelemetry_metrics.record_pending_retry(recorder) + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + None, RuntimeError("e"), attempts_before=0, will_retry=True + ) + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + disabled_recorder, RuntimeError("e"), attempts_before=0, will_retry=True + ) + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, RuntimeError("e"), attempts_before=0, will_retry=True + ) + + # Standalone record_retry and record_ttfb outside start_operation without optional attrs + recorder.record_retry(operation=None, error_type="UNAVAILABLE", network_path=None) + _, retry_no_op_kw = recorder.retries_counter.add.call_args + assert "gcp.storage.operation" not in retry_no_op_kw["attributes"] + assert "network.path" not in retry_no_op_kw["attributes"] + + recorder.record_ttfb(0.02, operation=None, network_path=None) + _, ttfb_no_op_kw = recorder.ttfb.record.call_args + assert "gcp.storage.operation" not in ttfb_no_op_kw["attributes"] + assert "network.path" not in ttfb_no_op_kw["attributes"] + + recorder.retries_counter.add.reset_mock() + with _opentelemetry_metrics.start_operation( + recorder, + "storage.asyncio.AsyncMultiRangeDownloader.open", + "ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ) as op_state: + assert op_state is not None + op_state.mark_attempt_failed("OK") + op_state.mark_attempt_failed("") + assert op_state.last_failure_error_type is None + + # Pending retry when no failure occurred is a no-op + _opentelemetry_metrics.record_pending_retry(recorder) + recorder.retries_counter.add.assert_not_called() + + # When transport already recorded the attempt, handle_unrecorded_retryable_failure skips + op_state.recorded_attempts = 1 + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + api_exceptions.ServiceUnavailable("503"), + attempts_before=0, + will_retry=True, + network_path="directpath", + ) + assert op_state.recorded_attempts == 1 + assert op_state.last_failure_error_type is None + + # Unrecorded failure 1 (will_retry=True) -> marks failure, no retry yet + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + api_exceptions.ServiceUnavailable("503"), + attempts_before=1, + will_retry=True, + network_path="unknown", + ) + assert op_state.recorded_attempts == 2 + assert op_state.last_failure_error_type == "UNAVAILABLE" + recorder.retries_counter.add.assert_not_called() + + # Consecutive unrecorded failure 2 (will_retry=True) -> emits retry for failure 1, marks failure 2 + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + api_exceptions.DeadlineExceeded("timeout"), + attempts_before=2, + will_retry=True, + network_path="cloudpath", + ) + assert op_state.recorded_attempts == 3 + assert op_state.last_failure_error_type == "DEADLINE_EXCEEDED" + recorder.retries_counter.add.assert_called_once() + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "UNAVAILABLE" + ) + + # Unrecorded terminal failure 3 (will_retry=False) -> emits retry for failure 2, leaves error cleared + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + api_exceptions.PermissionDenied("403"), + attempts_before=3, + will_retry=False, + network_path="cloudpath", + ) + assert op_state.recorded_attempts == 4 + assert op_state.last_failure_error_type is None + assert recorder.retries_counter.add.call_count == 2 + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "DEADLINE_EXCEEDED" + ) + + # Mark failure and consume via record_pending_retry + op_state.mark_attempt_failed("ABORTED") + _opentelemetry_metrics.record_pending_retry(recorder, network_path="directpath") + assert recorder.retries_counter.add.call_count == 3 + _, pending_kw = recorder.retries_counter.add.call_args + assert pending_kw["attributes"]["gcp.storage.operation"] == "ReadObject" + assert pending_kw["attributes"]["error.type"] == "ABORTED" + assert pending_kw["attributes"]["network.path"] == "directpath" + + import contextvars + + copied_ctx = contextvars.copy_context() + + # Once start_operation exits, state.finished is True even in copied contexts + assert op_state.finished is True + assert copied_ctx.run(_opentelemetry_metrics.get_current_operation) is None diff --git a/packages/google-cloud-storage/tests/unit/test_otel_fileio_metrics.py b/packages/google-cloud-storage/tests/unit/test_otel_fileio_metrics.py new file mode 100644 index 000000000000..7915f82fd901 --- /dev/null +++ b/packages/google-cloud-storage/tests/unit/test_otel_fileio_metrics.py @@ -0,0 +1,683 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for OpenTelemetry streaming file I/O (BlobReader, BlobWriter, Blob.open) metrics.""" + +import base64 +import gc +import http.client +import io +import json +from unittest import mock + +import google_crc32c +import pytest +import requests +from google.auth import credentials as auth_credentials +from google.cloud.exceptions import NotFound +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader + +from google.cloud.storage import _helpers, _opentelemetry_metrics, fileio +from google.cloud.storage._media.requests import _request_helpers +from google.cloud.storage.client import Client +from google.cloud.storage.exceptions import InvalidResponse + + +def _make_credentials(): + return auth_credentials.AnonymousCredentials() + + +def _crc32c_b64(data: bytes) -> str: + checksum = google_crc32c.Checksum(data) + return base64.b64encode(checksum.digest()).decode("ascii") + + +class _FakeRawStream: + """Minimal urllib3-like raw response backing streaming chunk reads.""" + + def __init__(self, chunks): + self._chunks = list(chunks) + self.headers = {} + + def stream(self, amt=8192, decode_content=None): + for chunk in self._chunks: + yield chunk + + +def _media_response( + body=b"", + status_code=http.client.OK, + headers=None, + streaming=False, +): + response = requests.Response() + response.status_code = status_code + response.headers.update(headers or {}) + chunk_list = [body] if body else [] + response.raw = _FakeRawStream(chunk_list) + if streaming: + response._content = False + else: + response._content = body + response.request = requests.Request( + "GET", "https://storage.googleapis.com/download/storage/v1/b/b1/o/o1" + ).prepare() + return response + + +def _json_response(payload=None, status_code=http.client.OK): + hdrs = {"Content-Type": "application/json"} + body = b"" if payload is None else json.dumps(payload).encode("utf-8") + return _media_response(body=body, status_code=status_code, headers=hdrs) + + +def _collect_metrics(reader): + data = reader.get_metrics_data() + collected = {} + if data is None: # pragma: NO COVER + return collected + for resource_metric in data.resource_metrics: + for scope_metric in resource_metric.scope_metrics: + for metric in scope_metric.metrics: + collected.setdefault(metric.name, []).extend(metric.data.data_points) + return collected + + +@pytest.fixture(autouse=True) +def _enable_dev_gate(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + + +@pytest.fixture +def metrics_env(): + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + session = mock.MagicMock() + client = Client( + project="test-project", + credentials=_make_credentials(), + _http=session, + enable_metrics=True, + meter_provider=provider, + ) + client._bucket_metadata_cache = None + yield client, session, reader + client.close() + + +def test_bind_suboperation_context_branches(metrics_env): + client, session, _ = metrics_env + recorder = client._metrics_recorder + + with _helpers._suppress_internal_metrics(): + with _helpers._bind_suboperation_context( + client, "storage.fileio.BlobReader", "ReadObject" + ) as st: + assert st is None + + disabled_client = Client( + project="test-project", + credentials=_make_credentials(), + _http=mock.MagicMock(), + enable_metrics=False, + ) + disabled_client._bucket_metadata_cache = None + with _helpers._bind_suboperation_context( + disabled_client, "storage.fileio.BlobReader", "ReadObject" + ) as st: + assert st is None + disabled_client.close() + + plain_transport = mock.MagicMock(spec=["request"]) + with _helpers._bind_suboperation_context( + client, "storage.fileio.BlobReader", "ReadObject" + ) as st: + assert st is not None + assert _helpers._SUPPRESS_BODY_SIZE.get() is True + assert _request_helpers.resolve_transport_recorder(plain_transport) is recorder + assert _helpers._SUPPRESS_BODY_SIZE.get() is False + assert st.finished is True + + +def test_blob_open_and_fileio_stream_lifetime_metrics(metrics_env): + client, session, reader = metrics_env + bucket = client.bucket("b1") + + # 1. Blob.open validation errors before stream construction + b_open = bucket.blob("open-validate") + invalid_open_calls = [ + ( + {"mode": "rb", "encoding": "utf-8"}, + ValueError, + "ReadObject", + "CLIENT_VALIDATION_ERROR", + ), + ( + {"mode": "rb", "ignore_flush": True}, + ValueError, + "ReadObject", + "CLIENT_VALIDATION_ERROR", + ), + ( + {"mode": "wb", "encoding": "utf-8"}, + ValueError, + "WriteObject", + "CLIENT_VALIDATION_ERROR", + ), + ( + {"mode": "rt", "ignore_flush": True}, + ValueError, + "ReadObject", + "CLIENT_VALIDATION_ERROR", + ), + ( + {"mode": "wt", "ignore_flush": False}, + ValueError, + "WriteObject", + "CLIENT_VALIDATION_ERROR", + ), + ({"mode": "rw"}, NotImplementedError, "ReadObject", "UNKNOWN"), + ({"mode": "wx"}, NotImplementedError, "WriteObject", "UNKNOWN"), + ({"mode": "ab"}, NotImplementedError, "UNKNOWN", "UNKNOWN"), + ] + for kwargs, exc_cls, expected_op, expected_err in invalid_open_calls: + with pytest.raises(exc_cls): + b_open.open(**kwargs) + pts_val = _collect_metrics(reader) + assert sum(p.value for p in pts_val["gcp.storage.client.operations"]) == len( + invalid_open_calls + ) + seen_pairs = [ + (p.attributes["gcp.storage.operation"], p.attributes["error.type"]) + for p in pts_val["gcp.storage.client.operations"] + ] + for _, _, expected_op, expected_err in invalid_open_calls: + assert (expected_op, expected_err) in seen_pairs + + # 2. Blob.open TextIOWrapper construction failure cleans up raw_reader / raw_writer + with pytest.raises(ValueError): + b_open.open("rt", newline="invalid-newline") + with pytest.raises(ValueError): + b_open.open("wt", newline="invalid-newline") + + # 3. Blob.open("rb"), ("rt"), ("wb"), ("wt") attribute stream lifetime to storage.Blob.open + # and record exact caller-consumed / caller-written bytes (not prefetched bytes!) + read_blob = bucket.blob("stream-read") + session.request.side_effect = [ + _media_response( + b"01234567", + status_code=http.client.PARTIAL_CONTENT, + headers={"x-goog-generation": "1", "content-range": "bytes 0-7/16"}, + streaming=True, + ), + _media_response( + b"hello", + status_code=http.client.PARTIAL_CONTENT, + headers={"x-goog-generation": "1", "content-range": "bytes 0-4/5"}, + streaming=True, + ), + _json_response( + {"error": {"message": "range not satisfiable"}}, status_code=416 + ), + ] + with read_blob.open("rb", chunk_size=8) as r_bin: + assert r_bin.read1(3) == b"012" + r_bin.close() # Idempotent second close emits nothing + + with read_blob.open("rt", chunk_size=8, encoding="utf-8") as r_txt: + assert r_txt.read() == "hello" + + write_blob = bucket.blob("stream-write") + session_uri = "https://storage.googleapis.com/upload/storage/v1/b/b1/o?uploadType=resumable&upload_id=w1" + chunk_sz = 256 * 1024 + wb_payload = b"x" * chunk_sz + b"12345" + session.request.side_effect = [ + _media_response(b"", status_code=200, headers={"location": session_uri}), + _media_response( + b"", + status_code=http.client.PERMANENT_REDIRECT, + headers={"range": f"bytes=0-{chunk_sz - 1}"}, + ), + _json_response( + { + "name": "stream-write", + "size": str(chunk_sz + 5), + "crc32c": _crc32c_b64(wb_payload), + } + ), + _media_response(b"", status_code=200, headers={"location": session_uri}), + _json_response( + { + "name": "stream-write", + "size": "4", + "crc32c": _crc32c_b64(b"text"), + } + ), + ] + with write_blob.open("wb", chunk_size=chunk_sz) as w_bin: + w_bin.write(b"x" * chunk_sz + b"12345") + w_bin.close() # Idempotent second close + + with write_blob.open("wt", encoding="utf-8") as w_txt: + w_txt.write("text") + + pts_streams = _collect_metrics(reader) + resp_pts = [ + p + for p in pts_streams["gcp.storage.client.response.body.size"] + if p.attributes["gcp.client.method"] == "storage.Blob.open" + ] + assert sum(p.sum for p in resp_pts) == 8 + assert sum(p.count for p in resp_pts) == 2 + + req_pts = [ + p + for p in pts_streams["gcp.storage.client.request.body.size"] + if p.attributes["gcp.client.method"] == "storage.Blob.open" + ] + assert sum(p.sum for p in req_pts) == (chunk_sz + 5) + 4 + assert sum(p.count for p in req_pts) == 2 + + +def test_direct_blob_reader_and_writer_edge_cases_and_errors(metrics_env): + client, session, reader = metrics_env + bucket = client.bucket("b1") + blob = bucket.blob("direct-fileio") + + # 1. Suppressed / disabled BlobReader & BlobWriter construction (with valid kwargs) + with _helpers._suppress_internal_metrics(): + sup_r = fileio.BlobReader(blob, raw_download=True) + sup_r.close() + sup_w = fileio.BlobWriter(blob, content_type="text/plain") + sup_w.terminate() + + # 2. Invalid constructor arguments on direct BlobReader & BlobWriter + with pytest.raises(ValueError): + fileio.BlobReader(blob, unsupported_kwarg=123) + with pytest.raises(ValueError): + fileio.BlobWriter(blob, unsupported_kwarg=123) + with pytest.raises(ValueError): + fileio.BlobWriter(blob, chunk_size=100) # Not a multiple of 256 KiB + + # 3. BlobReader seek(whence=2) triggering blob.reload() (success & failure) + invalid whence + seek_blob = bucket.blob("seek-blob") + session.request.side_effect = [ + _json_response({"name": "seek-blob", "size": "10", "generation": "1"}), + ] + sr = fileio.BlobReader(seek_blob, chunk_size=4) + assert sr.seek(-2, 2) == 8 + with pytest.raises(ValueError): + sr.seek(0, 99) + sr.close() + + seek_fail_blob = bucket.blob("seek-fail") + session.request.side_effect = [ + _json_response({"error": {"message": "missing"}}, status_code=404), + ] + sr_fail = fileio.BlobReader(seek_fail_blob, retry=None) + with pytest.raises(NotFound): + sr_fail.seek(0, 2) + sr_fail.close() + + # 4. BlobReader mid-stream read error (with 0 bytes read vs >0 bytes read) + err_blob = bucket.blob("read-err") + session.request.side_effect = [ + _json_response({"error": {"message": "missing"}}, status_code=404), + _media_response( + b"abcd", + status_code=http.client.PARTIAL_CONTENT, + headers={"x-goog-generation": "1", "content-range": "bytes 0-3/8"}, + streaming=True, + ), + _json_response({"error": {"message": "gone"}}, status_code=404), + ] + r_zero = fileio.BlobReader(err_blob, chunk_size=4, retry=None) + with pytest.raises(NotFound): + r_zero.read(4) + r_zero.close() + + r_partial = fileio.BlobReader(err_blob, chunk_size=4, retry=None) + assert r_partial.read(4) == b"abcd" + with pytest.raises(NotFound): + r_partial.read(4) + r_partial.close() + + # 5. BlobWriter terminate, write error, and close error + session_uri = "https://storage.googleapis.com/upload/storage/v1/b/b1/o?uploadType=resumable&upload_id=term1" + chunk_sz = 256 * 1024 + + w_cancel_early = fileio.BlobWriter(blob, chunk_size=chunk_sz) + w_cancel_early.terminate() + + session.request.side_effect = [ + _media_response(b"", status_code=200, headers={"location": session_uri}), + _media_response( + b"", + status_code=http.client.PERMANENT_REDIRECT, + headers={"range": f"bytes=0-{chunk_sz - 1}"}, + ), + ] + session.delete.return_value = _json_response(None, status_code=499) + w_cancel_active = fileio.BlobWriter(blob, chunk_size=chunk_sz) + w_cancel_active.write(b"a" * chunk_sz) + w_cancel_active.terminate() + session.delete.assert_called_once() + + session.request.side_effect = [ + _json_response({"error": {"message": "denied"}}, status_code=403), + ] + with pytest.raises(InvalidResponse): + with fileio.BlobWriter(blob, chunk_size=chunk_sz, retry=None) as w_err: + w_err.write(b"a" * chunk_sz) + + session.request.side_effect = [ + _json_response({"error": {"message": "unavailable"}}, status_code=503), + ] + w_close_err = fileio.BlobWriter(blob, chunk_size=chunk_sz, retry=None) + w_close_err.write(b"small") + with pytest.raises(InvalidResponse): + w_close_err.close() + + pts = _collect_metrics(reader) + reader_ops = [ + p + for p in pts["gcp.storage.client.operations"] + if p.attributes["gcp.client.method"] == "storage.fileio.BlobReader" + ] + writer_ops = [ + p + for p in pts["gcp.storage.client.operations"] + if p.attributes["gcp.client.method"] == "storage.fileio.BlobWriter" + ] + assert sum(p.value for p in reader_ops) == 5 + assert sum(p.value for p in writer_ops) == 6 + cancelled_pts = [p for p in writer_ops if p.attributes["error.type"] == "CANCELLED"] + assert sum(p.value for p in cancelled_pts) == 2 + assert not any( + p.attributes["gcp.storage.operation"] == "GetObject" + for p in pts["gcp.storage.client.operations"] + ) + + +def test_text_io_wrapper_gc_safety_and_error_recovery(metrics_env): + client, session, reader = metrics_env + bucket = client.bucket("b1") + session_uri = "https://storage.googleapis.com/upload/storage/v1/b/b1/o?uploadType=resumable&upload_id=txt1" + + # 1. Disabled client preserves exact stdlib io.TextIOWrapper; enabled uses _MetricsTextIOWrapper + disabled_client = Client( + project="test-project", + credentials=_make_credentials(), + _http=mock.MagicMock(), + enable_metrics=False, + ) + disabled_client._bucket_metadata_cache = None + d_blob = disabled_client.bucket("b1").blob("plain-txt") + t_read = d_blob.open("rt") + t_write = d_blob.open("wt") + assert type(t_read) is io.TextIOWrapper + assert type(t_write) is io.TextIOWrapper + t_read.close() + t_write.buffer.terminate() + disabled_client.close() + + # 2. _MetricsTextIOWrapper read, readline, seek(0), __next__, UnsupportedOperation, detached + txt_blob = bucket.blob("txt-read") + txt_blob._properties["size"] = 12 + session.request.side_effect = [ + _media_response( + b"line1\nline2\n", + status_code=http.client.PARTIAL_CONTENT, + headers={"x-goog-generation": "1", "content-range": "bytes 0-11/12"}, + streaming=True, + ), + _json_response({"error": {"message": "eof"}}, status_code=416), + _media_response( + b"\xff\xfe", + status_code=http.client.PARTIAL_CONTENT, + headers={"x-goog-generation": "1", "content-range": "bytes 0-1/2"}, + streaming=True, + ), + _media_response( + b"ok\n\xff\xfe", + status_code=http.client.PARTIAL_CONTENT, + headers={"x-goog-generation": "1", "content-range": "bytes 0-4/5"}, + streaming=True, + ), + ] + with txt_blob.open("rt", chunk_size=16, encoding="utf-8") as f_ok: + assert type(f_ok) is fileio._MetricsTextIOWrapper + assert f_ok.seek(0) == 0 + assert f_ok.readline() == "line1\n" + assert list(f_ok) == ["line2\n"] + + with pytest.raises(UnicodeDecodeError): + with txt_blob.open("rt", chunk_size=16, encoding="utf-8") as f_bad_read: + f_bad_read.read() + + with pytest.raises(UnicodeDecodeError): + with txt_blob.open("rt", chunk_size=16, encoding="utf-8") as f_bad_iter: + list(f_bad_iter) + + with pytest.raises(io.UnsupportedOperation): + with txt_blob.open("rt", chunk_size=16, encoding="utf-8") as f_bad_seek: + f_bad_seek.seek(1, io.SEEK_CUR) + + with pytest.raises(io.UnsupportedOperation): + with txt_blob.open("rt", chunk_size=16, encoding="utf-8") as f_bad_trunc: + f_bad_trunc.truncate() + + f_detached = txt_blob.open("rt", chunk_size=16, encoding="utf-8") + raw_detached = f_detached.detach() + with pytest.raises(ValueError, match="detached"): + f_detached.read() + with pytest.raises(ValueError, match="detached"): + next(f_detached) + del f_detached + gc.collect() + raw_detached.close() + + # 3. _MetricsTextIOWrapper write, writelines, UnicodeEncodeError abort, context-exit abort, + # closed exit, and BlobWriter.close() primary error preservation + chunk_sz = 256 * 1024 + w_txt_blob = bucket.blob("txt-write") + session.request.side_effect = [ + _media_response(b"", status_code=200, headers={"location": session_uri}), + _json_response( + {"name": "txt-write", "size": "6", "crc32c": _crc32c_b64(b"a\nb\nc\n")} + ), + _media_response(b"", status_code=200, headers={"location": session_uri}), + _json_response( + {"name": "txt-write", "size": "2", "crc32c": _crc32c_b64(b"hi")} + ), + ] + with w_txt_blob.open("wt", encoding="ascii") as fw_ok: + fw_ok.writelines(["a\n", "b\n"]) + fw_ok.write("c\n") + + session.request.reset_mock() + session.request.side_effect = None + with pytest.raises(UnicodeEncodeError): + with w_txt_blob.open("wt", encoding="ascii") as fw_enc_err: + fw_enc_err.write("\u2603") + assert session.request.call_count == 0 + assert fw_enc_err.closed + + with pytest.raises(RuntimeError, match="caller-boom"): + with w_txt_blob.open("wt", encoding="utf-8") as fw_abort: + fw_abort.write("buffered-data-never-flushed") + raise RuntimeError("caller-boom") + assert session.request.call_count == 0 + assert fw_abort.closed + + session.request.side_effect = [ + _media_response(b"", status_code=200, headers={"location": session_uri}), + _json_response( + {"name": "txt-write", "size": "2", "crc32c": _crc32c_b64(b"hi")} + ), + ] + with pytest.raises(RuntimeError, match="after-explicit-close"): + with w_txt_blob.open("wt", encoding="utf-8") as fw_already_closed: + fw_already_closed.write("hi") + fw_already_closed.close() + raise RuntimeError("after-explicit-close") + + session.request.side_effect = [ + _media_response(b"", status_code=200, headers={"location": session_uri}), + _json_response({"error": {"message": "forbidden"}}, status_code=403), + ] + fw_flush_403 = w_txt_blob.open( + "wt", encoding="ascii", chunk_size=chunk_sz, retry=None + ) + fw_flush_403.buffer.write(b"a" * (chunk_sz - 2)) + fw_flush_403.write("bc") + with pytest.raises(ValueError, match="invalid state"): + fw_flush_403.close() + + session.request.side_effect = [ + _json_response({"error": {"message": "unavailable"}}, status_code=503), + ] + fw_close_503 = w_txt_blob.open( + "wt", encoding="ascii", chunk_size=chunk_sz, retry=None + ) + fw_close_503.write("ok") + with pytest.raises(UnicodeEncodeError): + fw_close_503.write("\u2603") + with pytest.raises(InvalidResponse): + fw_close_503.close() + + # 4. GC safety (__del__) on unclosed streams + pts_before_gc = _collect_metrics(reader) + ops_before_gc = sum(p.value for p in pts_before_gc["gcp.storage.client.operations"]) + session.request.reset_mock() + session.request.side_effect = AssertionError("GC must never trigger network I/O") + + unclosed_r = fileio.BlobReader(bucket.blob("gc-r")) + unclosed_w = fileio.BlobWriter(bucket.blob("gc-w")) + unclosed_w.write(b"unflushed") + uninit_r = fileio.BlobReader.__new__(fileio.BlobReader) + unclosed_tw = bucket.blob("gc-tw").open("wt") + unclosed_tw.write("unflushed-text") + del unclosed_r, unclosed_w, uninit_r, unclosed_tw + gc.collect() + + assert session.request.call_count == 0 + pts_after_gc = _collect_metrics(reader) + ops_after_gc = sum(p.value for p in pts_after_gc["gcp.storage.client.operations"]) + assert ops_after_gc == ops_before_gc + + # 5. BlobWriter.terminate() finalizes CANCELLED metric and closes buffer even if transport.delete raises + session.request.side_effect = [ + _media_response(b"", status_code=200, headers={"location": session_uri}), + _media_response( + b"", + status_code=http.client.PERMANENT_REDIRECT, + headers={"range": f"bytes=0-{chunk_sz - 1}"}, + ), + ] + session.delete.side_effect = requests.ConnectionError("abort transport down") + w_term_err = fileio.BlobWriter(bucket.blob("term-err"), chunk_size=chunk_sz) + w_term_err.write(b"x" * chunk_sz) + with pytest.raises(requests.ConnectionError, match="abort transport down"): + w_term_err.terminate() + assert w_term_err.closed + session.delete.side_effect = None + + # 6. Transient error recovery on BlobReader & BlobWriter + rec_r_blob = bucket.blob("recover-read") + rec_r_blob._properties["size"] = 4 + session.request.side_effect = [ + _json_response({"error": {"message": "transient"}}, status_code=404), + _media_response( + b"good", + status_code=http.client.PARTIAL_CONTENT, + headers={"x-goog-generation": "1", "content-range": "bytes 0-3/4"}, + streaming=True, + ), + ] + rec_r = fileio.BlobReader(rec_r_blob, chunk_size=4, retry=None) + with pytest.raises(ValueError): + rec_r.seek(0, 99) + with pytest.raises(NotFound): + rec_r.read(4) + assert rec_r.read(4) == b"good" + rec_r.close() + + rec_w_blob = bucket.blob("recover-write") + w_data = b"a" * chunk_sz + b"b" * chunk_sz + session.request.side_effect = [ + _json_response({"error": {"message": "transient"}}, status_code=503), + _media_response(b"", status_code=200, headers={"location": session_uri}), + _media_response( + b"", + status_code=http.client.PERMANENT_REDIRECT, + headers={"range": f"bytes=0-{chunk_sz - 1}"}, + ), + _media_response( + b"", + status_code=http.client.PERMANENT_REDIRECT, + headers={"range": f"bytes=0-{2 * chunk_sz - 1}"}, + ), + _json_response( + { + "name": "recover-write", + "size": str(2 * chunk_sz), + "crc32c": _crc32c_b64(w_data), + } + ), + ] + rec_w = fileio.BlobWriter(rec_w_blob, chunk_size=chunk_sz, retry=None) + with pytest.raises(InvalidResponse): + rec_w.write(b"a" * chunk_sz) + rec_w.write(b"b" * chunk_sz) + rec_w.close() + + pts_final = _collect_metrics(reader) + r_ok = [ + p + for p in pts_final["gcp.storage.client.operations"] + if p.attributes["gcp.client.method"] == "storage.fileio.BlobReader" + and p.attributes["error.type"] == "OK" + ] + assert sum(p.value for p in r_ok) == 1 + w_ops = [ + p + for p in pts_final["gcp.storage.client.operations"] + if p.attributes["gcp.client.method"] == "storage.fileio.BlobWriter" + ] + assert {(p.attributes["error.type"], p.value) for p in w_ops} == { + ("CANCELLED", 1), + ("OK", 1), + } + open_ops = [ + p + for p in pts_final["gcp.storage.client.operations"] + if p.attributes["gcp.client.method"] == "storage.Blob.open" + ] + open_by_pair = {} + for p in open_ops: + k = (p.attributes["gcp.storage.operation"], p.attributes["error.type"]) + open_by_pair[k] = open_by_pair.get(k, 0) + p.value + assert open_by_pair == { + ("ReadObject", "OK"): 2, + ("ReadObject", "CLIENT_VALIDATION_ERROR"): 2, + ("ReadObject", "CONNECTION_ERROR"): 2, + ("WriteObject", "OK"): 2, + ("WriteObject", "CLIENT_VALIDATION_ERROR"): 1, + ("WriteObject", "CANCELLED"): 1, + ("WriteObject", "PERMISSION_DENIED"): 1, + ("WriteObject", "UNAVAILABLE"): 1, + } diff --git a/packages/google-cloud-storage/tests/unit/test_otel_http_media_metrics.py b/packages/google-cloud-storage/tests/unit/test_otel_http_media_metrics.py new file mode 100644 index 000000000000..616871066a59 --- /dev/null +++ b/packages/google-cloud-storage/tests/unit/test_otel_http_media_metrics.py @@ -0,0 +1,982 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for OpenTelemetry HTTP media downloads, uploads, and XML MPU metrics.""" + +import base64 +import datetime +import http.client +import io +import json +from unittest import mock + +import google_crc32c +import pytest +import requests +from google.api_core import retry +from google.auth import credentials as auth_credentials +from google.cloud.exceptions import Forbidden, NotFound +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader + +from google.cloud.storage import _helpers, _opentelemetry_metrics +from google.cloud.storage._media.requests import _request_helpers +from google.cloud.storage._media.requests.download import ( + ChunkedDownload, + Download, + RawChunkedDownload, + RawDownload, +) +from google.cloud.storage._media.requests.upload import ( + MultipartUpload, + ResumableUpload, + SimpleUpload, + XMLMPUContainer, + XMLMPUPart, +) +from google.cloud.storage.client import Client +from google.cloud.storage.exceptions import DataCorruption, InvalidResponse +from google.cloud.storage.retry import DEFAULT_RETRY_IF_GENERATION_SPECIFIED + + +def _make_credentials(): + return auth_credentials.AnonymousCredentials() + + +def _crc32c_b64(data: bytes) -> str: + checksum = google_crc32c.Checksum(data) + return base64.b64encode(checksum.digest()).decode("ascii") + + +class _FakeRawStream: + """Minimal urllib3-like raw response backing streaming and single-shot reads.""" + + def __init__(self, chunks, error_after=None): + self._chunks = list(chunks) + self._buf = b"".join(self._chunks) + self._error_after = error_after + self.headers = {} + + def read(self, amt=None, decode_content=None): + if amt is None: + data, self._buf = self._buf, b"" + self._chunks = [] + return data + data, self._buf = self._buf[:amt], self._buf[amt:] + if not self._buf: + self._chunks = [] + return data + + def stream(self, amt=8192, decode_content=None): + for idx, chunk in enumerate(self._chunks): + if self._error_after is not None and idx >= self._error_after: + raise requests.ReadTimeout("mid-stream timeout") + yield chunk + + def close(self): + pass + + +def _media_response( + body=b"", + status_code=http.client.OK, + headers=None, + chunks=None, + streaming=False, + error_after=None, +): + response = requests.Response() + response.status_code = status_code + if headers: + response.headers.update(headers) + chunk_list = list(chunks) if chunks is not None else ([body] if body else []) + response.raw = _FakeRawStream(chunk_list, error_after=error_after) + if streaming: + response._content = False + else: + response._content = body + response.request = requests.Request( + "GET", "https://storage.googleapis.com/download/storage/v1/b/b1/o/o1" + ).prepare() + return response + + +def _json_response(payload=None, status_code=http.client.OK): + hdrs = {"Content-Type": "application/json"} + body = b"" if payload is None else json.dumps(payload).encode("utf-8") + return _media_response(body=body, status_code=status_code, headers=hdrs) + + +def _collect_metrics(reader): + data = reader.get_metrics_data() + collected = {} + if data is None: # pragma: NO COVER + return collected + for resource_metric in data.resource_metrics: + for scope_metric in resource_metric.scope_metrics: + for metric in scope_metric.metrics: + collected.setdefault(metric.name, []).extend(metric.data.data_points) + return collected + + +@pytest.fixture(autouse=True) +def _enable_dev_gate(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + + +@pytest.fixture +def metrics_env(): + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + session = mock.MagicMock() + client = Client( + project="test-project", + credentials=_make_credentials(), + _http=session, + enable_metrics=True, + meter_provider=provider, + ) + client._bucket_metadata_cache = None + yield client, session, reader + client.close() + + +def test_request_helpers_branches(metrics_env): + client, session, reader = metrics_env + recorder = client._metrics_recorder + + # 1. resolve_transport_recorder suppressed, disabled, ContextVar, and transport attribute + with _helpers._suppress_internal_metrics(): + assert _request_helpers.resolve_transport_recorder(session) is None + + disabled_client = Client( + project="test-project", + credentials=_make_credentials(), + _http=mock.MagicMock(), + enable_metrics=False, + ) + disabled_client._bucket_metadata_cache = None + assert _request_helpers.resolve_transport_recorder(disabled_client._http) is None + disabled_client.close() + + plain_transport = mock.MagicMock(spec=["request"]) + assert _request_helpers.resolve_transport_recorder(plain_transport) is None + with _helpers._start_storage_operation( + client, "storage.Blob.download_to_file", "ReadObject" + ): + assert _request_helpers.resolve_transport_recorder(plain_transport) is recorder + assert _request_helpers.resolve_transport_recorder(session) is recorder + + # Inactive recorder in _ACTIVE_METRICS_RECORDER falls through to transport + inactive_rec = mock.Mock(standard_enabled=False) + tok = _helpers._ACTIVE_METRICS_RECORDER.set(inactive_rec) + try: + assert _request_helpers.resolve_transport_recorder(session) is recorder + finally: + _helpers._ACTIVE_METRICS_RECORDER.reset(tok) + + # 2. Direct helper edge cases outside wait_and_retry + _request_helpers.record_streaming_ttfb() # No-op when state is None + plain_transport.request.return_value = _json_response({"ok": True}) + direct_resp = _request_helpers.transport_request( + plain_transport, "GET", "https://storage.googleapis.com/b1/o1" + ) + assert direct_resp.status_code == 200 + assert _request_helpers._extract_status_code(direct_resp) == 200 + assert _request_helpers._extract_status_code(mock.Mock(status_code=True)) is None + assert _request_helpers._extract_status_code(None) is None + + # 3. transport_request when ttfb_recorded is already True vs streaming=False + manual_state = _request_helpers._MediaAttemptState( + recorder=recorder, + method="GET", + fallback_url="https://storage.googleapis.com/b1/o1", + streaming=False, + ) + manual_state.ttfb_recorded = True + st_tok = _request_helpers._MEDIA_ATTEMPT_STATE.set(manual_state) + try: + _request_helpers.transport_request( + plain_transport, "GET", "https://storage.googleapis.com/b1/o1" + ) + finally: + _request_helpers._MEDIA_ATTEMPT_STATE.reset(st_tok) + + # 4. RequestsMixin & RawRequestsMixin._get_body branches: multi-chunk, 0-chunk, and pre-populated _content + req_mixin = _request_helpers.RequestsMixin() + req_chunks = _media_response(chunks=[b"ab", b"cd"], streaming=True) + assert req_mixin._get_body(req_chunks) == b"abcd" + assert req_mixin._get_body(req_chunks) == b"abcd" + req_empty = _media_response(chunks=[], streaming=True) + assert req_mixin._get_body(req_empty) == b"" + + raw_mixin = _request_helpers.RawRequestsMixin() + resp_chunks = _media_response(chunks=[b"ab", b"cd"], streaming=True) + assert raw_mixin._get_body(resp_chunks) == b"abcd" + assert raw_mixin._get_body(resp_chunks) == b"abcd" + resp_empty_stream = _media_response(chunks=[], streaming=True) + assert raw_mixin._get_body(resp_empty_stream) == b"" + + # 5. wait_and_retry edge branches: + fast_retry = retry.Retry( + predicate=retry.if_exception_type(RuntimeError), + initial=0.001, + maximum=0.002, + deadline=1.0, + ) + calls = {"n": 0} + + def _flaky_uninstrumented(): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("retry me") + return "done" + + assert ( + _request_helpers.wait_and_retry( + _flaky_uninstrumented, fast_retry, transport=plain_transport + ) + == "done" + ) + assert ( + _request_helpers.wait_and_retry( + lambda: "no-retry", None, transport=plain_transport + ) + == "no-retry" + ) + + assert ( + _request_helpers.wait_and_retry( + lambda: None, + None, + transport=session, + method="GET", + url="https://storage.googleapis.com/download/storage/v1/b/b1/o/o1", + ) + is None + ) + assert ( + _request_helpers.wait_and_retry( + lambda: _json_response({"ok": True}), + None, + transport=session, + method="GET", + url="https://storage.googleapis.com/download/storage/v1/b/b1/o/o1", + ).status_code + == 200 + ) + + err_with_resp = requests.HTTPError("502 Bad Gateway") + err_with_resp.response = _json_response({"err": "bad gateway"}, status_code=502) + with pytest.raises(requests.HTTPError): + _request_helpers.wait_and_retry( + lambda: (_ for _ in ()).throw(err_with_resp), + None, + transport=session, + method="GET", + url="https://storage.googleapis.com/download/storage/v1/b/b1/o/o1", + ) + + session.request.return_value = _json_response({"ok": True}) + assert ( + _request_helpers.wait_and_retry( + lambda: _request_helpers.transport_request( + session, + "GET", + "https://storage.googleapis.com/download/storage/v1/b/b1/o/o1", + ), + None, + transport=session, + method="GET", + url="https://storage.googleapis.com/download/storage/v1/b/b1/o/o1", + streaming=True, + ).status_code + == 200 + ) + + # 6. Missing status_code (None) must NOT record TTFB + pts_before_nosc = _collect_metrics(reader) + ttfb_before_nosc = sum(p.count for p in pts_before_nosc["gcp.storage.client.ttfb"]) + + resp_no_sc = mock.Mock(status_code=None, headers={}) + session.request.return_value = resp_no_sc + _request_helpers.wait_and_retry( + lambda: _request_helpers.transport_request( + session, + "GET", + "https://storage.googleapis.com/download/storage/v1/b/b1/o/o1", + ), + None, + transport=session, + method="GET", + url="https://storage.googleapis.com/download/storage/v1/b/b1/o/o1", + streaming=False, + ) + + def _streaming_no_sc_invalid(): + r = _request_helpers.transport_request( + session, + "GET", + "https://storage.googleapis.com/download/storage/v1/b/b1/o/o1", + ) + _request_helpers.record_streaming_ttfb() + raise InvalidResponse(r, "missing status code") + + with pytest.raises(InvalidResponse): + _request_helpers.wait_and_retry( + _streaming_no_sc_invalid, + None, + transport=session, + method="GET", + url="https://storage.googleapis.com/download/storage/v1/b/b1/o/o1", + streaming=True, + ) + + pts_after_nosc = _collect_metrics(reader) + ttfb_after_nosc = sum(p.count for p in pts_after_nosc["gcp.storage.client.ttfb"]) + assert ttfb_after_nosc == ttfb_before_nosc + + +def test_media_downloads_ttfb_retries_and_416_zero_byte(metrics_env): + client, session, reader = metrics_env + media_url = "https://storage.googleapis.com/download/storage/v1/b/b1/o/o1?alt=media" + fast_retry = retry.Retry( + predicate=retry.if_exception_type( + InvalidResponse, requests.ConnectionError, requests.Timeout + ), + initial=0.001, + maximum=0.002, + deadline=5.0, + ) + + payload = b"hello world!" + crc_hdr = f"crc32c={_crc32c_b64(payload)}" + session.request.side_effect = [ + requests.ConnectTimeout("connect timed out"), + requests.ConnectionError("connection reset"), + _media_response(b"unavailable", status_code=503, streaming=True), + _media_response( + chunks=[b"hello ", b"world!"], + status_code=200, + streaming=True, + error_after=0, + ), + _media_response( + chunks=[b"hello ", b"world!"], + status_code=200, + streaming=True, + error_after=1, + ), + _media_response( + chunks=[b"hello ", b"world!"], + status_code=200, + headers={"x-goog-generation": "1", "x-goog-hash": crc_hdr}, + streaming=True, + ), + ] + out = io.BytesIO() + with _helpers._start_storage_operation( + client, "storage.Blob.download_to_file", "ReadObject" + ): + dl = Download(media_url, stream=out, checksum="crc32c") + dl._retry_strategy = fast_retry + dl.consume(session) + + assert out.getvalue().endswith(payload) + pts = _collect_metrics(reader) + assert sum(p.value for p in pts["gcp.storage.client.operations"]) == 1 + assert sum(p.value for p in pts["gcp.storage.client.attempts"]) == 6 + assert sum(p.value for p in pts["gcp.storage.client.retries"]) == 5 + assert sum(p.value for p in pts["gcp.storage.client.errors"]) == 5 + err_by_type = {} + for p in pts["gcp.storage.client.errors"]: + et = p.attributes["error.type"] + err_by_type[et] = err_by_type.get(et, 0) + p.value + assert err_by_type == { + "TIMEOUT": 3, + "CONNECTION_ERROR": 1, + "UNAVAILABLE": 1, + } + assert sum(p.count for p in pts["gcp.storage.client.ttfb"]) == 3 + for h_pt in pts["http.client.request.duration"]: + assert ( + h_pt.attributes["url.template"] + == "/download/storage/v1/b/{bucket}/o/{object}" + ) + + large_oneshot = b"x" * 9000 + delayed_dl_resp = _media_response(b"", status_code=200, streaming=True) + delayed_dl_resp.raw.read = mock.Mock(side_effect=[b"", b"delayed-gzip-dl"]) + session.request.side_effect = [ + _media_response(large_oneshot, status_code=200, streaming=True), + delayed_dl_resp, + _media_response(b"", status_code=200, streaming=True), + _media_response(chunks=[], status_code=200, streaming=True), + _media_response(b"nostream", status_code=200, streaming=False), + ] + dl_single_buf = io.BytesIO() + dl_single = Download(media_url, stream=dl_single_buf, single_shot_download=True) + dl_single.consume(session) + assert dl_single_buf.getvalue() == large_oneshot + dl_delayed_buf = io.BytesIO() + Download( + media_url, stream=dl_delayed_buf, single_shot_download=True, checksum=None + ).consume(session) + assert dl_delayed_buf.getvalue() == b"delayed-gzip-dl" + Download(media_url, stream=io.BytesIO(), single_shot_download=True).consume(session) + dl_empty = Download(media_url, stream=io.BytesIO(), single_shot_download=False) + dl_empty.consume(session) + dl_nostream = Download(media_url, stream=None) + dl_nostream.consume(session) + + plain_transport = mock.MagicMock(spec=["request"]) + plain_transport.request.side_effect = [ + _media_response(b"plain-dl", status_code=200, streaming=True), + _media_response(b"plain-raw", status_code=200, streaming=True), + ] + plain_dl_buf = io.BytesIO() + Download(media_url, stream=plain_dl_buf, single_shot_download=True).consume( + plain_transport + ) + assert plain_dl_buf.getvalue() == b"plain-dl" + plain_raw_buf = io.BytesIO() + RawDownload(media_url, stream=plain_raw_buf, single_shot_download=True).consume( + plain_transport + ) + assert plain_raw_buf.getvalue() == b"plain-raw" + + delayed_raw_resp = _media_response(b"", status_code=200, streaming=True) + delayed_raw_resp.raw.read = mock.Mock(side_effect=[b"", b"delayed-raw-body"]) + session.request.side_effect = [ + _media_response(large_oneshot, status_code=200, streaming=True), + delayed_raw_resp, + _media_response(b"", status_code=200, streaming=True), + _media_response(chunks=[b"raw-", b"multi"], status_code=200, streaming=True), + _media_response(chunks=[], status_code=200, streaming=True), + _media_response(chunks=[b"raw-nostream"], status_code=200, streaming=True), + ] + raw_single_buf = io.BytesIO() + RawDownload(media_url, stream=raw_single_buf, single_shot_download=True).consume( + session + ) + assert raw_single_buf.getvalue() == large_oneshot + raw_delayed_buf = io.BytesIO() + RawDownload( + media_url, stream=raw_delayed_buf, single_shot_download=True, checksum=None + ).consume(session) + assert raw_delayed_buf.getvalue() == b"delayed-raw-body" + RawDownload(media_url, stream=io.BytesIO(), single_shot_download=True).consume( + session + ) + RawDownload(media_url, stream=io.BytesIO(), single_shot_download=False).consume( + session + ) + RawDownload(media_url, stream=io.BytesIO(), single_shot_download=False).consume( + session + ) + RawDownload(media_url, stream=None).consume(session) + + err_cd_active = _media_response(b"err", status_code=500, streaming=True) + err_cd_active.close = mock.MagicMock(wraps=err_cd_active.close) + err_cd_plain = _media_response(b"err", status_code=500, streaming=False) + err_cd_plain.close = mock.MagicMock(wraps=err_cd_plain.close) + + session.request.side_effect = [ + _media_response( + chunks=[b"ab", b"cd"], + status_code=http.client.PARTIAL_CONTENT, + headers={"content-range": "bytes 0-3/4", "content-length": "4"}, + streaming=True, + ), + err_cd_active, + _media_response( + chunks=[b"ab", b"cd"], + status_code=http.client.PARTIAL_CONTENT, + headers={"content-range": "bytes 0-3/4", "content-length": "4"}, + streaming=True, + ), + _media_response( + b"", + status_code=http.client.REQUESTED_RANGE_NOT_SATISFIABLE, + headers={"content-range": "bytes */0"}, + streaming=True, + ), + ] + cd_buf = io.BytesIO() + cd = ChunkedDownload(media_url, chunk_size=4, stream=cd_buf) + cd.consume_next_chunk(session) + assert cd.finished + assert cd_buf.getvalue() == b"abcd" + + cd_fail_active = ChunkedDownload( + media_url, chunk_size=4, stream=io.BytesIO(), retry=None + ) + with pytest.raises(InvalidResponse): + cd_fail_active.consume_next_chunk(session) + err_cd_active.close.assert_called_once() + + plain_transport.request.side_effect = [err_cd_plain] + cd_fail_plain = ChunkedDownload( + media_url, chunk_size=4, stream=io.BytesIO(), retry=None + ) + with pytest.raises(InvalidResponse): + cd_fail_plain.consume_next_chunk(plain_transport) + err_cd_plain.close.assert_not_called() + + rcd = RawChunkedDownload(media_url, chunk_size=4, stream=io.BytesIO()) + rcd.consume_next_chunk(session) + assert rcd.finished + + bucket = client.bucket("b1") + zero_blob = bucket.blob("empty-obj", chunk_size=262144) + zero_buf = io.BytesIO() + zero_blob.download_to_file(zero_buf) + assert zero_buf.getvalue() == b"" + + pts_after = _collect_metrics(reader) + dl_ops = [ + p + for p in pts_after["gcp.storage.client.operations"] + if p.attributes["gcp.client.method"] == "storage.Blob.download_to_file" + ] + assert sum(p.value for p in dl_ops) == 2 + assert all(p.attributes["error.type"] == "OK" for p in dl_ops) + body_pts = pts_after["gcp.storage.client.response.body.size"] + assert any(p.sum == 0 for p in body_pts) + + +def test_all_media_upload_classes_and_xml_mpu(metrics_env, tmp_path): + client, session, reader = metrics_env + upload_url = "https://storage.googleapis.com/upload/storage/v1/b/b1/o?uploadType=media&name=o1" + mp_url = ( + "https://storage.googleapis.com/upload/storage/v1/b/b1/o?uploadType=multipart" + ) + res_url = ( + "https://storage.googleapis.com/upload/storage/v1/b/b1/o?uploadType=resumable" + ) + session_uri = "https://storage.googleapis.com/upload/storage/v1/b/b1/o?uploadType=resumable&upload_id=abc" + xml_url = "https://storage.googleapis.com/b1/o1" + + session.request.side_effect = [ + _json_response({"name": "o1", "size": "3"}), + _json_response({"name": "o1", "size": "5"}), + ] + su = SimpleUpload(upload_url) + su.transmit(session, b"abc", "text/plain") + mu = MultipartUpload(mp_url) + mu.transmit(session, b"hello", {"name": "o1"}, "text/plain") + + chunk_size = 256 * 1024 + ru_bytes = b"x" * chunk_size + b"y" * 10 + stream = io.BytesIO(ru_bytes) + ru = ResumableUpload(res_url, chunk_size) + session.request.side_effect = [ + _media_response( + b"", + status_code=http.client.OK, + headers={"location": session_uri}, + ), + _media_response( + b"", + status_code=http.client.PERMANENT_REDIRECT, + headers={"range": f"bytes=0-{chunk_size - 1}"}, + ), + _media_response( + b"", + status_code=http.client.PERMANENT_REDIRECT, + headers={"range": f"bytes=0-{chunk_size - 1}"}, + ), + _json_response( + { + "name": "o1", + "size": str(chunk_size + 10), + "crc32c": _crc32c_b64(ru_bytes), + } + ), + ] + ru.initiate(session, stream, {"name": "o1"}, "application/octet-stream") + ru.transmit_next_chunk(session) + ru._invalid = True + ru.recover(session) + assert not ru.invalid + ru.transmit_next_chunk(session) + assert ru.finished + + sample_file = tmp_path / "mpu.bin" + sample_file.write_bytes(b"0123456789") + xml_ns = 'xmlns="http://s3.amazonaws.com/doc/2006-03-01/"' + session.request.side_effect = [ + _media_response( + f"u-123".encode( + "utf-8" + ), + status_code=200, + ), + _media_response( + b"", + status_code=200, + headers={ + "ETag": '"etag-1"', + "x-goog-hash": f"crc32c={_crc32c_b64(b'0123456789')}", + }, + ), + _media_response( + f'"final"'.encode( + "utf-8" + ), + status_code=200, + ), + _media_response(b"", status_code=204), + ] + container = XMLMPUContainer(xml_url, str(sample_file)) + container.initiate(session, content_type="application/octet-stream") + assert container.upload_id == "u-123" + part = XMLMPUPart( + xml_url, + "u-123", + str(sample_file), + start=0, + end=10, + part_number=1, + checksum="crc32c", + ) + part.upload(session) + container.register_part(1, part.etag) + container.finalize(session) + container.cancel(session) + + pts = _collect_metrics(reader) + templates = { + p.attributes["url.template"] for p in pts["http.client.request.duration"] + } + assert "/upload/storage/v1/b/{bucket}/o" in templates + assert "/{bucket}/{object}" in templates + assert sum(p.value for p in pts["gcp.storage.client.attempts"]) == 10 + + +def test_blob_media_methods_body_size_and_corruption_cleanup(metrics_env, tmp_path): + client, session, reader = metrics_env + bucket = client.bucket("b1") + blob = bucket.blob("o1") + payload = b"storage-payload" + crc_hdr = f"crc32c={_crc32c_b64(payload)}" + + dl_file = tmp_path / "downloaded.bin" + tm_dl_file = tmp_path / "tm_downloaded.bin" + blob._properties["updated"] = "2026-01-02T03:04:05.000Z" + session.request.side_effect = [ + _media_response( + payload, + status_code=200, + headers={"x-goog-generation": "7", "x-goog-hash": crc_hdr}, + streaming=True, + ), + _media_response( + payload, + status_code=200, + headers={"x-goog-generation": "7", "x-goog-hash": crc_hdr}, + streaming=True, + ), + _media_response( + payload, + status_code=200, + headers={"x-goog-generation": "7", "x-goog-hash": crc_hdr}, + streaming=True, + ), + _media_response( + payload, + status_code=200, + headers={"x-goog-generation": "7", "x-goog-hash": crc_hdr}, + streaming=True, + ), + _media_response( + payload, + status_code=200, + headers={"x-goog-generation": "7", "x-goog-hash": crc_hdr}, + streaming=True, + ), + _media_response( + payload, + status_code=200, + headers={"x-goog-generation": "7", "x-goog-hash": crc_hdr}, + streaming=True, + ), + ] + blob.download_to_file( + io.BytesIO(), + if_generation_match=7, + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + ) + assert blob.download_as_bytes() == payload + with pytest.warns(PendingDeprecationWarning): + assert blob.download_as_string() == payload + blob.download_to_filename(str(dl_file)) + assert dl_file.read_bytes() == payload + assert isinstance(blob.updated, datetime.datetime) + # Exercise internal transfer_manager default_method branches on Blob helpers + blob._handle_filename_and_download(str(tm_dl_file), command="tm.download_many") + blob._prep_and_do_download(io.BytesIO(), command="tm.download_many") + + missing_dest = tmp_path / "missing.bin" + corrupt_dest = tmp_path / "corrupt.bin" + session.request.side_effect = [ + _json_response({"error": {"message": "not found"}}, status_code=404), + _media_response( + b"tampered", + status_code=200, + headers={"x-goog-generation": "7", "x-goog-hash": crc_hdr}, + streaming=True, + ), + ] + with pytest.raises(NotFound): + blob.download_to_filename(str(missing_dest), retry=None) + assert not missing_dest.exists() + + with pytest.raises(DataCorruption): + blob.download_to_filename(str(corrupt_dest), retry=None) + assert not corrupt_dest.exists() + + up_file = tmp_path / "upload.txt" + up_file.write_bytes(b"from-disk") + session.request.side_effect = [ + _json_response({"name": "o1", "size": "9"}), + _json_response({"name": "o1", "size": "9"}), + _json_response({"name": "o1", "size": "9"}), + _json_response({"name": "o1", "size": "9"}), + _json_response({"error": {"message": "forbidden"}}, status_code=403), + ] + blob.upload_from_string(b"123456789") + blob.upload_from_filename(str(up_file)) + blob._handle_filename_and_upload(str(up_file), command="tm.upload_many") + blob._prep_and_do_upload(io.BytesIO(b"123456789"), size=9, command="tm.upload_many") + with pytest.raises(Forbidden): + blob.upload_from_file(io.BytesIO(b"denied"), size=6, retry=None) + + res_blob = bucket.blob("resumable-obj", chunk_size=256 * 1024) + res_data = b"a" * (256 * 1024) + b"b" * 100 + session_uri = "https://storage.googleapis.com/upload/storage/v1/b/b1/o?uploadType=resumable&upload_id=r1" + session.request.side_effect = [ + _media_response(b"", status_code=200, headers={"location": session_uri}), + _media_response( + b"", + status_code=http.client.PERMANENT_REDIRECT, + headers={"range": f"bytes=0-{256 * 1024 - 1}"}, + ), + _json_response( + { + "name": "resumable-obj", + "size": str(len(res_data)), + "crc32c": _crc32c_b64(res_data), + } + ), + ] + res_blob.upload_from_file(io.BytesIO(res_data)) + + # Resumable DataCorruption triggers self.delete() which MUST be suppressed! + session.request.side_effect = [ + _media_response(b"", status_code=200, headers={"location": session_uri}), + _json_response(None, status_code=204), # suppressed self.delete() + ] + with mock.patch.object( + ResumableUpload, + "transmit_next_chunk", + side_effect=DataCorruption(mock.Mock(), "hash mismatch"), + ): + with pytest.raises(DataCorruption): + res_blob.upload_from_file(io.BytesIO(b"corrupt"), retry=None) + + disabled_session = mock.MagicMock() + disabled_client = Client( + project="test-project", + credentials=_make_credentials(), + _http=disabled_session, + enable_metrics=False, + ) + disabled_client._bucket_metadata_cache = None + d_blob = disabled_client.bucket("b1").blob("o1") + disabled_session.request.side_effect = [ + _media_response(b"ok", status_code=200, streaming=True), + _json_response({"name": "o1", "size": "2"}), + _media_response(b"", status_code=200, headers={"location": session_uri}), + _json_response({"name": "o1", "size": "2", "crc32c": _crc32c_b64(b"ok")}), + ] + d_blob.download_to_file(io.BytesIO()) + d_blob.upload_from_string(b"ok") + d_res_blob = disabled_client.bucket("b1").blob("o2", chunk_size=256 * 1024) + d_res_blob.upload_from_file(io.BytesIO(b"ok")) + disabled_client.close() + + with mock.patch.object( + Download, + "consume", + autospec=True, + side_effect=lambda self_dl, *a, **kw: ( + setattr(self_dl, "_bytes_downloaded", True) or _media_response(b"") + ), + ): + blob._do_download( + session, + io.BytesIO(), + "https://storage.googleapis.com/download/storage/v1/b/b1/o/o1?alt=media", + {}, + ) + mock_up = mock.MagicMock() + mock_up.finished = False + mock_up.resumable_url = session_uri + + def _finish_bool(*a, **kw): + mock_up.finished = True + mock_up.bytes_uploaded = True + return _json_response({"name": "o1"}) + + mock_up.transmit_next_chunk.side_effect = _finish_bool + with mock.patch.object( + res_blob, + "_initiate_resumable_upload", + return_value=(mock_up, session), + ): + res_blob.upload_from_file(io.BytesIO(b"x")) + + with _helpers._suppress_body_size(): + session.request.side_effect = [ + _media_response( + b"sup", + status_code=200, + headers={ + "x-goog-generation": "7", + "x-goog-hash": f"crc32c={_crc32c_b64(b'sup')}", + }, + streaming=True, + ), + _json_response({"name": "o1", "size": "3"}), + _media_response(b"", status_code=200, headers={"location": session_uri}), + _json_response({"name": "o1", "size": "3", "crc32c": _crc32c_b64(b"sup")}), + ] + blob.download_to_file(io.BytesIO()) + blob.upload_from_string(b"sup") + res_blob.upload_from_file(io.BytesIO(b"sup")) + + pts = _collect_metrics(reader) + ops = pts["gcp.storage.client.operations"] + assert not any(p.attributes["gcp.storage.operation"] == "DeleteObject" for p in ops) + op_err_pairs = { + (p.attributes["gcp.storage.operation"], p.attributes["error.type"]) for p in ops + } + assert ("ReadObject", "NOT_FOUND") in op_err_pairs + assert ("ReadObject", "CHECKSUM_MISMATCH") in op_err_pairs + assert ("WriteObject", "CHECKSUM_MISMATCH") in op_err_pairs + resp_pts = pts["gcp.storage.client.response.body.size"] + assert sum(p.sum for p in resp_pts) == 6 * len(payload) + assert sum(p.count for p in resp_pts) == 6 + req_pts = pts["gcp.storage.client.request.body.size"] + assert sum(p.sum for p in req_pts) == len(res_data) + 4 * 9 + assert sum(p.count for p in req_pts) == 5 + + +def test_primary_exception_pinning_and_resumable_cleanup_chaining(metrics_env): + client, session, reader = metrics_env + bucket = client.bucket("r1-bucket") + session_uri = "https://storage.googleapis.com/upload/storage/v1/b/r1-bucket/o?uploadType=resumable&upload_id=r1" + + _helpers._record_primary_exception( + RuntimeError("outside-op"), RuntimeError("outside-cleanup") + ) + c1 = RuntimeError("cleanup-1") + c2 = RuntimeError("cleanup-2") + c2.__context__ = c1 + with pytest.raises(RuntimeError, match="cleanup-2"): + with _helpers._start_storage_operation(client, "outer.chained", "WriteObject"): + _helpers._record_primary_exception(ValueError("first-pin"), c1) + _helpers._record_primary_exception(KeyError("second-ignored"), c2) + with pytest.raises(RuntimeError, match="nested-boom"): + with _helpers._start_storage_operation( + client, "inner.method", "DeleteObject" + ): + raise RuntimeError("nested-boom") + raise c2 + + with pytest.raises(ValueError, match="unrelated-after-caught"): + with _helpers._start_storage_operation(client, "outer.caught", "WriteObject"): + _helpers._record_primary_exception( + DataCorruption(mock.Mock(), "stale"), RuntimeError("caught-cleanup") + ) + raise ValueError("unrelated-after-caught") + + c_second = RuntimeError("independent-cleanup") + with pytest.raises(RuntimeError, match="independent-cleanup"): + with _helpers._start_storage_operation(client, "outer.repinned", "WriteObject"): + _helpers._record_primary_exception( + ValueError("caught-pin"), RuntimeError("caught-cleanup") + ) + _helpers._record_primary_exception(KeyError("fresh-pin"), c_second) + raise c_second + + res_blob = bucket.blob("corrupt-cleanup-err", chunk_size=256 * 1024) + session.request.side_effect = [ + _media_response(b"", status_code=200, headers={"location": session_uri}), + ] + corrupt_exc = DataCorruption(mock.Mock(), "bad crc") + mid_conn_exc = requests.ConnectionError("delete retry transport failure") + top_cleanup_exc = Forbidden("delete denied after retry") + + def _chained_delete(*args, **kwargs): + try: + raise mid_conn_exc + except requests.ConnectionError: + raise top_cleanup_exc + + with ( + mock.patch.object( + ResumableUpload, + "transmit_next_chunk", + side_effect=corrupt_exc, + ), + mock.patch.object(res_blob, "delete", side_effect=_chained_delete), + ): + with pytest.raises(Forbidden) as exc_info: + res_blob.upload_from_file(io.BytesIO(b"bad"), retry=None) + assert exc_info.value is top_cleanup_exc + assert exc_info.value.__context__ is mid_conn_exc + assert exc_info.value.__context__.__context__ is corrupt_exc + + pts = _collect_metrics(reader) + by_method_err = { + ( + p.attributes["gcp.client.method"], + p.attributes["gcp.storage.operation"], + p.attributes["error.type"], + ): p.value + for p in pts["gcp.storage.client.operations"] + } + assert ( + "outer.chained", + "WriteObject", + "CLIENT_VALIDATION_ERROR", + ) in by_method_err + assert ( + "outer.caught", + "WriteObject", + "CLIENT_VALIDATION_ERROR", + ) in by_method_err + assert ( + "outer.repinned", + "WriteObject", + "UNKNOWN", + ) in by_method_err + assert ( + "storage.Blob.upload_from_file", + "WriteObject", + "CHECKSUM_MISMATCH", + ) in by_method_err diff --git a/packages/google-cloud-storage/tests/unit/test_otel_http_transport_metrics.py b/packages/google-cloud-storage/tests/unit/test_otel_http_transport_metrics.py new file mode 100644 index 000000000000..a93242efdecc --- /dev/null +++ b/packages/google-cloud-storage/tests/unit/test_otel_http_transport_metrics.py @@ -0,0 +1,535 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for OpenTelemetry HTTP JSON transport, Client lifecycle, and core helpers.""" + +import functools +import http.client +import json +from unittest import mock + +import pytest +import requests +from google.api_core import retry +from google.auth import credentials as auth_credentials +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace import TracerProvider + +from google.cloud.storage import _helpers, _opentelemetry_metrics +from google.cloud.storage._bucket_metadata_cache import BucketMetadataCache +from google.cloud.storage.client import Client + + +def _make_credentials(): + return auth_credentials.AnonymousCredentials() + + +def _json_response(payload, status_code=http.client.OK): + response = requests.Response() + response.status_code = status_code + response.headers.update({"Content-Type": "application/json"}) + response._content = json.dumps(payload).encode("utf-8") + response.request = requests.Request( + "GET", "https://storage.googleapis.com/storage/v1/b" + ).prepare() + return response + + +def _collect_metrics(reader): + data = reader.get_metrics_data() + collected = {} + if data is None: # pragma: NO COVER + return collected + for resource_metric in data.resource_metrics: + for scope_metric in resource_metric.scope_metrics: + for metric in scope_metric.metrics: + collected.setdefault(metric.name, []).extend(metric.data.data_points) + return collected + + +@pytest.fixture(autouse=True) +def _enable_dev_gate(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + + +@pytest.fixture +def metrics_env(): + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + session = mock.MagicMock() + client = Client( + project="test-project", + credentials=_make_credentials(), + _http=session, + enable_metrics=True, + meter_provider=provider, + ) + # Disable async daemon cache fetches from consuming session.request.side_effect + client._bucket_metadata_cache = None + yield client, session, reader + client.close() + + +def test_client_metrics_disabled_by_default_and_fast_paths(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", False) + session = mock.MagicMock() + session.request.return_value = _json_response({"items": []}) + client = Client( + project="test-project", + credentials=_make_credentials(), + _http=session, + enable_metrics=True, + ) + assert client._metrics_recorder.enabled is False + with _helpers._list_metrics_context("storage.Client.list_buckets", "ListBuckets"): + iterator = client._list_resource("/b", lambda it, item: item) + assert isinstance(iterator.api_request, functools.partial) + assert list(iterator) == [] + resp = client._connection._make_request("GET", "/b") + assert resp.status_code == http.client.OK + client.close() + + +def test_client_custom_endpoint_and_owned_shutdown(): + session = mock.MagicMock() + session.request.return_value = _json_response({"email_address": "sa@example.com"}) + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + client = Client( + project="test-project", + credentials=_make_credentials(), + _http=session, + client_options={"api_endpoint": "https://custom.storage.example.com:8443"}, + enable_metrics=True, + meter_provider=provider, + ) + with _helpers._start_storage_operation( + client, + "storage.Client.get_service_account_email", + "GetServiceAccount", + ): + resp = client._get_resource("/projects/test-project/serviceAccount") + assert resp["email_address"] == "sa@example.com" + + pts = _collect_metrics(reader) + op_pt = pts["gcp.storage.client.operations"][0] + dur_pt = pts["gcp.client.request.duration"][0] + assert op_pt.attributes["server.address"] == "custom.storage.example.com" + assert dur_pt.attributes["server.address"] == "custom.storage.example.com" + assert op_pt.attributes["rpc.system.name"] == "http" + + att_pt = pts["gcp.storage.client.attempts"][0] + http_dur_pt = pts["http.client.request.duration"][0] + assert att_pt.attributes["server.address"] == "custom.storage.example.com" + assert http_dur_pt.attributes["server.address"] == "custom.storage.example.com" + assert att_pt.attributes["network.path"] == "cloudpath" + + with client: + pass + + with mock.patch.object(client._metrics_recorder, "shutdown") as mock_shutdown: + client.close() + mock_shutdown.assert_called_once() + + # Immutable _http and missing _metrics_recorder on close() are safe no-ops + ro_client = Client( + project="test-project", + credentials=_make_credentials(), + _http=object(), + enable_metrics=True, + ) + assert ro_client._http is ro_client._http_internal + assert ro_client.metrics_enabled is True + assert ro_client.debug_metrics_enabled is False + ro_client._metrics_recorder.shutdown() + ro_client._metrics_recorder = None + assert ro_client.metrics_enabled is False + assert ro_client.debug_metrics_enabled is False + assert ro_client._http is ro_client._http_internal + ro_client.close() + + # Lazy _http session creation attaches _gcs_metrics_recorder on first access + lazy_client = Client( + project="test-project", + credentials=_make_credentials(), + _http=None, + enable_metrics=True, + meter_provider=provider, + ) + lazy_client._bucket_metadata_cache = None + lazy_session = mock.MagicMock() + with mock.patch( + "google.cloud.client.ClientWithProject._http", + new_callable=mock.PropertyMock, + return_value=lazy_session, + ): + assert lazy_client._http is lazy_session + assert lazy_session._gcs_metrics_recorder is lazy_client._metrics_recorder + lazy_client.close() + + +def test_list_resource_metrics_context_and_unannotated_passthrough(metrics_env): + client, session, reader = metrics_env + + # Unannotated _list_resource leaves raw functools.partial untouched + raw_iter = client._list_resource("/b", lambda it, item: item) + assert isinstance(raw_iter.api_request, functools.partial) + + # Annotated _list_resource wraps api_request and emits one operation per page + with _helpers._list_metrics_context("storage.Client.list_buckets", "ListBuckets"): + paged_iter = client._list_resource("/b", lambda it, item: item["name"]) + assert not isinstance(paged_iter.api_request, functools.partial) + assert _collect_metrics(reader) == {} + + session.request.side_effect = [ + _json_response({"items": [{"name": "b1"}], "nextPageToken": "p2"}), + _json_response({"items": [{"name": "b2"}]}), + ] + assert list(paged_iter) == ["b1", "b2"] + pts = _collect_metrics(reader) + ops = pts["gcp.storage.client.operations"] + assert sum(p.value for p in ops) == 2 + assert ops[0].attributes["gcp.client.method"] == "storage.Client.list_buckets" + assert ops[0].attributes["gcp.storage.operation"] == "ListBuckets" + + +def test_helpers_context_managers_batch_deferral_and_property_mixin(metrics_env): + client, session, reader = metrics_env + + class _BrokenClientObj: + _client = "fallback-client" + + @property + def client(self): + raise RuntimeError("unbound") + + broken = _BrokenClientObj() + assert _helpers._resolve_operation_client(broken, client="explicit") == "explicit" + assert _helpers._resolve_operation_client(broken) == "fallback-client" + + with _helpers._suppress_body_size(): + assert _helpers._SUPPRESS_BODY_SIZE.get() is True + assert _helpers._SUPPRESS_BODY_SIZE.get() is False + + with _helpers._list_metrics_context("outer.method", "ListObjects"): + with _helpers._list_metrics_context("inner.method", "ListBuckets"): + assert _helpers._LIST_METRICS_CONTEXT.get() == ( + "outer.method", + "ListObjects", + ) + assert _helpers._LIST_METRICS_CONTEXT.get() is None + + with _helpers._client_method_override("outer.override"): + with _helpers._client_method_override("inner.override"): + assert _helpers._CLIENT_METHOD_OVERRIDE.get() == "outer.override" + assert _helpers._CLIENT_METHOD_OVERRIDE.get() is None + + # Disabled client yields None in _start_storage_operation + disabled_client = Client( + project="test-project", + credentials=_make_credentials(), + _http=mock.MagicMock(), + enable_metrics=False, + ) + disabled_client._bucket_metadata_cache = None + with _helpers._start_storage_operation( + disabled_client, "storage.Client.get_bucket", "GetBucket" + ) as st_disabled: + assert st_disabled is None + disabled_client.close() + + # Batch deferral in Connection.api_request and _start_storage_operation skips + # queued/batchable ops while keeping non-batchable ops + batch = client.batch() + batch.__enter__() + try: + bkt = client.bucket("b1") + bkt.storage_class = "NEARLINE" + bkt.patch() + assert _collect_metrics(reader) == {} + + with _helpers._start_storage_operation( + client, "storage.Bucket.patch", "UpdateBucket" + ) as st_deferred: + assert st_deferred is None + with _helpers._client_method_override("storage.Blob.download_as_bytes"): + with _helpers._start_storage_operation( + client, "storage.Blob.download_to_file", "ReadObject" + ) as st_non_batch: + assert st_non_batch is not None + finally: + client._pop_batch() + + # Custom _PropertyMixin without _METRICS_GET_OP / _METRICS_UPDATE_OP executes uninstrumented branch + class _CustomResource(_helpers._PropertyMixin): + path = "/custom/1" + user_project = None + + def __init__(self, cl): + super().__init__(name="c1") + self._client = cl + + @property + def client(self): + return self._client + + mock_client = mock.MagicMock() + mock_client._get_resource.return_value = {"k": "v"} + mock_client._patch_resource.return_value = {"k": "v2"} + mock_client._put_resource.return_value = {"k": "v3"} + res = _CustomResource(mock_client) + res.reload() + res.patch() + res.update() + assert res._properties == {"k": "v3"} + + # Instrumented _PropertyMixin records reload, patch, and update with canonical class name + class _InstrumentedResource(_helpers._PropertyMixin): + _METRICS_CLASS_NAME = "Bucket" + _METRICS_GET_OP = "GetBucket" + _METRICS_UPDATE_OP = "UpdateBucket" + path = "/b/my-bucket" + user_project = None + + def __init__(self, cl): + super().__init__(name="my-bucket") + self._client = cl + + @property + def client(self): + return self._client + + session.request.side_effect = [ + _json_response({"name": "my-bucket"}), + _json_response({"name": "my-bucket"}), + _json_response({"name": "my-bucket"}), + ] + inst_res = _InstrumentedResource(client) + inst_res.reload() + inst_res.patch() + inst_res.update() + + pts = _collect_metrics(reader) + methods_seen = { + p.attributes["gcp.client.method"] for p in pts["gcp.storage.client.operations"] + } + assert methods_seen == { + "storage.Blob.download_as_bytes", + "storage.Bucket.reload", + "storage.Bucket.patch", + "storage.Bucket.update", + } + + +def test_http_connection_direct_make_request_and_edge_branches(metrics_env): + client, session, reader = metrics_env + conn = client._connection + + # 1. Suppressed internal metrics bypasses recording + session.request.return_value = _json_response({"ok": True}) + with _helpers._suppress_internal_metrics(): + with _helpers._start_storage_operation( + client, "storage.Client.get_bucket", "GetBucket" + ) as st: + assert st is None + conn._make_request("GET", "https://storage.googleapis.com/storage/v1/b/b1") + conn.api_request(method="GET", path="/b/b1") + assert _collect_metrics(reader) == {} + + # 2. Direct _make_request network exception -> CONNECTION_ERROR, no TTFB, no status_code + session.request.side_effect = requests.ConnectionError("reset by peer") + with _helpers._start_storage_operation(client, "storage.Batch.finish", "Batch"): + with pytest.raises(requests.ConnectionError): + conn._make_request( + "POST", "https://storage.googleapis.com/batch/storage/v1" + ) + pts = _collect_metrics(reader) + assert "gcp.storage.client.ttfb" not in pts + att = pts["gcp.storage.client.attempts"][-1] + assert att.attributes["error.type"] == "CONNECTION_ERROR" + assert "http.response.status_code" not in att.attributes + + # 3. Direct _make_request non-2xx with normal response AND broken response object + session.request.side_effect = None + session.request.return_value = _json_response( + {"error": {"message": "unavailable"}}, status_code=503 + ) + resp_503 = conn._make_request( + "POST", "https://storage.googleapis.com/batch/storage/v1" + ) + assert resp_503.status_code == 503 + + broken_resp = mock.Mock() + broken_resp.status_code = 500 + session.request.return_value = broken_resp + conn._make_request("POST", "https://storage.googleapis.com/batch/storage/v1") + + bool_sc_resp = mock.Mock() + bool_sc_resp.status_code = True + session.request.return_value = bool_sc_resp + conn._make_request("POST", "https://storage.googleapis.com/batch/storage/v1") + + pts2 = _collect_metrics(reader) + statuses = [ + p.attributes.get("http.response.status_code") + for p in pts2["http.client.request.duration"] + ] + assert 503 in statuses + assert 500 in statuses + assert statuses[-1] is None + assert sum(p.count for p in pts2["gcp.storage.client.ttfb"]) == 2 + + # 4. api_request fallback branches when base_call bypasses _make_request or url is empty + with mock.patch("google.cloud._http.JSONConnection.api_request", return_value={}): + conn.api_request("GET", "/b/my-bucket") + weird_resp = _json_response({"ok": True}) + weird_resp.status_code = True # bool is excluded from int status_code + with mock.patch.object( + _helpers._opentelemetry_metrics, + "normalize_url_template", + wraps=_opentelemetry_metrics.normalize_url_template, + ): + session.request.return_value = _json_response({"ok": True}) + with mock.patch( + "google.cloud._http.JSONConnection._make_request", + return_value=weird_resp, + ): + with mock.patch( + "google.cloud._http.JSONConnection.api_request", + side_effect=lambda *a, **kw: (conn._make_request("GET", "") and {}), + ): + conn.api_request(method="GET", path="/b/my-bucket") + + with mock.patch( + "google.cloud._http.JSONConnection._make_request", + return_value=weird_resp, + ): + with mock.patch( + "google.cloud._http.JSONConnection.api_request", + side_effect=lambda *a, **kw: ( + conn._make_request("GET", "") + and (_ for _ in ()).throw(RuntimeError("boom")) + ), + ): + with pytest.raises(RuntimeError): + conn.api_request(method="GET", path="/b/my-bucket") + + +def test_retries_errors_and_ttfb_on_api_request(metrics_env): + client, session, reader = metrics_env + session.request.side_effect = [ + _json_response({"error": {"message": "503-1"}}, status_code=503), + requests.ConnectionError("transient reset"), + _json_response({"id": "my-bucket", "name": "my-bucket"}, status_code=200), + ] + fast_retry = retry.Retry(initial=0.001, maximum=0.002, deadline=5.0) + with _helpers._start_storage_operation( + client, "storage.Client.get_bucket", "GetBucket" + ): + resp = client._connection.api_request( + method="GET", path="/b/my-bucket", retry=fast_retry + ) + assert resp["name"] == "my-bucket" + + pts = _collect_metrics(reader) + assert len(pts["gcp.storage.client.operations"]) == 1 + op = pts["gcp.storage.client.operations"][0] + dur = pts["gcp.client.request.duration"][0] + assert op.attributes["gcp.storage.operation"] == "GetBucket" + assert op.attributes["gcp.client.method"] == "storage.Client.get_bucket" + assert op.attributes["error.type"] == "OK" + assert dur.attributes["rpc.method"] == "google.storage.v2.Storage/GetBucket" + assert dur.attributes["error.type"] == "OK" + + attempts = pts["gcp.storage.client.attempts"] + assert sum(p.value for p in attempts) == 3 + err_types = {p.attributes.get("error.type", "OK") for p in attempts} + assert err_types == {"UNAVAILABLE", "CONNECTION_ERROR", "OK"} + + retries_pts = pts["gcp.storage.client.retries"] + assert sum(p.value for p in retries_pts) == 2 + retry_errs = {p.attributes["error.type"] for p in retries_pts} + assert retry_errs == {"UNAVAILABLE", "CONNECTION_ERROR"} + + errors_pts = pts["gcp.storage.client.errors"] + assert sum(p.value for p in errors_pts) == 2 + + # TTFB recorded only for the 503 response and the 200 response (NOT ConnectionError) + ttfb_pts = pts["gcp.storage.client.ttfb"] + assert sum(p.count for p in ttfb_pts) == 2 + + # Terminal HTTP 400 BadRequest maps to INVALID_ARGUMENT + session.request.side_effect = [ + _json_response( + {"error": {"message": "Invalid bucket configuration"}}, + status_code=400, + ) + ] + with pytest.raises(Exception): + with _helpers._start_storage_operation( + client, "storage.Bucket.patch", "UpdateBucket" + ): + client._connection.api_request( + method="PATCH", path="/b/my-bucket", retry=None + ) + pts_after = _collect_metrics(reader) + patch_ops = [ + p + for p in pts_after["gcp.storage.client.operations"] + if p.attributes["gcp.client.method"] == "storage.Bucket.patch" + ] + assert len(patch_ops) == 1 + assert patch_ops[0].attributes["error.type"] == "INVALID_ARGUMENT" + + +def test_bucket_metadata_cache_background_suppression(metrics_env): + client, session, reader = metrics_env + cache = BucketMetadataCache(client) + + session.request.side_effect = [ + _json_response({"id": "b1", "name": "b1", "location": "US"}), + _json_response({"error": {"message": "404"}}, status_code=404), + ] + cache._fetch_background("b1") + cache._verify_existence_background("b1") + + assert _collect_metrics(reader) == {} + + +def test_trace_exemplar_correlation_on_http_json_metrics(metrics_env): + client, session, reader = metrics_env + session.request.return_value = _json_response({"name": "my-bucket"}) + + tracer_provider = TracerProvider() + tracer = tracer_provider.get_tracer("test-tracer") + with tracer.start_as_current_span("parent-span") as span: + expected_trace_id = span.get_span_context().trace_id + expected_span_id = span.get_span_context().span_id + with _helpers._start_storage_operation( + client, "storage.Client.get_bucket", "GetBucket" + ): + client._connection.api_request(method="GET", path="/b/my-bucket") + + pts = _collect_metrics(reader) + op_exemplars = pts["gcp.client.request.duration"][0].exemplars + http_exemplars = pts["http.client.request.duration"][0].exemplars + assert len(op_exemplars) >= 1 + assert len(http_exemplars) >= 1 + assert op_exemplars[0].trace_id == expected_trace_id + assert op_exemplars[0].span_id != 0 + assert http_exemplars[0].trace_id == expected_trace_id + assert http_exemplars[0].span_id != 0 + assert expected_span_id != 0 diff --git a/packages/google-cloud-storage/tests/unit/test_otel_transfer_manager_metrics.py b/packages/google-cloud-storage/tests/unit/test_otel_transfer_manager_metrics.py new file mode 100644 index 000000000000..46e300f5daa3 --- /dev/null +++ b/packages/google-cloud-storage/tests/unit/test_otel_transfer_manager_metrics.py @@ -0,0 +1,852 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for OpenTelemetry transfer_manager concurrent and multiprocessing metrics.""" + +import base64 +import concurrent.futures +import http.client +import io +import json +from unittest import mock + +import google_crc32c +import pytest +import requests +from google.auth import credentials as auth_credentials +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader + +from google.cloud.storage import ( + _helpers, + _opentelemetry_metrics, + transfer_manager, +) +from google.cloud.storage._media.requests.upload import XMLMPUContainer, XMLMPUPart +from google.cloud.storage.client import Client +from google.cloud.storage.exceptions import DataCorruption, InvalidResponse + + +def _make_credentials(): + return auth_credentials.AnonymousCredentials() + + +def _crc32c_b64(data: bytes) -> str: + checksum = google_crc32c.Checksum(data) + return base64.b64encode(checksum.digest()).decode("ascii") + + +class _FakeRawStream: + """Minimal urllib3-like raw response backing streaming chunk reads.""" + + def __init__(self, chunks): + self._chunks = list(chunks) + self.headers = {} + + def stream(self, amt=8192, decode_content=None): + for chunk in self._chunks: + yield chunk + + +def _media_response( + body=b"", + status_code=http.client.OK, + headers=None, + streaming=False, +): + response = requests.Response() + response.status_code = status_code + response.headers.update(headers or {}) + chunk_list = [body] if body else [] + response.raw = _FakeRawStream(chunk_list) + if streaming: + response._content = False + else: + response._content = body + response.request = requests.Request( + "GET", "https://storage.googleapis.com/download/storage/v1/b/b1/o/o1" + ).prepare() + return response + + +def _json_response(payload=None, status_code=http.client.OK): + hdrs = {"Content-Type": "application/json"} + body = b"" if payload is None else json.dumps(payload).encode("utf-8") + return _media_response(body=body, status_code=status_code, headers=hdrs) + + +def _collect_metrics(reader): + data = reader.get_metrics_data() + collected = {} + if data is None: # pragma: NO COVER + return collected + for resource_metric in data.resource_metrics: + for scope_metric in resource_metric.scope_metrics: + for metric in scope_metric.metrics: + collected.setdefault(metric.name, []).extend(metric.data.data_points) + return collected + + +@pytest.fixture(autouse=True) +def _enable_dev_gate(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + + +@pytest.fixture +def metrics_env(): + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + session = mock.MagicMock() + client = Client( + project="test-project", + credentials=_make_credentials(), + _http=session, + enable_metrics=True, + meter_provider=provider, + ) + client._bucket_metadata_cache = None + yield client, session, reader + client.close() + + +def test_transfer_manager_many_operations_thread_and_process(metrics_env, tmp_path): + client, session, reader = metrics_env + bucket = client.bucket("tm-bucket") + + f1 = tmp_path / "u1.txt" + f1.write_bytes(b"file-1") + f2 = tmp_path / "u2.txt" + f2.write_bytes(b"file-2") + + # 1. upload_many (THREAD) with filename (multipart) + BytesIO (resumable) + tm_res_uri = "https://storage.googleapis.com/upload/storage/v1/b/tm-bucket/o?uploadType=resumable&upload_id=tm1" + + def _tm_upload_transport(method, url, **kw): + if method == "POST" and "uploadType=resumable" in url: + return _media_response( + b"", status_code=200, headers={"location": tm_res_uri} + ) + if method == "PUT": + return _json_response( + {"name": "b2", "size": "5", "crc32c": _crc32c_b64(b"mem-2")} + ) + return _json_response({"name": "b1", "size": "6"}) + + session.request.side_effect = _tm_upload_transport + res_up = transfer_manager.upload_many( + [(str(f1), bucket.blob("b1")), (io.BytesIO(b"mem-2"), bucket.blob("b2"))], + worker_type=transfer_manager.THREAD, + max_workers=2, + raise_exception=True, + ) + assert res_up == [None, None] + + # 2. upload_many_from_filenames (THREAD) attributes gcp.client.method properly + session.request.side_effect = lambda method, url, **kw: _json_response( + {"name": "uploaded", "size": "6"} + ) + transfer_manager.upload_many_from_filenames( + bucket, + ["u1.txt", "u2.txt"], + source_directory=str(tmp_path), + worker_type=transfer_manager.THREAD, + max_workers=2, + raise_exception=True, + ) + + # 3. download_many (THREAD) with filename, skipped existing file, and BytesIO + existing_dest = tmp_path / "already_exists.txt" + existing_dest.write_bytes(b"keep") + new_dest = tmp_path / "d1.txt" + payload = b"dl-bytes" + crc_hdr = f"crc32c={_crc32c_b64(payload)}" + session.request.side_effect = lambda method, url, **kw: _media_response( + payload, + status_code=200, + headers={"x-goog-generation": "1", "x-goog-hash": crc_hdr}, + streaming=True, + ) + mem_out = io.BytesIO() + transfer_manager.download_many( + [ + (bucket.blob("skip-me"), str(existing_dest)), + (bucket.blob("d1"), str(new_dest)), + (bucket.blob("d2"), mem_out), + ], + skip_if_exists=True, + worker_type=transfer_manager.THREAD, + max_workers=2, + raise_exception=True, + ) + assert new_dest.read_bytes() == payload + assert mem_out.getvalue() == payload + + # 4. download_many_to_path (THREAD) attributes gcp.client.method properly + dl_dir = tmp_path / "dl_dir" + dl_dir.mkdir() + session.request.side_effect = [ + _media_response( + payload, + status_code=200, + headers={"x-goog-generation": "1", "x-goog-hash": crc_hdr}, + streaming=True, + ), + ] + transfer_manager.download_many_to_path( + bucket, + ["p1.txt"], + destination_directory=str(dl_dir), + worker_type=transfer_manager.THREAD, + max_workers=1, + raise_exception=True, + ) + assert (dl_dir / "p1.txt").read_bytes() == payload + + # 5. PROCESS worker_type pickling branches + with pytest.raises(ValueError): + transfer_manager.upload_many( + [(io.BytesIO(b"bad"), bucket.blob("bad"))], + worker_type=transfer_manager.PROCESS, + ) + with pytest.raises(ValueError): + transfer_manager.download_many( + [(bucket.blob("bad"), io.BytesIO())], + worker_type=transfer_manager.PROCESS, + ) + + proc_client = Client( + project="test-project", + credentials=_make_credentials(), + enable_metrics=False, + ) + proc_client._bucket_metadata_cache = None + proc_bucket = proc_client.bucket("proc-b") + proc_dest = tmp_path / "proc_dl.txt" + with mock.patch( + "google.cloud.storage.transfer_manager._get_pool_class_and_requirements", + return_value=(concurrent.futures.ThreadPoolExecutor, True), + ): + with mock.patch.object( + transfer_manager.Blob, "_handle_filename_and_upload", return_value=None + ): + transfer_manager.upload_many( + [(str(f1), proc_bucket.blob("pb1"))], + worker_type=transfer_manager.PROCESS, + max_workers=1, + raise_exception=True, + ) + with mock.patch.object( + transfer_manager.Blob, "_handle_filename_and_download", return_value=None + ): + transfer_manager.download_many( + [(proc_bucket.blob("pb1"), str(proc_dest))], + worker_type=transfer_manager.PROCESS, + max_workers=1, + raise_exception=True, + ) + proc_client.close() + + pts = _collect_metrics(reader) + by_method = {} + for p in pts["gcp.storage.client.operations"]: + key = ( + p.attributes["gcp.client.method"], + p.attributes["gcp.storage.operation"], + ) + by_method[key] = by_method.get(key, 0) + p.value + + assert by_method == { + ("storage.transfer_manager.upload_many", "WriteObject"): 2, + ("storage.transfer_manager.upload_many_from_filenames", "WriteObject"): 2, + ("storage.transfer_manager.download_many", "ReadObject"): 2, + ("storage.transfer_manager.download_many_to_path", "ReadObject"): 1, + } + + +def test_transfer_manager_concurrent_chunked_operations(metrics_env, tmp_path): + client, session, reader = metrics_env + bucket = client.bucket("chunked-bucket") + + # 1. download_chunks_concurrently validation errors -> CLIENT_VALIDATION_ERROR + v_blob = bucket.blob("val-blob") + with pytest.raises(ValueError): + transfer_manager.download_chunks_concurrently( + v_blob, str(tmp_path / "v1"), download_kwargs={"start": 0} + ) + with pytest.raises(ValueError): + transfer_manager.download_chunks_concurrently( + v_blob, str(tmp_path / "v2"), download_kwargs={"checksum": "md5"} + ) + + # 2. download_chunks_concurrently (THREAD): reload + 2 shards -> 1 ReadObject, 1 body size, 3 HTTP attempts + full_data = b"0123456789" + full_crc = _crc32c_b64(full_data) + d_blob = bucket.blob("sharded-dl") + + def _sharded_get(method, url, **kwargs): + if "/storage/v1/b/" in url and "/download/" not in url: + return _json_response( + { + "name": "sharded-dl", + "size": "10", + "generation": "42", + "crc32c": full_crc, + } + ) + rng = (kwargs.get("headers") or {}).get("range", "") + if rng == "bytes=0-5": + return _media_response( + b"012345", + status_code=http.client.PARTIAL_CONTENT, + headers={"content-range": "bytes 0-5/10", "x-goog-generation": "42"}, + streaming=True, + ) + return _media_response( + b"6789", + status_code=http.client.PARTIAL_CONTENT, + headers={"content-range": "bytes 6-9/10", "x-goog-generation": "42"}, + streaming=True, + ) + + session.request.side_effect = _sharded_get + dl_target = tmp_path / "sharded_out.bin" + transfer_manager.download_chunks_concurrently( + d_blob, + str(dl_target), + chunk_size=6, + worker_type=transfer_manager.THREAD, + max_workers=2, + crc32c_checksum=True, + ) + assert dl_target.read_bytes() == full_data + + # 3. download_chunks_concurrently CRC32C mismatch -> CHECKSUM_MISMATCH, no body size recorded + bad_blob = bucket.blob("bad-crc") + bad_blob._properties.update( + {"size": 4, "generation": "9", "crc32c": _crc32c_b64(b"zzzz")} + ) + session.request.side_effect = [ + _media_response( + b"abcd", + status_code=http.client.PARTIAL_CONTENT, + headers={"content-range": "bytes 0-3/4", "x-goog-generation": "9"}, + streaming=True, + ) + ] + with pytest.raises(DataCorruption): + transfer_manager.download_chunks_concurrently( + bad_blob, + str(tmp_path / "bad_crc.bin"), + chunk_size=4, + worker_type=transfer_manager.THREAD, + max_workers=1, + crc32c_checksum=True, + ) + + # 4. upload_chunks_concurrently (THREAD): initiate + 2 parts + finalize -> 1 WriteObject, 1 body size, 4 attempts + up_src = tmp_path / "sharded_up.bin" + up_src.write_bytes(b"abcdefghij") + u_blob = bucket.blob("sharded-up") + u_blob._bucket = bucket.__class__(client, "chunked-bucket", user_project="up-proj") + u_blob.kms_key_name = "projects/p/locations/l/keyRings/r/cryptoKeys/k" + + xml_ns = 'xmlns="http://s3.amazonaws.com/doc/2006-03-01/"' + + def _mpu_transport(method, url, **kwargs): + if method == "POST" and "uploads" in url: + return _media_response( + f"mpu-1".encode( + "utf-8" + ), + status_code=200, + ) + if method == "PUT": + part_bytes = kwargs.get("data", b"") + return _media_response( + b"", + status_code=200, + headers={ + "ETag": '"part-etag"', + "x-goog-hash": f"crc32c={_crc32c_b64(part_bytes)}", + }, + ) + return _media_response( + f'"done"'.encode( + "utf-8" + ), + status_code=200, + ) + + session.request.side_effect = _mpu_transport + transfer_manager.upload_chunks_concurrently( + str(up_src), + u_blob, + chunk_size=6, + worker_type=transfer_manager.THREAD, + max_workers=2, + ) + + # 5. upload_chunks_concurrently part failure triggers suppressed container.cancel() + def _mpu_fail_transport(method, url, **kwargs): + if method == "POST" and "uploads" in url: + return _media_response( + f"mpu-fail".encode( + "utf-8" + ), + status_code=200, + ) + if method == "PUT": + return _json_response({"error": {"message": "503"}}, status_code=503) + return _media_response(b"", status_code=204) + + session.request.side_effect = _mpu_fail_transport + with pytest.raises(InvalidResponse): + transfer_manager.upload_chunks_concurrently( + str(up_src), + bucket.blob("sharded-up-fail"), + chunk_size=10, + worker_type=transfer_manager.THREAD, + max_workers=1, + retry=None, + ) + + # 6. PROCESS pickling & suppressed / disabled metrics branches for chunked helpers + with _helpers._suppress_body_size(): + session.request.side_effect = _mpu_transport + transfer_manager.upload_chunks_concurrently( + str(up_src), + bucket.blob("suppressed-body-up"), + chunk_size=10, + worker_type=transfer_manager.THREAD, + max_workers=1, + ) + zero_sharded = bucket.blob("zero-sharded") + zero_sharded._properties.update({"size": 4, "generation": "1"}) + session.request.side_effect = [ + _media_response( + b"0123", + status_code=http.client.PARTIAL_CONTENT, + headers={"content-range": "bytes 0-3/4", "x-goog-generation": "1"}, + streaming=True, + ) + ] + transfer_manager.download_chunks_concurrently( + zero_sharded, + str(tmp_path / "sup_dl.bin"), + chunk_size=4, + worker_type=transfer_manager.THREAD, + max_workers=1, + crc32c_checksum=False, + ) + + proc_client = Client( + project="test-project", + credentials=_make_credentials(), + enable_metrics=False, + ) + proc_client._bucket_metadata_cache = None + p_blob = proc_client.bucket("pb").blob("po") + p_blob._properties.update({"size": 4, "generation": "1"}) + with mock.patch( + "google.cloud.storage.transfer_manager._get_pool_class_and_requirements", + return_value=(concurrent.futures.ThreadPoolExecutor, True), + ): + with mock.patch.object( + transfer_manager.Blob, "_prep_and_do_download", return_value=None + ): + transfer_manager.download_chunks_concurrently( + p_blob, + str(tmp_path / "proc_sharded.bin"), + chunk_size=4, + worker_type=transfer_manager.PROCESS, + max_workers=1, + crc32c_checksum=False, + ) + with mock.patch.object(XMLMPUContainer, "initiate"): + with mock.patch.object(XMLMPUContainer, "finalize"): + with mock.patch.object(XMLMPUPart, "upload"): + transfer_manager.upload_chunks_concurrently( + str(up_src), + p_blob, + chunk_size=10, + worker_type=transfer_manager.PROCESS, + max_workers=1, + ) + proc_client.close() + + pts = _collect_metrics(reader) + dl_chunk_ops = [ + p + for p in pts["gcp.storage.client.operations"] + if p.attributes["gcp.client.method"] + == "storage.transfer_manager.download_chunks_concurrently" + ] + assert sum(p.value for p in dl_chunk_ops) == 5 + assert any(p.attributes["error.type"] == "CHECKSUM_MISMATCH" for p in dl_chunk_ops) + + up_chunk_ops = [ + p + for p in pts["gcp.storage.client.operations"] + if p.attributes["gcp.client.method"] + == "storage.transfer_manager.upload_chunks_concurrently" + ] + assert sum(p.value for p in up_chunk_ops) == 3 + + assert not any( + p.attributes.get("http.request.method") == "DELETE" + for p in pts["http.client.request.duration"] + ) + resp_body = pts["gcp.storage.client.response.body.size"] + assert sum(p.count for p in resp_body) == 1 + assert resp_body[0].sum == 10 + req_body = pts["gcp.storage.client.request.body.size"] + assert sum(p.count for p in req_body) == 1 + assert req_body[0].sum == 10 + + +def test_transfer_manager_cleanup_chaining_multiprocessing_and_lazy_client( + metrics_env, tmp_path, monkeypatch +): + client, session, reader = metrics_env + bucket = client.bucket("r1-bucket") + xml_ns = 'xmlns="http://s3.amazonaws.com/doc/2006-03-01/"' + + # 1. upload_chunks_concurrently where part upload raises 403 InvalidResponse AND container.cancel raises ConnectionError + up_src = tmp_path / "mpu_cleanup.bin" + up_src.write_bytes(b"0123456789") + session.request.side_effect = [ + _media_response( + f"u-clean".encode( + "utf-8" + ), + status_code=200, + ), + _json_response({"error": {"message": "part forbidden"}}, status_code=403), + requests.ConnectionError("cancel network down"), + ] + with pytest.raises(requests.ConnectionError, match="cancel network down"): + transfer_manager.upload_chunks_concurrently( + str(up_src), + bucket.blob("mpu-cleanup-err"), + chunk_size=10, + worker_type=transfer_manager.THREAD, + max_workers=1, + retry=None, + ) + + # 2. download_chunks_concurrently honors download_kwargs["client"] + disabled_client = Client( + project="test-project", + credentials=_make_credentials(), + _http=mock.MagicMock(), + enable_metrics=False, + ) + disabled_client._bucket_metadata_cache = None + foreign_blob = disabled_client.bucket("r1-bucket").blob("foreign-sharded") + full_bytes = b"abcdefgh" + session.request.side_effect = [ + _json_response( + { + "name": "foreign-sharded", + "size": "8", + "generation": "5", + "crc32c": _crc32c_b64(full_bytes), + } + ), + _media_response( + b"abcd", + status_code=http.client.PARTIAL_CONTENT, + headers={"content-range": "bytes 0-3/8", "x-goog-generation": "5"}, + streaming=True, + ), + _media_response( + b"efgh", + status_code=http.client.PARTIAL_CONTENT, + headers={"content-range": "bytes 4-7/8", "x-goog-generation": "5"}, + streaming=True, + ), + ] + foreign_out = tmp_path / "foreign_out.bin" + transfer_manager.download_chunks_concurrently( + foreign_blob, + str(foreign_out), + chunk_size=4, + download_kwargs={"client": client}, + worker_type=transfer_manager.THREAD, + max_workers=1, + crc32c_checksum=True, + ) + assert foreign_out.read_bytes() == full_bytes + disabled_client.close() + + # 3. upload_many_from_filenames & download_many_to_path with needs_pickling=True, + # plus _suppress_internal_metrics() propagation across needs_pickling=True + src_dir = tmp_path / "proc_src" + src_dir.mkdir() + (src_dir / "f1.txt").write_bytes(b"hello") + dst_dir = tmp_path / "proc_dst" + dst_dir.mkdir() + + with monkeypatch.context() as mp_cache: + mp_cache.setattr(transfer_manager, "_cached_clients", {id(client): client}) + with mock.patch( + "google.cloud.storage.transfer_manager._get_pool_class_and_requirements", + return_value=(concurrent.futures.ThreadPoolExecutor, True), + ): + session.request.side_effect = [ + _json_response({"name": "f1.txt", "size": "5"}), + _media_response( + b"hello", + status_code=200, + headers={ + "x-goog-generation": "1", + "x-goog-hash": f"crc32c={_crc32c_b64(b'hello')}", + }, + streaming=True, + ), + ] + transfer_manager.upload_many_from_filenames( + bucket, + ["f1.txt"], + source_directory=str(src_dir), + worker_type=transfer_manager.PROCESS, + max_workers=1, + raise_exception=True, + ) + transfer_manager.download_many_to_path( + bucket, + ["f1.txt"], + destination_directory=str(dst_dir), + worker_type=transfer_manager.PROCESS, + max_workers=1, + raise_exception=True, + ) + assert (dst_dir / "f1.txt").read_bytes() == b"hello" + + pts_pre_sup = _collect_metrics(reader) + attempts_pre_sup = sum( + p.value for p in pts_pre_sup["gcp.storage.client.attempts"] + ) + ops_pre_sup = sum( + p.value for p in pts_pre_sup["gcp.storage.client.operations"] + ) + + sup_blob = bucket.blob("sup-sharded") + sup_blob._properties.update({"size": 4, "generation": "1"}) + session.request.side_effect = [ + _json_response({"name": "f1.txt", "size": "5"}), + _media_response( + b"hello", + status_code=200, + headers={ + "x-goog-generation": "1", + "x-goog-hash": f"crc32c={_crc32c_b64(b'hello')}", + }, + streaming=True, + ), + _media_response( + b"0123", + status_code=http.client.PARTIAL_CONTENT, + headers={"content-range": "bytes 0-3/4", "x-goog-generation": "1"}, + streaming=True, + ), + _media_response( + f"u-sup".encode( + "utf-8" + ), + status_code=200, + ), + _media_response( + b"", + status_code=200, + headers={ + "ETag": '"e1"', + "x-goog-hash": f"crc32c={_crc32c_b64(b'0123456789')}", + }, + ), + _media_response( + f'"done"'.encode( + "utf-8" + ), + status_code=200, + ), + ] + with _helpers._suppress_internal_metrics(): + transfer_manager.upload_many( + [(str(src_dir / "f1.txt"), bucket.blob("sup-u"))], + worker_type=transfer_manager.PROCESS, + max_workers=1, + raise_exception=True, + ) + transfer_manager.download_many( + [(bucket.blob("sup-d"), str(tmp_path / "sup_d.txt"))], + worker_type=transfer_manager.PROCESS, + max_workers=1, + raise_exception=True, + ) + transfer_manager.download_chunks_concurrently( + sup_blob, + str(tmp_path / "sup_chunk_d.bin"), + chunk_size=4, + worker_type=transfer_manager.PROCESS, + max_workers=1, + crc32c_checksum=False, + ) + transfer_manager.upload_chunks_concurrently( + str(up_src), + bucket.blob("sup-chunk-u"), + chunk_size=10, + worker_type=transfer_manager.PROCESS, + max_workers=1, + ) + + pts_post_sup = _collect_metrics(reader) + assert ( + sum(p.value for p in pts_post_sup["gcp.storage.client.attempts"]) + == attempts_pre_sup + ) + assert ( + sum(p.value for p in pts_post_sup["gcp.storage.client.operations"]) + == ops_pre_sup + ) + + # 4. Isolated _LazyClient & _cleanup_cached_clients branches, plus real subprocess smoke test + with monkeypatch.context() as mp: + fake_atexit = mock.MagicMock() + fake_finalize = mock.MagicMock() + mp.setattr(transfer_manager, "_cached_clients", {}) + mp.setattr(transfer_manager, "_cleanup_registered_pid", None) + mp.setattr(transfer_manager.atexit, "register", fake_atexit) + mp.setattr(transfer_manager.multiprocessing.util, "Finalize", fake_finalize) + + base_pos = ("proj", _make_credentials(), mock.MagicMock(), None, None) + c5 = transfer_manager._LazyClient("id-5", *base_pos, {}, True, False) + assert transfer_manager._LazyClient("id-5", *base_pos) is c5 + transfer_manager._LazyClient("id-5only", *base_pos, enable_metrics=False) + transfer_manager._LazyClient( + "id-6", *base_pos, {"X-Test": "1"}, enable_metrics=False + ) + transfer_manager._LazyClient("id-7", *base_pos, {}, False) + transfer_manager._LazyClient("id-8", *base_pos, {}, False, False) + fake_atexit.assert_called_once_with(transfer_manager._cleanup_cached_clients) + fake_finalize.assert_called_once_with( + None, transfer_manager._cleanup_cached_clients, exitpriority=10 + ) + + flush_provider = mock.Mock(spec=["force_flush"]) + with mock.patch.object( + _opentelemetry_metrics.metrics, + "get_meter_provider", + return_value=flush_provider, + ): + transfer_manager._cleanup_cached_clients() + transfer_manager._cleanup_cached_clients() + flush_provider.force_flush.assert_called_once() + assert transfer_manager._cached_clients == {} + + bad_close_cl = mock.Mock( + metrics_enabled=False, + debug_metrics_enabled=False, + close=mock.Mock(side_effect=RuntimeError("close boom")), + ) + ok_close_cl = mock.Mock(metrics_enabled=False, debug_metrics_enabled=False) + transfer_manager._cached_clients.update({"b1": bad_close_cl, "b2": ok_close_cl}) + with mock.patch.object( + _opentelemetry_metrics.metrics, + "get_meter_provider", + side_effect=AssertionError("must not flush when metrics disabled"), + ): + transfer_manager._cleanup_cached_clients() + bad_close_cl.close.assert_called_once() + ok_close_cl.close.assert_called_once() + + err_flush_provider = mock.Mock( + spec=["force_flush"], + force_flush=mock.Mock(side_effect=RuntimeError("flush boom")), + ) + transfer_manager._cached_clients["dbg"] = mock.Mock( + metrics_enabled=False, debug_metrics_enabled=True + ) + with mock.patch.object( + _opentelemetry_metrics.metrics, + "get_meter_provider", + return_value=err_flush_provider, + ): + transfer_manager._cleanup_cached_clients() + err_flush_provider.force_flush.assert_called_once() + + transfer_manager._cached_clients["act-no-flush"] = mock.Mock( + metrics_enabled=True, debug_metrics_enabled=False + ) + with mock.patch.object( + _opentelemetry_metrics.metrics, + "get_meter_provider", + return_value=object(), + ): + transfer_manager._cleanup_cached_clients() + + transfer_manager._cached_clients["act-no-otel"] = mock.Mock( + metrics_enabled=True, debug_metrics_enabled=False + ) + mp.setattr(_opentelemetry_metrics, "metrics", None) + transfer_manager._cleanup_cached_clients() + + real_proc_client = Client( + project="test-project", + credentials=_make_credentials(), + client_options={"api_endpoint": "http://127.0.0.1:1"}, + enable_metrics=True, + ) + real_proc_client._bucket_metadata_cache = None + real_results = transfer_manager.upload_many( + [(str(src_dir / "f1.txt"), real_proc_client.bucket("b").blob("o"))], + upload_kwargs={"retry": None, "timeout": 0.2}, + worker_type=transfer_manager.PROCESS, + max_workers=1, + raise_exception=False, + ) + assert len(real_results) == 1 + assert isinstance(real_results[0], Exception) + real_proc_client.close() + + pts = _collect_metrics(reader) + by_method_err = { + ( + p.attributes["gcp.client.method"], + p.attributes["gcp.storage.operation"], + p.attributes["error.type"], + ): p.value + for p in pts["gcp.storage.client.operations"] + } + assert ( + "storage.transfer_manager.upload_chunks_concurrently", + "WriteObject", + "PERMISSION_DENIED", + ) in by_method_err + assert ( + "storage.transfer_manager.download_chunks_concurrently", + "ReadObject", + "OK", + ) in by_method_err + assert ( + "storage.transfer_manager.upload_many_from_filenames", + "WriteObject", + "OK", + ) in by_method_err + assert ( + "storage.transfer_manager.download_many_to_path", + "ReadObject", + "OK", + ) in by_method_err diff --git a/packages/google-cloud-storage/tests/unit/test_otel_veneer_blob_metrics.py b/packages/google-cloud-storage/tests/unit/test_otel_veneer_blob_metrics.py new file mode 100644 index 000000000000..29fd63676549 --- /dev/null +++ b/packages/google-cloud-storage/tests/unit/test_otel_veneer_blob_metrics.py @@ -0,0 +1,357 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for OpenTelemetry Blob control-plane and Veneer metrics.""" + +import http.client +import io +import json +from unittest import mock + +import pytest +import requests +from google.api_core import iam +from google.auth import credentials as auth_credentials +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader + +from google.cloud.storage import _opentelemetry_metrics +from google.cloud.storage.blob import Blob +from google.cloud.storage.client import Client + + +def _make_credentials(): + return auth_credentials.AnonymousCredentials() + + +def _json_response(payload=None, status_code=http.client.OK, headers=None): + response = requests.Response() + response.status_code = status_code + response.headers.update({"Content-Type": "application/json"}) + if headers: + response.headers.update(headers) + if payload is not None: + response._content = ( + payload + if isinstance(payload, bytes) + else json.dumps(payload).encode("utf-8") + ) + else: + response._content = b"" + response.request = requests.Request( + "GET", "https://storage.googleapis.com/storage/v1/b" + ).prepare() + return response + + +def _collect_metrics(reader): + data = reader.get_metrics_data() + collected = {} + if data is None: # pragma: NO COVER + return collected + for resource_metric in data.resource_metrics: + for scope_metric in resource_metric.scope_metrics: + for metric in scope_metric.metrics: + collected.setdefault(metric.name, []).extend(metric.data.data_points) + return collected + + +@pytest.fixture(autouse=True) +def _enable_dev_gate(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + + +@pytest.fixture +def metrics_env(): + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + session = mock.MagicMock() + client = Client( + project="test-project", + credentials=_make_credentials(), + _http=session, + enable_metrics=True, + meter_provider=provider, + ) + client._bucket_metadata_cache = None + yield client, session, reader + client.close() + + +def test_blob_exists_404_and_subclass_reload(metrics_env): + client, session, reader = metrics_env + bucket = client.bucket("my-bucket") + blob = bucket.blob("missing-obj") + + session.request.side_effect = [ + _json_response({"error": {"message": "not found"}}, status_code=404), + _json_response({"name": "sub-o"}), + ] + assert blob.exists() is False + + class _UserBlob(Blob): + pass + + uo = _UserBlob("sub-o", bucket) + uo.reload() + + pts = _collect_metrics(reader) + ops = pts["gcp.storage.client.operations"] + assert sum(p.value for p in ops) == 2 + assert {p.attributes["gcp.client.method"] for p in ops} == { + "storage.Blob.exists", + "storage.Blob.reload", + } + assert all(p.attributes["error.type"] == "OK" for p in ops) + + att_404 = [ + p + for p in pts["gcp.storage.client.attempts"] + if p.attributes["error.type"] == "NOT_FOUND" + ] + assert len(att_404) == 1 + assert att_404[0].attributes["gcp.storage.operation"] == "GetObject" + + +def test_all_blob_veneer_control_plane_and_entrypoint_methods(metrics_env): + client, session, reader = metrics_env + bucket = client.bucket("my-bucket") + blob = bucket.blob("my-obj") + + iam_payload = { + "version": 1, + "etag": "CAE=", + "bindings": [{"role": "roles/storage.objectViewer", "members": ["allUsers"]}], + } + session.request.side_effect = [ + _json_response({"name": "my-obj"}), # blob.update + _json_response({"name": "my-obj"}), # blob.patch + _json_response(iam_payload), # blob.get_iam_policy + _json_response(iam_payload), # blob.set_iam_policy + _json_response({"permissions": ["storage.objects.get"]}), # blob.test_iam + _json_response({"items": []}), # blob.make_public reload acl + _json_response({"acl": []}), # blob.make_public save acl + _json_response({"acl": []}), # blob.make_private save acl (loaded) + _json_response({"name": "my-obj"}), # blob.compose + _json_response( + { + "done": False, + "rewriteToken": "tok1", + "totalBytesRewritten": "10", + "objectSize": "20", + } + ), # blob.rewrite step 1 + _json_response( + { + "done": False, + "rewriteToken": "tok2", + "totalBytesRewritten": "10", + "objectSize": "20", + } + ), # blob.update_storage_class step 1 + _json_response( + { + "done": True, + "totalBytesRewritten": "20", + "objectSize": "20", + "resource": {"name": "my-obj", "storageClass": "NEARLINE"}, + } + ), # blob.update_storage_class step 2 + _json_response(None, status_code=204), # blob.delete + ] + blob.update() + blob.patch() + b_pol = iam.Policy() + assert blob.get_iam_policy().version == 1 + blob.set_iam_policy(b_pol) + assert blob.test_iam_permissions(["storage.objects.get"]) == ["storage.objects.get"] + blob.make_public() + blob.make_private() + src1 = bucket.blob("s1") + src2 = bucket.blob("s2") + blob.compose([src1, src2]) + token, _, _ = blob.rewrite(src1) + assert token == "tok1" + blob._encryption_key = b"0" * 32 + blob.update_storage_class("NEARLINE") + blob._encryption_key = None + blob.delete() + + with mock.patch.object(Blob, "_prep_and_do_download", return_value=None): + blob.download_to_file(io.BytesIO()) + with mock.patch("builtins.open", mock.mock_open()): + blob.download_to_filename("/tmp/fake-download-unused") + with mock.patch.object( + Blob, + "_prep_and_do_download", + side_effect=lambda buf, **kw: buf.write(b"hello"), + ): + assert blob.download_as_bytes() == b"hello" + with pytest.warns(PendingDeprecationWarning): + assert blob.download_as_string() == b"hello" + assert blob.download_as_text() == "hello" + + with mock.patch.object(Blob, "_do_upload", return_value={"name": "my-obj"}): + blob.upload_from_file(io.BytesIO(b"abc")) + blob.upload_from_string("abc") + with mock.patch("builtins.open", mock.mock_open(read_data=b"abc")): + with mock.patch("os.path.getsize", return_value=3): + blob.upload_from_filename("/tmp/fake-upload-unused") + + with mock.patch.object( + Blob, + "_initiate_resumable_upload", + return_value=(mock.Mock(resumable_url="https://u"), None), + ): + assert blob.create_resumable_upload_session() == "https://u" + + pts = _collect_metrics(reader) + methods_seen = { + p.attributes["gcp.client.method"] for p in pts["gcp.storage.client.operations"] + } + expected_methods = { + "storage.Blob.update", + "storage.Blob.patch", + "storage.Blob.delete", + "storage.Blob.get_iam_policy", + "storage.Blob.set_iam_policy", + "storage.Blob.test_iam_permissions", + "storage.Blob.make_public", + "storage.Blob.make_private", + "storage.Blob.compose", + "storage.Blob.rewrite", + "storage.Blob.update_storage_class", + "storage.Blob.download_to_file", + "storage.Blob.download_to_filename", + "storage.Blob.download_as_bytes", + "storage.Blob.download_as_string", + "storage.Blob.download_as_text", + "storage.Blob.upload_from_file", + "storage.Blob.upload_from_filename", + "storage.Blob.upload_from_string", + "storage.Blob.create_resumable_upload_session", + } + assert expected_methods == methods_seen + + +def test_blob_batch_deferral_non_batchable_ops_and_client_override(metrics_env): + client, session, reader = metrics_env + bucket = client.bucket("my-bucket") + + # 1. Batchable Blob ops defer to Batch.finish; non-batchable Blob ops record immediately + multipart_ok_and_403 = ( + b"--b\r\n" + b"Content-Type: application/http\r\n" + b"Content-ID: \r\n\r\n" + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: application/json\r\n\r\n" + b'{"name": "o1"}\r\n' + b"--b\r\n" + b"Content-Type: application/http\r\n" + b"Content-ID: \r\n\r\n" + b"HTTP/1.1 403 Forbidden\r\n" + b"Content-Type: application/json\r\n\r\n" + b'{"error": {"message": "denied"}}\r\n' + b"--b--\r\n" + ) + session.request.return_value = _json_response( + multipart_ok_and_403, + status_code=200, + headers={"Content-Type": "multipart/mixed; boundary=b"}, + ) + blob = bucket.blob("media-in-batch") + with ( + mock.patch.object(blob, "_do_download"), + mock.patch.object(blob, "_do_upload", return_value={"name": "media-in-batch"}), + mock.patch( + "google.cloud.storage.blob.ResumableUpload.initiate", + return_value=_json_response( + {}, headers={"location": "https://upload.example/1"} + ), + ), + ): + with client.batch(raise_exception=False): + bucket.blob("o1").patch() # Batchable JSON -> deferred to Batch.finish + bucket.blob("o2").delete() # Batchable JSON -> deferred to Batch.finish + blob.download_as_bytes() + blob.upload_from_string(b"abc") + blob.create_resumable_upload_session() + + pts_batch = _collect_metrics(reader) + batch_methods = { + p.attributes["gcp.client.method"] + for p in pts_batch["gcp.storage.client.operations"] + } + assert batch_methods == { + "storage.Batch.finish", + "storage.Blob.download_as_bytes", + "storage.Blob.upload_from_string", + "storage.Blob.create_resumable_upload_session", + } + assert "storage.Blob.patch" not in batch_methods + assert "storage.Blob.delete" not in batch_methods + + # 2. Blob.update_storage_class(..., client=other_client) routes both logical and HTTP metrics to other_client + uninstrumented_session = mock.MagicMock() + uninstrumented_client = Client( + project="test-project", + credentials=_make_credentials(), + _http=uninstrumented_session, + enable_metrics=False, + ) + uninstrumented_client._bucket_metadata_cache = None + other_reader = InMemoryMetricReader() + other_provider = MeterProvider(metric_readers=[other_reader]) + other_session = mock.MagicMock() + other_session.request.side_effect = [ + _json_response( + { + "done": False, + "rewriteToken": "tok-1", + "totalBytesRewritten": "5", + "objectSize": "10", + } + ), + _json_response( + { + "done": True, + "totalBytesRewritten": "10", + "objectSize": "10", + "resource": {"name": "obj", "storageClass": "NEARLINE"}, + } + ), + ] + other_client = Client( + project="test-project", + credentials=_make_credentials(), + _http=other_session, + enable_metrics=True, + meter_provider=other_provider, + ) + other_client._bucket_metadata_cache = None + ublob = uninstrumented_client.bucket("my-bucket").blob("obj") + ublob.update_storage_class("NEARLINE", client=other_client) + + assert uninstrumented_session.request.call_count == 0 + assert other_session.request.call_count == 2 + other_pts = _collect_metrics(other_reader) + assert len(other_pts["gcp.storage.client.operations"]) == 1 + assert ( + other_pts["gcp.storage.client.operations"][0].attributes["gcp.client.method"] + == "storage.Blob.update_storage_class" + ) + assert sum(p.value for p in other_pts["gcp.storage.client.attempts"]) == 2 + uninstrumented_client.close() + other_client.close() diff --git a/packages/google-cloud-storage/tests/unit/test_otel_veneer_bucket_metrics.py b/packages/google-cloud-storage/tests/unit/test_otel_veneer_bucket_metrics.py new file mode 100644 index 000000000000..6de64f2502f9 --- /dev/null +++ b/packages/google-cloud-storage/tests/unit/test_otel_veneer_bucket_metrics.py @@ -0,0 +1,576 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for OpenTelemetry Client, Bucket, Batch, ACL, HMAC & Notification Veneer metrics.""" + +import http.client +import io +import json +from unittest import mock + +import pytest +import requests +from google.api_core import retry +from google.auth import credentials as auth_credentials +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader + +from google.cloud.storage import _opentelemetry_metrics +from google.cloud.storage.acl import ( + ACL, + BucketACL, + DefaultObjectACL, + ObjectACL, + _ACLEntity, +) +from google.cloud.storage.blob import Blob +from google.cloud.storage.bucket import Bucket +from google.cloud.storage.client import Client +from google.cloud.storage.hmac_key import HMACKeyMetadata +from google.cloud.storage.notification import BucketNotification + + +def _make_credentials(): + return auth_credentials.AnonymousCredentials() + + +def _json_response(payload=None, status_code=http.client.OK, headers=None): + response = requests.Response() + response.status_code = status_code + response.headers.update({"Content-Type": "application/json"}) + if headers: + response.headers.update(headers) + if payload is not None: + response._content = ( + payload + if isinstance(payload, bytes) + else json.dumps(payload).encode("utf-8") + ) + else: + response._content = b"" + response.request = requests.Request( + "GET", "https://storage.googleapis.com/storage/v1/b" + ).prepare() + return response + + +def _collect_metrics(reader): + data = reader.get_metrics_data() + collected = {} + if data is None: # pragma: NO COVER + return collected + for resource_metric in data.resource_metrics: + for scope_metric in resource_metric.scope_metrics: + for metric in scope_metric.metrics: + collected.setdefault(metric.name, []).extend(metric.data.data_points) + return collected + + +@pytest.fixture(autouse=True) +def _enable_dev_gate(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + + +@pytest.fixture +def metrics_env(): + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + session = mock.MagicMock() + client = Client( + project="test-project", + credentials=_make_credentials(), + _http=session, + enable_metrics=True, + meter_provider=provider, + ) + client._bucket_metadata_cache = None + yield client, session, reader + client.close() + + +def test_client_get_bucket_retries_and_bucket_patch_error(metrics_env): + client, session, reader = metrics_env + session.request.side_effect = [ + _json_response({"error": {"message": "503-1"}}, status_code=503), + requests.ConnectionError("transient reset"), + _json_response({"id": "my-bucket", "name": "my-bucket"}, status_code=200), + ] + fast_retry = retry.Retry(initial=0.001, maximum=0.002, deadline=5.0) + bucket = client.get_bucket("my-bucket", retry=fast_retry) + assert bucket.name == "my-bucket" + + pts = _collect_metrics(reader) + assert len(pts["gcp.storage.client.operations"]) == 1 + op = pts["gcp.storage.client.operations"][0] + dur = pts["gcp.client.request.duration"][0] + assert op.attributes["gcp.storage.operation"] == "GetBucket" + assert op.attributes["gcp.client.method"] == "storage.Client.get_bucket" + assert op.attributes["error.type"] == "OK" + assert dur.attributes["rpc.method"] == "google.storage.v2.Storage/GetBucket" + assert dur.attributes["error.type"] == "OK" + assert sum(p.value for p in pts["gcp.storage.client.attempts"]) == 3 + assert sum(p.value for p in pts["gcp.storage.client.retries"]) == 2 + assert sum(p.value for p in pts["gcp.storage.client.errors"]) == 2 + + # Terminal HTTP 400 BadRequest on Bucket.patch maps to INVALID_ARGUMENT + session.request.side_effect = [ + _json_response( + {"error": {"message": "Invalid bucket configuration"}}, + status_code=400, + ) + ] + b = client.bucket("my-bucket") + with pytest.raises(Exception): + b.patch(retry=None) + pts_after = _collect_metrics(reader) + patch_ops = [ + p + for p in pts_after["gcp.storage.client.operations"] + if p.attributes["gcp.client.method"] == "storage.Bucket.patch" + ] + assert len(patch_ops) == 1 + assert patch_ops[0].attributes["error.type"] == "INVALID_ARGUMENT" + + +def test_existence_and_lookup_404_semantics(metrics_env): + client, session, reader = metrics_env + session.request.side_effect = [ + _json_response({"error": {"message": "not found"}}, status_code=404), + _json_response({"error": {"message": "not found"}}, status_code=404), + _json_response({"error": {"message": "not found"}}, status_code=404), + _json_response({"error": {"message": "not found"}}, status_code=404), + _json_response({"error": {"message": "not found"}}, status_code=404), + ] + + assert client.lookup_bucket("missing-b") is None + bucket = client.bucket("missing-b") + assert bucket.exists() is False + assert bucket.get_blob("missing-obj") is None + hmac_meta = HMACKeyMetadata(client, access_id="GOOG123") + assert hmac_meta.exists() is False + notif = BucketNotification(bucket, notification_id="n1") + assert notif.exists() is False + + pts = _collect_metrics(reader) + ops = pts["gcp.storage.client.operations"] + assert sum(p.value for p in ops) == 5 + for op in ops: + assert op.attributes["error.type"] == "OK" + for dur in pts["gcp.client.request.duration"]: + assert dur.attributes["error.type"] == "OK" + + attempts = pts["gcp.storage.client.attempts"] + assert sum(p.value for p in attempts) == 5 + for att in attempts: + assert att.attributes["error.type"] == "NOT_FOUND" + for http_dur in pts["http.client.request.duration"]: + assert http_dur.attributes["http.response.status_code"] == 404 + + +def test_lazy_pagination_iterators_and_precedence(metrics_env): + client, session, reader = metrics_env + + # Constructing iterators emits 0 metrics + b_iter = client.bucket("my-bucket").list_blobs() + c_iter = client.list_blobs("my-bucket") + buckets_iter = client.list_buckets() + hmac_iter = client.list_hmac_keys() + notif_iter = client.bucket("my-bucket").list_notifications() + assert _collect_metrics(reader) == {} + + # Iterating 2 pages on Bucket.list_blobs emits 2 ListObjects operations + # attributed to storage.Bucket.list_blobs (outermost wins over Client.list_blobs) + session.request.side_effect = [ + _json_response({"items": [{"name": "o1"}], "nextPageToken": "tok2"}), + _json_response({"items": [{"name": "o2"}]}), + _json_response({"items": [{"name": "o3"}]}), + _json_response({"items": [{"name": "b1"}]}), + _json_response({"items": [{"accessId": "k1"}]}), + _json_response( + { + "items": [ + { + "id": "n1", + "topic": "//pubsub.googleapis.com/projects/my-proj/topics/my-topic", + } + ] + } + ), + ] + assert [b.name for b in b_iter] == ["o1", "o2"] + assert [b.name for b in c_iter] == ["o3"] + assert [b.name for b in buckets_iter] == ["b1"] + assert [k.access_id for k in hmac_iter] == ["k1"] + assert [n.notification_id for n in notif_iter] == ["n1"] + + pts = _collect_metrics(reader) + by_method = {} + for p in pts["gcp.storage.client.operations"]: + key = ( + p.attributes["gcp.client.method"], + p.attributes["gcp.storage.operation"], + ) + by_method[key] = by_method.get(key, 0) + p.value + + assert by_method == { + ("storage.Bucket.list_blobs", "ListObjects"): 2, + ("storage.Client.list_blobs", "ListObjects"): 1, + ("storage.Client.list_buckets", "ListBuckets"): 1, + ("storage.Client.list_hmac_keys", "ListHMACKeys"): 1, + ("storage.Bucket.list_notifications", "ListNotifications"): 1, + } + + +def test_all_client_bucket_acl_hmac_and_notification_methods(metrics_env): + client, session, reader = metrics_env + + # Client methods + session.request.side_effect = [ + _json_response({"email_address": "sa@example.com"}), # get_service_account + _json_response({"name": "created-b"}), # create_bucket + _json_response({"name": "restored-b"}), # restore_bucket + _json_response( + {"secret": "sec", "metadata": {"accessId": "GOOG1"}} + ), # create_hmac_key + _json_response({"accessId": "GOOG1", "state": "ACTIVE"}), # get_hmac_key_meta + ] + assert client.get_service_account_email() == "sa@example.com" + assert client.create_bucket("created-b").name == "created-b" + assert client.restore_bucket("restored-b", generation=123).name == "restored-b" + meta, secret = client.create_hmac_key("sa@test.iam.gserviceaccount.com") + assert secret == "sec" + assert meta.access_id == "GOOG1" + assert client.get_hmac_key_metadata("GOOG1").state == "ACTIVE" + + # Client.download_blob_to_file (mocking _prep_and_do_download to test Veneer wrapper) + bucket = client.bucket("my-bucket") + blob = bucket.blob("my-obj") + with mock.patch.object(Blob, "_prep_and_do_download"): + client.download_blob_to_file(blob, io.BytesIO()) + + # Bucket methods + iam_payload = { + "version": 1, + "etag": "CAE=", + "bindings": [{"role": "roles/storage.objectViewer", "members": ["allUsers"]}], + } + session.request.side_effect = [ + _json_response({"name": "my-bucket"}), # bucket.create + _json_response({"name": "my-bucket"}), # bucket.reload + _json_response({"name": "my-bucket"}), # bucket.update + _json_response({"name": "my-bucket"}), # bucket.patch + _json_response( + { + "id": "n1", + "topic": "//pubsub.googleapis.com/projects/my-proj/topics/my-topic", + } + ), # bucket.get_notification + _json_response(None, status_code=204), # bucket.delete_blob + _json_response({"resource": {"name": "copied"}}), # bucket.copy_blob + _json_response({"name": "moved"}), # bucket.move_blob + _json_response({"name": "restored"}), # bucket.restore_blob + _json_response(iam_payload), # bucket.get_iam_policy + _json_response(iam_payload), # bucket.set_iam_policy + _json_response({"permissions": ["storage.objects.get"]}), # test_iam + _json_response({"name": "my-bucket", "metageneration": "2"}), # lock_retention + _json_response(None, status_code=204), # bucket.delete + ] + bucket.create() + bucket.reload() + bucket.update() + bucket.patch() + assert bucket.get_notification("n1").notification_id == "n1" + bucket.delete_blob("old-obj") + bucket.copy_blob(blob, bucket, new_name="copied") + bucket.move_blob(blob, new_name="moved") + bucket.restore_blob("restored", generation=1) + pol = bucket.get_iam_policy() + bucket.set_iam_policy(pol) + assert bucket.test_iam_permissions(["storage.objects.get"]) == [ + "storage.objects.get" + ] + bucket._properties["metageneration"] = 1 + bucket._properties["retentionPolicy"] = {"retentionPeriod": 100} + bucket.lock_retention_policy() + bucket.delete() + + # Subclass of Bucket preserves canonical class name in gcp.client.method + class _UserBucket(Bucket): + pass + + session.request.side_effect = [_json_response({"name": "sub-b"})] + ub = _UserBucket(client, "sub-b") + ub.reload() + + # Multi-RPC Bucket methods: delete(force=True), delete_blobs(on_error=...), + # rename_blob, make_public(recursive=True, future=True), make_private(recursive=True, future=True) + session.request.side_effect = [ + _json_response({"items": [{"name": "o1"}]}), # delete(force=True) list_blobs + _json_response(None, status_code=204), # delete(force=True) delete o1 + _json_response(None, status_code=204), # delete(force=True) delete bucket + _json_response( + {"error": {"message": "404"}}, status_code=404 + ), # delete_blobs 1 + _json_response(None, status_code=204), # delete_blobs 2 + _json_response({"resource": {"name": "renamed"}}), # rename_blob copy + _json_response(None, status_code=204), # rename_blob delete + _json_response({"items": []}), # make_public reload bucket acl + _json_response({"acl": []}), # make_public patch bucket acl + _json_response({"items": []}), # make_public reload default acl + _json_response({"defaultObjectAcl": []}), # make_public patch default acl + _json_response({"items": [{"name": "o1", "acl": []}]}), # make_public list + _json_response({"acl": []}), # make_public blob o1 patch + _json_response({"acl": []}), # make_private patch bucket acl (already loaded) + _json_response({"defaultObjectAcl": []}), # make_private patch default acl + _json_response({"items": [{"name": "o1", "acl": []}]}), # make_private list + _json_response({"acl": []}), # make_private blob o1 patch + ] + bucket.delete(force=True) + errors = [] + bucket.delete_blobs(["missing", "present"], on_error=errors.append) + assert len(errors) == 1 + bucket.rename_blob(blob, new_name="renamed") + bucket.make_public(recursive=True, future=True) + bucket.make_private(recursive=True, future=True) + + # ACL, DefaultObjectACL, ObjectACL + for acl_obj in (BucketACL(bucket), DefaultObjectACL(bucket), ObjectACL(blob)): + # save() when loaded=False is a local no-op -> 0 metrics + acl_obj.loaded = False + acl_obj.save() + session.request.side_effect = [ + _json_response({"items": [{"entity": "allUsers", "role": "READER"}]}), + _json_response({}), + _json_response({}), + _json_response({}), + ] + acl_obj.reload() + acl_obj.save() + acl_obj.save_predefined("publicRead") + acl_obj.clear() + assert isinstance(acl_obj, ACL) + + # HMACKeyMetadata + hmac_meta = HMACKeyMetadata(client, access_id="GOOG999") + hmac_meta.state = "INACTIVE" + session.request.side_effect = [ + _json_response({"accessId": "GOOG999", "state": "ACTIVE"}), + _json_response({"accessId": "GOOG999", "state": "ACTIVE"}), + _json_response({"accessId": "GOOG999", "state": "INACTIVE"}), + _json_response(None, status_code=204), + ] + assert hmac_meta.exists() is True + hmac_meta.reload() + hmac_meta.state = "INACTIVE" + hmac_meta.update() + hmac_meta.delete() + + # BucketNotification (including client validation error when notification_id is None) + notif_missing_id = BucketNotification(bucket) + for fn in ( + notif_missing_id.exists, + notif_missing_id.reload, + notif_missing_id.delete, + ): + with pytest.raises(ValueError): + fn() + + notif = BucketNotification(bucket, topic_name="my-topic", topic_project="my-proj") + session.request.side_effect = [ + _json_response( + { + "id": "notif-1", + "topic": "//pubsub.googleapis.com/projects/my-proj/topics/my-topic", + } + ), + _json_response({"id": "notif-1"}), + _json_response( + { + "id": "notif-1", + "topic": "//pubsub.googleapis.com/projects/my-proj/topics/my-topic", + } + ), + _json_response(None, status_code=204), + ] + notif.create() + assert notif.exists() is True + notif.reload() + notif.delete() + + pts = _collect_metrics(reader) + methods_seen = { + p.attributes["gcp.client.method"] for p in pts["gcp.storage.client.operations"] + } + expected_subset = { + "storage.Client.get_service_account_email", + "storage.Client.create_bucket", + "storage.Client.restore_bucket", + "storage.Client.create_hmac_key", + "storage.Client.get_hmac_key_metadata", + "storage.Client.download_blob_to_file", + "storage.Bucket.create", + "storage.Bucket.reload", + "storage.Bucket.update", + "storage.Bucket.patch", + "storage.Bucket.get_notification", + "storage.Bucket.delete_blob", + "storage.Bucket.delete_blobs", + "storage.Bucket.copy_blob", + "storage.Bucket.rename_blob", + "storage.Bucket.move_blob", + "storage.Bucket.restore_blob", + "storage.Bucket.get_iam_policy", + "storage.Bucket.set_iam_policy", + "storage.Bucket.test_iam_permissions", + "storage.Bucket.make_public", + "storage.Bucket.make_private", + "storage.Bucket.lock_retention_policy", + "storage.Bucket.delete", + "storage.BucketACL.reload", + "storage.BucketACL.save", + "storage.BucketACL.save_predefined", + "storage.BucketACL.clear", + "storage.DefaultObjectACL.reload", + "storage.DefaultObjectACL.save", + "storage.DefaultObjectACL.save_predefined", + "storage.DefaultObjectACL.clear", + "storage.ObjectACL.reload", + "storage.ObjectACL.save", + "storage.ObjectACL.save_predefined", + "storage.ObjectACL.clear", + "storage.HMACKeyMetadata.exists", + "storage.HMACKeyMetadata.reload", + "storage.HMACKeyMetadata.update", + "storage.HMACKeyMetadata.delete", + "storage.BucketNotification.create", + "storage.BucketNotification.exists", + "storage.BucketNotification.reload", + "storage.BucketNotification.delete", + } + assert expected_subset.issubset(methods_seen) + assert not any("_UserBucket" in m for m in methods_seen) + assert "gcp.storage.client.retries" not in pts + + +def test_batch_deferral_and_subresponse_semantics_on_bucket(metrics_env): + client, session, reader = metrics_env + bucket = client.bucket("my-bucket") + + # Empty batch finish -> CLIENT_VALIDATION_ERROR + empty_batch = client.batch() + with pytest.raises(ValueError): + empty_batch.finish() + pts_empty = _collect_metrics(reader) + assert ( + pts_empty["gcp.storage.client.operations"][0].attributes["error.type"] + == "CLIENT_VALIDATION_ERROR" + ) + + multipart_ok_and_403 = ( + b"--batch_boundary\r\n" + b"Content-Type: application/http\r\n" + b"Content-ID: \r\n\r\n" + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: application/json\r\n\r\n" + b'{"name": "my-bucket"}\r\n' + b"--batch_boundary\r\n" + b"Content-Type: application/http\r\n" + b"Content-ID: \r\n\r\n" + b"HTTP/1.1 403 Forbidden\r\n" + b"Content-Type: application/json\r\n\r\n" + b'{"error": {"message": "denied"}}\r\n' + b"--batch_boundary--\r\n" + ) + + # 1. raise_exception=False: queuing emits 0 metrics; finish emits Batch OK + session.request.return_value = _json_response( + multipart_ok_and_403, + status_code=200, + headers={"Content-Type": "multipart/mixed; boundary=batch_boundary"}, + ) + with client.batch(raise_exception=False): + bucket.patch() + bucket.delete() + pts_during = _collect_metrics(reader) + assert len(pts_during["gcp.storage.client.operations"]) == 1 + + pts_after_no_raise = _collect_metrics(reader) + assert len(pts_after_no_raise["gcp.storage.client.operations"]) == 2 + + # 2. raise_exception=True: subresponse 403 fails Batch with PERMISSION_DENIED + session.request.return_value = _json_response( + multipart_ok_and_403, + status_code=200, + headers={"Content-Type": "multipart/mixed; boundary=batch_boundary"}, + ) + with pytest.raises(Exception): + with client.batch(raise_exception=True): + bucket.patch() + bucket.delete() + + pts_final = _collect_metrics(reader) + batch_ops = pts_final["gcp.storage.client.operations"] + err_map = [p.attributes.get("error.type", "OK") for p in batch_ops] + assert "CLIENT_VALIDATION_ERROR" in err_map + assert "OK" in err_map + assert "PERMISSION_DENIED" in err_map + + http_att = pts_final["http.client.request.duration"][0] + assert http_att.attributes["url.template"] == "/batch/storage/v1" + assert http_att.attributes["http.request.method"] == "POST" + + +def test_lazy_acl_methods_attribution(metrics_env): + client, session, reader = metrics_env + bucket = client.bucket("my-bucket") + + acl_payload = {"items": [{"entity": "user-a@example.com", "role": "READER"}]} + lazy_cases = [ + ("__iter__", lambda acl: list(acl)), + ( + "entity_from_dict", + lambda acl: acl.entity_from_dict( + {"entity": "user-b@example.com", "role": "READER"} + ), + ), + ("has_entity", lambda acl: acl.has_entity("user-a@example.com")), + ("get_entity", lambda acl: acl.get_entity("user-a@example.com")), + ( + "add_entity", + lambda acl: acl.add_entity(_ACLEntity("user", "c@example.com")), + ), + ("entity", lambda acl: acl.entity("user", "a@example.com")), + ("user", lambda acl: acl.user("a@example.com")), + ("group", lambda acl: acl.group("g@example.com")), + ("domain", lambda acl: acl.domain("example.com")), + ("all", lambda acl: acl.all()), + ("all_authenticated", lambda acl: acl.all_authenticated()), + ("get_entities", lambda acl: acl.get_entities()), + ] + for _, fn in lazy_cases: + session.request.return_value = _json_response(acl_payload) + acl = BucketACL(bucket) + fn(acl) + fn(acl) # loaded=True fast path -> 0 additional metrics + + pts_acl = _collect_metrics(reader) + acl_ops = pts_acl["gcp.storage.client.operations"] + assert len(acl_ops) == len(lazy_cases) + assert {p.attributes["gcp.client.method"] for p in acl_ops} == { + f"storage.BucketACL.{name}" for name, _ in lazy_cases + } + assert all( + p.attributes["gcp.storage.operation"] == "ListBucketACLs" for p in acl_ops + )